diff --git a/doc/source/config/object_server_config.rst b/doc/source/config/object_server_config.rst
index 414be1dafb..b6b6b0991a 100644
--- a/doc/source/config/object_server_config.rst
+++ b/doc/source/config/object_server_config.rst
@@ -746,4 +746,19 @@ ionice_priority None I/O scheduling pri
priority of the process. Work only with
ionice_class.
Ignored if IOPRIO_CLASS_IDLE is set.
+delay_reaping_<ACCT> 0.0 A dynamic configuration option for
+ setting account level delay_reaping values.
+ The delay_reaping value is configured for
+ the account with the name placed in
+ <ACCT>. The object expirer will reap objects in
+ this account from disk only after this delay
+ following their x-delete-at time.
+delay_reaping_<ACCT>/<CNTR> 0.0 A dynamic configuration option for
+ setting container level delay_reaping values.
+ The delay_reaping value is configured for
+ the container with the account name placed
+ in <ACCT> and the container name in <CNTR>.
+ The object expirer will reap objects in this
+ container from disk only after this delay
+ following their x-delete-at time.
============================= =============================== ==========================================
diff --git a/doc/source/overview_expiring_objects.rst b/doc/source/overview_expiring_objects.rst
index 78d8d3e3b7..ef39d7ba74 100644
--- a/doc/source/overview_expiring_objects.rst
+++ b/doc/source/overview_expiring_objects.rst
@@ -55,6 +55,49 @@ it will then look for and use the ``/etc/swift/object-expirer.conf`` config.
The latter config file is considered deprecated and is searched for to aid
in cluster upgrades.
+Delay Reaping of Objects from Disk
+----------------------------------
+
+Swift's expiring object ``x-delete-at`` feature can be used to have the cluster
+reap user's objects automatically from disk on their behalf when they no longer
+want them stored in their account. In some cases it may be necessary to
+"intervene" in the expected expiration process to prevent accidental or
+premature data loss if an object marked for expiration should NOT be deleted
+immediately when it expires for whatever reason. In these cases
+``swift-object-expirer`` offers configuration of a ``delay_reaping`` value
+on accounts and containers, which provides a delay between when an object
+is marked for deletion, or expired, and when it is actually reaped from disk.
+When this is set in the object expirer config the object expirer leaves expired
+objects on disk (and in container listings) for the ``delay_reaping`` time.
+After this delay has passed objects will be reaped as normal.
+
+The ``delay_reaping`` value can be set either at an account level or a
+container level. When set at an account level, the object expirer will
+only reap objects within the account after the delay. A container level
+``delay_reaping`` works similarly for containers and overrides an account
+level ``delay_reaping`` value.
+
+The ``delay_reaping`` values are set in the ``[object-expirer]`` section in
+either the object-server or object-expirer config files. They are configured
+with dynamic config option names prefixed with ``delay_reaping_<ACCT>``
+at the account level and ``delay_reaping_<ACCT>/<CNTR>`` at the container
+level, with the ``delay_reaping`` value in seconds.
+
+Here is an example of ``delay_reaping`` configs in the``object-expirer``
+section in the ``object-server.conf``::
+
+ [object-expirer]
+ delay_reaping_AUTH_test = 300.0
+ delay_reaping_AUTH_test2 = 86400.0
+ delay_reaping_AUTH_test/test = 0.0
+ delay_reaping_AUTH_test/test2 = 600.0
+
+.. note::
+ A container level ``delay_reaping`` value does not require an account level
+ ``delay_reaping`` value but overrides the account level value for the same
+ account if it exists. By default, no ``delay_reaping`` value is configured
+ for any accounts or containers.
+
Upgrading impact: General Task Queue vs Legacy Queue
----------------------------------------------------
diff --git a/etc/object-expirer.conf-sample b/etc/object-expirer.conf-sample
index a83d9e0013..d722dcf9da 100644
--- a/etc/object-expirer.conf-sample
+++ b/etc/object-expirer.conf-sample
@@ -72,6 +72,27 @@
# queue.
# reclaim_age = 604800
#
+# The expirer can delay the reaping of expired objects on disk (and in
+# container listings) with an account level or container level delay_reaping
+# time.
+# After the delay_reaping time has passed objects will be reaped as normal.
+# You may configure this delay_reaping value in seconds with dynamic config
+# option names prefixed with delay_reaping_<ACCT> for account level delays
+# and delay_reaping_<ACCT>/<CNTR> for container level delays.
+# Special characters in <ACCT> or <CNTR> should be quoted.
+# The delay_reaping value should be a float value greater than or equal to
+# zero.
+# A container level delay_reaping does not require an account level
+# delay_reaping but overrides the account level delay_reaping for the same
+# account if it exists.
+# For example:
+# delay_reaping_AUTH_test = 300.0
+# delay_reaping_AUTH_test2 = 86400.0
+# delay_reaping_AUTH_test/test = 400.0
+# delay_reaping_AUTH_test/test2 = 600.0
+# N.B. By default no delay_reaping value is configured for any accounts or
+# containers.
+#
# recon_cache_path = /var/cache/swift
#
# You can set scheduling priority of processes. Niceness values range from -20
diff --git a/etc/object-server.conf-sample b/etc/object-server.conf-sample
index 67c879b1ac..a95dfb9952 100644
--- a/etc/object-server.conf-sample
+++ b/etc/object-server.conf-sample
@@ -692,10 +692,31 @@ use = egg:swift#backend_ratelimit
# ionice_class =
# ionice_priority =
#
-# Note: Put it at the beginning of the pipleline to profile all middleware. But
-# it is safer to put this after healthcheck.
+# The expirer can delay the reaping of expired objects on disk (and in
+# container listings) with an account level or container level delay_reaping
+# time.
+# After the delay_reaping time has passed objects will be reaped as normal.
+# You may configure this delay_reaping value in seconds with dynamic config
+# option names prefixed with delay_reaping_<ACCT> for account level delays
+# and delay_reaping_<ACCT>/<CNTR> for container level delays.
+# Special characters in <ACCT> or <CNTR> should be quoted.
+# The delay_reaping value should be a float value greater than or equal to
+# zero.
+# A container level delay_reaping does not require an account level
+# delay_reaping but overrides the account level delay_reaping for the same
+# account if it exists.
+# For example:
+# delay_reaping_AUTH_test = 300.0
+# delay_reaping_AUTH_test2 = 86400.0
+# delay_reaping_AUTH_test/test = 400.0
+# delay_reaping_AUTH_test/test2 = 600.0
+# N.B. By default no delay_reaping value is configured for any accounts or
+# containers.
+
[filter:xprofile]
use = egg:swift#xprofile
+# Note: Put it at the beginning of the pipleline to profile all middleware. But
+# it is safer to put this after healthcheck.
# This option enable you to switch profilers which should inherit from python
# standard profiler. Currently the supported value can be 'cProfile',
# 'eventlet.green.profile' etc.
diff --git a/swift/obj/expirer.py b/swift/obj/expirer.py
index c832cd63bd..6d8f864b47 100644
--- a/swift/obj/expirer.py
+++ b/swift/obj/expirer.py
@@ -14,6 +14,7 @@
# limitations under the License.
import six
+from six.moves import urllib
from random import random
from time import time
@@ -28,7 +29,7 @@ from swift.common.daemon import Daemon
from swift.common.internal_client import InternalClient, UnexpectedResponse
from swift.common.utils import get_logger, dump_recon_cache, split_path, \
Timestamp, config_true_value, normalize_delete_at_timestamp, \
- RateLimitedIterator, md5
+ RateLimitedIterator, md5, non_negative_float
from swift.common.http import HTTP_NOT_FOUND, HTTP_CONFLICT, \
HTTP_PRECONDITION_FAILED
from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH
@@ -66,6 +67,49 @@ def parse_task_obj(task_obj):
return timestamp, target_account, target_container, target_obj
+def read_conf_for_delay_reaping_times(conf):
+ delay_reaping_times = {}
+ for conf_key in conf:
+ delay_reaping_prefix = "delay_reaping_"
+ if not conf_key.startswith(delay_reaping_prefix):
+ continue
+ delay_reaping_key = urllib.parse.unquote(
+ conf_key[len(delay_reaping_prefix):])
+ if delay_reaping_key.strip('/') != delay_reaping_key:
+ raise ValueError(
+ '%s '
+ 'should be in the form delay_reaping_<account> '
+ 'or delay_reaping_<account>/<container> '
+ '(leading or trailing "/" is not allowed)' % conf_key)
+ try:
+ # If split_path fails, have multiple '/' or
+ # account name is invalid
+ account, container = split_path(
+ '/' + delay_reaping_key, 1, 2
+ )
+ except ValueError:
+ raise ValueError(
+ '%s '
+ 'should be in the form delay_reaping_<account> '
+ 'or delay_reaping_<account>/<container> '
+ '(at most one "/" is allowed)' % conf_key)
+ try:
+ delay_reaping_times[(account, container)] = non_negative_float(
+ conf.get(conf_key)
+ )
+ except ValueError:
+ raise ValueError(
+ '%s must be a float '
+ 'greater than or equal to 0' % conf_key)
+ return delay_reaping_times
+
+
+def get_delay_reaping(delay_reaping_times, target_account, target_container):
+ return delay_reaping_times.get(
+ (target_account, target_container),
+ delay_reaping_times.get((target_account, None), 0.0))
+
+
class ObjectExpirer(Daemon):
"""
Daemon that queries the internal hidden task accounts to discover objects
@@ -113,6 +157,8 @@ class ObjectExpirer(Daemon):
# with the tombstone reclaim age in the consistency engine.
self.reclaim_age = int(conf.get('reclaim_age', 604800))
+ self.delay_reaping_times = read_conf_for_delay_reaping_times(conf)
+
def read_conf_for_queue_access(self, swift):
self.expiring_objects_account = AUTO_CREATE_ACCOUNT_PREFIX + \
(self.conf.get('expiring_objects_account_name') or
@@ -246,6 +292,10 @@ class ObjectExpirer(Daemon):
break
yield task_container
+ def get_delay_reaping(self, target_account, target_container):
+ return get_delay_reaping(self.delay_reaping_times, target_account,
+ target_container)
+
def iter_task_to_expire(self, task_account_container_list,
my_index, divisor):
"""
@@ -267,17 +317,24 @@ class ObjectExpirer(Daemon):
self.logger.exception('Unexcepted error handling task %r' %
task_object)
continue
+ is_async = o.get('content_type') == ASYNC_DELETE_TYPE
+ delay_reaping = self.get_delay_reaping(target_account,
+ target_container)
+
if delete_timestamp > Timestamp.now():
- # we shouldn't yield the object that doesn't reach
+ # we shouldn't yield ANY more objects that can't reach
# the expiration date yet.
break
+ if delete_timestamp > Timestamp(time() - delay_reaping) \
+ and not is_async:
+ # we shouldn't yield the object during the delay
+ continue
# Only one expirer daemon assigned for one task
if self.hash_mod('%s/%s' % (task_container, task_object),
divisor) != my_index:
continue
- is_async = o.get('content_type') == ASYNC_DELETE_TYPE
yield {'task_account': task_account,
'task_container': task_container,
'task_object': task_object,
diff --git a/test/unit/obj/test_expirer.py b/test/unit/obj/test_expirer.py
index d433f8a8df..c3ef26dc9a 100644
--- a/test/unit/obj/test_expirer.py
+++ b/test/unit/obj/test_expirer.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
# Copyright (c) 2011 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -284,6 +285,221 @@ class TestObjectExpirer(TestCase):
x.get_process_values(vals)
self.assertEqual(str(ctx.exception), expected_msg)
+ def test_valid_delay_reaping(self):
+ conf = {}
+ x = expirer.ObjectExpirer(conf, swift=self.fake_swift)
+ self.assertEqual(x.delay_reaping_times, {})
+
+ conf = {
+ 'delay_reaping_a': 1.0,
+ }
+ x = expirer.ObjectExpirer(conf, swift=self.fake_swift)
+ self.assertEqual(x.delay_reaping_times, {('a', None): 1.0})
+
+ # allow delay_reaping to be 0
+ conf = {
+ 'delay_reaping_a': 0.0,
+ }
+ x = expirer.ObjectExpirer(conf, swift=self.fake_swift)
+ self.assertEqual(x.delay_reaping_times, {('a', None): 0.0})
+
+ conf = {
+ 'delay_reaping_a/b': 0.0,
+ }
+ x = expirer.ObjectExpirer(conf, swift=self.fake_swift)
+ self.assertEqual(x.delay_reaping_times, {('a', 'b'): 0.0})
+
+ # test configure multi-account delay_reaping
+ conf = {
+ 'delay_reaping_a': 1.0,
+ 'delay_reaping_b': '259200.0',
+ 'delay_reaping_AUTH_aBC': 999,
+ u'delay_reaping_AUTH_aBáC': 555,
+ }
+ x = expirer.ObjectExpirer(conf, swift=self.fake_swift)
+ self.assertEqual(x.delay_reaping_times, {
+ ('a', None): 1.0,
+ ('b', None): 259200.0,
+ ('AUTH_aBC', None): 999,
+ (u'AUTH_aBáC', None): 555,
+ })
+
+ # test configure multi-account delay_reaping with containers
+ conf = {
+ 'delay_reaping_a': 10.0,
+ 'delay_reaping_a/test': 1.0,
+ 'delay_reaping_b': '259200.0',
+ 'delay_reaping_AUTH_aBC/test2': 999,
+ u'delay_reaping_AUTH_aBáC/tést': 555,
+ 'delay_reaping_AUTH_test/special%0Achars%3Dare%20quoted': 777,
+ 'delay_reaping_AUTH_test/plus+signs+are+preserved': 888,
+ }
+ x = expirer.ObjectExpirer(conf, swift=self.fake_swift)
+ self.assertEqual(x.delay_reaping_times, {
+ ('a', None): 10.0,
+ ('a', 'test'): 1.0,
+ ('b', None): 259200.0,
+ ('AUTH_aBC', 'test2'): 999,
+ (u'AUTH_aBáC', u'tést'): 555,
+ ('AUTH_test', 'special\nchars=are quoted'): 777,
+ ('AUTH_test', 'plus+signs+are+preserved'): 888,
+ })
+
+ def test_invalid_delay_reaping_keys(self):
+ # there is no global delay_reaping
+ conf = {
+ 'delay_reaping': 0.0,
+ }
+ x = expirer.ObjectExpirer(conf, swift=self.fake_swift)
+ self.assertEqual(x.delay_reaping_times, {})
+
+ # Multiple "/" or invalid parsing
+ conf = {
+ 'delay_reaping_A_U_TH_foo_bar/my-container_name/with/slash': 60400,
+ }
+ with self.assertRaises(ValueError) as ctx:
+ expirer.ObjectExpirer(conf, swift=self.fake_swift)
+ self.assertEqual(
+ 'delay_reaping_A_U_TH_foo_bar/my-container_name/with/slash '
+ 'should be in the form delay_reaping_<account> '
+ 'or delay_reaping_<account>/<container> '
+ '(at most one "/" is allowed)',
+ str(ctx.exception))
+
+ # Can't sneak around it by escaping
+ conf = {
+ 'delay_reaping_AUTH_test/sneaky%2fsneaky': 60400,
+ }
+ with self.assertRaises(ValueError) as ctx:
+ expirer.ObjectExpirer(conf, swift=self.fake_swift)
+ self.assertEqual(
+ 'delay_reaping_AUTH_test/sneaky%2fsneaky '
+ 'should be in the form delay_reaping_<account> '
+ 'or delay_reaping_<account>/<container> '
+ '(at most one "/" is allowed)',
+ str(ctx.exception))
+
+ conf = {
+ 'delay_reaping_': 60400
+ }
+ with self.assertRaises(ValueError) as ctx:
+ expirer.ObjectExpirer(conf, swift=self.fake_swift)
+ self.assertEqual(
+ 'delay_reaping_ '
+ 'should be in the form delay_reaping_<account> '
+ 'or delay_reaping_<account>/<container> '
+ '(at most one "/" is allowed)',
+ str(ctx.exception))
+
+ # Leading and trailing "/"
+ conf = {
+ 'delay_reaping_/a': 60400,
+ }
+ with self.assertRaises(ValueError) as ctx:
+ expirer.ObjectExpirer(conf, swift=self.fake_swift)
+ self.assertEqual(
+ 'delay_reaping_/a '
+ 'should be in the form delay_reaping_<account> '
+ 'or delay_reaping_<account>/<container> '
+ '(leading or trailing "/" is not allowed)',
+ str(ctx.exception))
+
+ conf = {
+ 'delay_reaping_a/': 60400,
+ }
+ with self.assertRaises(ValueError) as ctx:
+ expirer.ObjectExpirer(conf, swift=self.fake_swift)
+ self.assertEqual(
+ 'delay_reaping_a/ '
+ 'should be in the form delay_reaping_<account> '
+ 'or delay_reaping_<account>/<container> '
+ '(leading or trailing "/" is not allowed)',
+ str(ctx.exception))
+
+ conf = {
+ 'delay_reaping_/a/c/': 60400,
+ }
+ with self.assertRaises(ValueError) as ctx:
+ expirer.ObjectExpirer(conf, swift=self.fake_swift)
+ self.assertEqual(
+ 'delay_reaping_/a/c/ '
+ 'should be in the form delay_reaping_<account> '
+ 'or delay_reaping_<account>/<container> '
+ '(leading or trailing "/" is not allowed)',
+ str(ctx.exception))
+
+ def test_invalid_delay_reaping_values(self):
+ # negative tests
+ conf = {
+ 'delay_reaping_a': -1.0,
+ }
+ with self.assertRaises(ValueError) as ctx:
+ expirer.ObjectExpirer(conf, swift=self.fake_swift)
+ self.assertEqual(
+ 'delay_reaping_a must be a float greater than or equal to 0',
+ str(ctx.exception))
+ conf = {
+ 'delay_reaping_a': '-259200.0'
+ }
+ with self.assertRaises(ValueError) as ctx:
+ expirer.ObjectExpirer(conf, swift=self.fake_swift)
+ self.assertEqual(
+ 'delay_reaping_a must be a float greater than or equal to 0',
+ str(ctx.exception))
+ conf = {
+ 'delay_reaping_a': 'foo'
+ }
+ with self.assertRaises(ValueError) as ctx:
+ expirer.ObjectExpirer(conf, swift=self.fake_swift)
+ self.assertEqual(
+ 'delay_reaping_a must be a float greater than or equal to 0',
+ str(ctx.exception))
+
+ # negative tests with containers
+ conf = {
+ 'delay_reaping_a/b': -100.0
+ }
+ with self.assertRaises(ValueError) as ctx:
+ expirer.ObjectExpirer(conf, swift=self.fake_swift)
+ self.assertEqual(
+ 'delay_reaping_a/b must be a float greater than or equal to 0',
+ str(ctx.exception))
+ conf = {
+ 'delay_reaping_a/b': '-259200.0'
+ }
+ with self.assertRaises(ValueError) as ctx:
+ expirer.ObjectExpirer(conf, swift=self.fake_swift)
+ self.assertEqual(
+ 'delay_reaping_a/b must be a float greater than or equal to 0',
+ str(ctx.exception))
+ conf = {
+ 'delay_reaping_a/b': 'foo'
+ }
+ with self.assertRaises(ValueError) as ctx:
+ expirer.ObjectExpirer(conf, swift=self.fake_swift)
+ self.assertEqual(
+ 'delay_reaping_a/b must be a float greater than or equal to 0',
+ str(ctx.exception))
+
+ def test_get_delay_reaping(self):
+ conf = {
+ 'delay_reaping_a': 1.0,
+ 'delay_reaping_a/test': 2.0,
+ 'delay_reaping_b': '259200.0',
+ 'delay_reaping_b/a': '0.0',
+ 'delay_reaping_c/test': '3.0'
+ }
+ x = expirer.ObjectExpirer(conf, swift=self.fake_swift)
+ self.assertEqual(1.0, x.get_delay_reaping('a', None))
+ self.assertEqual(1.0, x.get_delay_reaping('a', 'not-test'))
+ self.assertEqual(2.0, x.get_delay_reaping('a', 'test'))
+ self.assertEqual(259200.0, x.get_delay_reaping('b', None))
+ self.assertEqual(0.0, x.get_delay_reaping('b', 'a'))
+ self.assertEqual(259200.0, x.get_delay_reaping('b', 'test'))
+ self.assertEqual(3.0, x.get_delay_reaping('c', 'test'))
+ self.assertEqual(0.0, x.get_delay_reaping('c', 'not-test'))
+ self.assertEqual(0.0, x.get_delay_reaping('no-conf', 'test'))
+
def test_init_concurrency_too_small(self):
conf = {
'concurrency': 0,
@@ -746,6 +962,233 @@ class TestObjectExpirer(TestCase):
task_account_container_list, my_index, divisor)),
expected)
+ def test_iter_task_to_expire_with_delay_reaping(self):
+ aco_dict = {
+ '.expiring_objects': {
+ self.past_time: [
+ # tasks well past ready for execution
+ {'name': self.past_time + '-a0/c0/o0'},
+ {'name': self.past_time + '-a1/c1/o1'},
+ {'name': self.past_time + '-a1/c2/o2'},
+ ],
+ self.just_past_time: [
+ # tasks only just ready for execution
+ {'name': self.just_past_time + '-a0/c0/o0'},
+ {'name': self.just_past_time + '-a1/c1/o1'},
+ {'name': self.just_past_time + '-a1/c2/o2'},
+ ],
+ self.future_time: [
+ # tasks not yet ready for execution
+ {'name': self.future_time + '-a0/c0/o0'},
+ {'name': self.future_time + '-a1/c1/o1'},
+ {'name': self.future_time + '-a1/c2/o2'},
+ ],
+ }
+ }
+ fake_swift = FakeInternalClient(aco_dict)
+ # sanity, no accounts configured with delay_reaping
+ x = expirer.ObjectExpirer(self.conf, logger=self.logger,
+ swift=fake_swift)
+ # ... we expect tasks past time to yield
+ expected = [
+ self.make_task(self.past_time, target_path)
+ for target_path in (
+ swob.wsgi_to_str(tgt) for tgt in (
+ 'a0/c0/o0',
+ 'a1/c1/o1',
+ 'a1/c2/o2',
+ )
+ )
+ ] + [
+ self.make_task(self.just_past_time, target_path)
+ for target_path in (
+ swob.wsgi_to_str(tgt) for tgt in (
+ 'a0/c0/o0',
+ 'a1/c1/o1',
+ 'a1/c2/o2',
+ )
+ )
+ ]
+ task_account_container_list = [
+ ('.expiring_objects', self.past_time),
+ ('.expiring_objects', self.just_past_time),
+ ]
+ observed = list(x.iter_task_to_expire(
+ task_account_container_list, 0, 1))
+ self.assertEqual(expected, observed)
+
+ # configure delay for account a1
+ self.conf['delay_reaping_a1'] = 300.0
+ x = expirer.ObjectExpirer(self.conf, logger=self.logger,
+ swift=fake_swift)
+ # ... and we don't expect *recent* a1 tasks or future tasks
+ expected = [
+ self.make_task(self.past_time, target_path)
+ for target_path in (
+ swob.wsgi_to_str(tgt) for tgt in (
+ 'a0/c0/o0',
+ 'a1/c1/o1',
+ 'a1/c2/o2',
+ )
+ )
+ ] + [
+ self.make_task(self.just_past_time, target_path)
+ for target_path in (
+ swob.wsgi_to_str(tgt) for tgt in (
+ 'a0/c0/o0',
+ )
+ )
+ ]
+ observed = list(x.iter_task_to_expire(
+ task_account_container_list, 0, 1))
+ self.assertEqual(expected, observed)
+
+ # configure delay for account a1 and for account a1 and container c2
+ # container a1/c2 expires expires almost immediately
+ # but other containers in account a1 remain (a1/c1 and a1/c3)
+ self.conf['delay_reaping_a1'] = 300.0
+ self.conf['delay_reaping_a1/c2'] = 0.1
+ x = expirer.ObjectExpirer(self.conf, logger=self.logger,
+ swift=fake_swift)
+ # ... and we don't expect *recent* a1 tasks, excluding c2
+ # or future tasks
+ expected = [
+ self.make_task(self.past_time, target_path)
+ for target_path in (
+ swob.wsgi_to_str(tgt) for tgt in (
+ 'a0/c0/o0',
+ 'a1/c1/o1',
+ 'a1/c2/o2',
+ )
+ )
+ ] + [
+ self.make_task(self.just_past_time, target_path)
+ for target_path in (
+ swob.wsgi_to_str(tgt) for tgt in (
+ 'a0/c0/o0',
+ 'a1/c2/o2',
+ )
+ )
+ ]
+ observed = list(x.iter_task_to_expire(
+ task_account_container_list, 0, 1))
+ self.assertEqual(expected, observed)
+
+ # configure delay for account a1 and for account a1 and container c2
+ # container a1/c2 does not expire but others in account a1 do
+ self.conf['delay_reaping_a1'] = 0.1
+ self.conf['delay_reaping_a1/c2'] = 300.0
+ x = expirer.ObjectExpirer(self.conf, logger=self.logger,
+ swift=fake_swift)
+ # ... and we don't expect *recent* a1 tasks, excluding c2
+ # or future tasks
+ expected = [
+ self.make_task(self.past_time, target_path)
+ for target_path in (
+ swob.wsgi_to_str(tgt) for tgt in (
+ 'a0/c0/o0',
+ 'a1/c1/o1',
+ 'a1/c2/o2',
+ )
+ )
+ ] + [
+ self.make_task(self.just_past_time, target_path)
+ for target_path in (
+ swob.wsgi_to_str(tgt) for tgt in (
+ 'a0/c0/o0',
+ 'a1/c1/o1',
+ )
+ )
+ ]
+ observed = list(x.iter_task_to_expire(
+ task_account_container_list, 0, 1))
+ self.assertEqual(expected, observed)
+
+ def test_iter_task_to_expire_with_delay_reaping_is_async(self):
+ aco_dict = {
+ '.expiring_objects': {
+ self.past_time: [
+ # tasks well past ready for execution
+ {'name': self.past_time + '-a0/c0/o0',
+ 'content_type': 'application/async-deleted'},
+ {'name': self.past_time + '-a1/c1/o1',
+ 'content_type': 'application/async-deleted'},
+ {'name': self.past_time + '-a1/c2/o2',
+ 'content_type': 'application/async-deleted'},
+ ],
+ self.just_past_time: [
+ # tasks only just ready for execution
+ {'name': self.just_past_time + '-a0/c0/o0',
+ 'content_type': 'application/async-deleted'},
+ {'name': self.just_past_time + '-a1/c1/o1',
+ 'content_type': 'application/async-deleted'},
+ {'name': self.just_past_time + '-a1/c2/o2',
+ 'content_type': 'application/async-deleted'},
+ ],
+ self.future_time: [
+ # tasks not yet ready for execution
+ {'name': self.future_time + '-a0/c0/o0',
+ 'content_type': 'application/async-deleted'},
+ {'name': self.future_time + '-a1/c1/o1',
+ 'content_type': 'application/async-deleted'},
+ {'name': self.future_time + '-a1/c2/o2',
+ 'content_type': 'application/async-deleted'},
+ ],
+ }
+ }
+ fake_swift = FakeInternalClient(aco_dict)
+ # no accounts configured with delay_reaping
+ x = expirer.ObjectExpirer(self.conf, logger=self.logger,
+ swift=fake_swift)
+ # ... we expect all past async tasks to yield
+ expected = [
+ self.make_task(self.past_time, target_path, is_async_delete=True)
+ for target_path in (
+ swob.wsgi_to_str(tgt) for tgt in (
+ 'a0/c0/o0',
+ 'a1/c1/o1',
+ 'a1/c2/o2',
+ )
+ )
+ ] + [
+ self.make_task(self.just_past_time, target_path,
+ is_async_delete=True)
+ for target_path in (
+ swob.wsgi_to_str(tgt) for tgt in (
+ 'a0/c0/o0',
+ 'a1/c1/o1',
+ 'a1/c2/o2',
+ )
+ )
+ ]
+ task_account_container_list = [
+ ('.expiring_objects', self.past_time),
+ ('.expiring_objects', self.just_past_time),
+ ]
+ observed = list(x.iter_task_to_expire(
+ task_account_container_list, 0, 1))
+ self.assertEqual(expected, observed)
+
+ # configure delay for account a1
+ self.conf['delay_reaping_a1'] = 300.0
+ x = expirer.ObjectExpirer(self.conf, logger=self.logger,
+ swift=fake_swift)
+ # ... and we still expect all past async tasks to yield
+ observed = list(x.iter_task_to_expire(
+ task_account_container_list, 0, 1))
+ self.assertEqual(expected, observed)
+
+ # configure delays for all containers
+ self.conf['delay_reaping_a1/c0'] = 300.0
+ self.conf['delay_reaping_a1/c1'] = 300.0
+ self.conf['delay_reaping_a1/c2'] = 300.0
+ x = expirer.ObjectExpirer(self.conf, logger=self.logger,
+ swift=fake_swift)
+ # ... and we we still expect all past async tasks to yield
+ observed = list(x.iter_task_to_expire(
+ task_account_container_list, 0, 1))
+ self.assertEqual(expected, observed)
+
def test_run_once_unicode_problem(self):
requests = []