tests: realistic task_container names
Change-Id: Ie5d8b555489d28c3b901e5bbebdcecbde7bb3367
This commit is contained in:
Clay Gerrard
parent
82debd2cda
commit
99c629edb8
@ -99,6 +99,16 @@ class TestObjectExpirer(TestCase):
|
||||
maxDiff = None
|
||||
internal_client = None
|
||||
|
||||
def get_expirer_container(self, delete_at, target_account='a',
|
||||
target_container='c', target_object='o',
|
||||
expirer_divisor=86400):
|
||||
# the actual target a/c/o used only matters for consistent
|
||||
# distribution, tests typically only create one task container per-day,
|
||||
# but we want the task container names to be realistic
|
||||
return utils.get_expirer_container(
|
||||
delete_at, expirer_divisor,
|
||||
target_account, target_container, target_object)
|
||||
|
||||
def setUp(self):
|
||||
global not_sleep
|
||||
|
||||
@ -111,23 +121,32 @@ class TestObjectExpirer(TestCase):
|
||||
self.logger = debug_logger('test-expirer')
|
||||
|
||||
self.ts = make_timestamp_iter()
|
||||
self.empty_time = str(int(time() - 864000))
|
||||
self.past_time = str(int(time() - 86400))
|
||||
self.just_past_time = str(int(time() - 1))
|
||||
self.future_time = str(int(time() + 86400))
|
||||
|
||||
now = int(time())
|
||||
|
||||
self.empty_time = str(now - 864000)
|
||||
self.empty_time_container = self.get_expirer_container(self.empty_time)
|
||||
self.past_time = str(now - 86400)
|
||||
self.past_time_container = self.get_expirer_container(self.past_time)
|
||||
self.just_past_time = str(now - 1)
|
||||
self.just_past_time_container = self.get_expirer_container(
|
||||
self.just_past_time)
|
||||
self.future_time = str(now + 86400)
|
||||
self.future_time_container = self.get_expirer_container(
|
||||
self.future_time)
|
||||
# Dummy task queue for test
|
||||
self.fake_swift = FakeInternalClient({
|
||||
'.expiring_objects': {
|
||||
# this task container will be checked
|
||||
self.empty_time: [],
|
||||
self.past_time: [
|
||||
self.empty_time_container: [],
|
||||
self.past_time_container: [
|
||||
# tasks ready for execution
|
||||
self.past_time + '-a0/c0/o0',
|
||||
self.past_time + '-a1/c1/o1',
|
||||
self.past_time + '-a2/c2/o2',
|
||||
self.past_time + '-a3/c3/o3',
|
||||
self.past_time + '-a4/c4/o4'],
|
||||
self.just_past_time: [
|
||||
self.just_past_time_container: [
|
||||
self.just_past_time + '-a5/c5/o5',
|
||||
self.just_past_time + '-a6/c6/o6',
|
||||
self.just_past_time + '-a7/c7/o7',
|
||||
@ -138,7 +157,7 @@ class TestObjectExpirer(TestCase):
|
||||
# *trying* to delete the container
|
||||
self.future_time + '-a10/c10/o10'],
|
||||
# this task container will be skipped
|
||||
self.future_time: [
|
||||
self.future_time_container: [
|
||||
self.future_time + '-a11/c11/o11']}
|
||||
})
|
||||
self.expirer = expirer.ObjectExpirer(self.conf, logger=self.logger,
|
||||
@ -537,10 +556,10 @@ class TestObjectExpirer(TestCase):
|
||||
deleted_objects = {
|
||||
con: sorted(o_set) for con, o_set in deleted_objects.items()}
|
||||
expected = {
|
||||
self.past_time: [
|
||||
self.past_time_container: [
|
||||
self.past_time + '-' + target_path
|
||||
for target_path in self.expired_target_paths[self.past_time]],
|
||||
self.just_past_time: [
|
||||
self.just_past_time_container: [
|
||||
self.just_past_time + '-' + target_path
|
||||
for target_path
|
||||
in self.expired_target_paths[self.just_past_time]]}
|
||||
@ -647,10 +666,11 @@ class TestObjectExpirer(TestCase):
|
||||
assert_parse_task_obj('1000-a/c/o', 1000, 'a', 'c', 'o')
|
||||
assert_parse_task_obj('0000-acc/con/obj', 0, 'acc', 'con', 'obj')
|
||||
|
||||
def make_task(self, delete_at, target, is_async_delete=False):
|
||||
def make_task(self, task_container, delete_at, target,
|
||||
is_async_delete=False):
|
||||
return {
|
||||
'task_account': '.expiring_objects',
|
||||
'task_container': delete_at,
|
||||
'task_container': task_container,
|
||||
'task_object': delete_at + '-' + target,
|
||||
'delete_timestamp': Timestamp(delete_at),
|
||||
'target_path': target,
|
||||
@ -660,55 +680,63 @@ class TestObjectExpirer(TestCase):
|
||||
def test_round_robin_order(self):
|
||||
x = expirer.ObjectExpirer(self.conf, logger=self.logger,
|
||||
swift=self.fake_swift)
|
||||
|
||||
def make_task(delete_at, target_path, is_async_delete=False):
|
||||
a, c, o = utils.split_path('/' + target_path, 1, 3, True)
|
||||
task_container = self.get_expirer_container(
|
||||
delete_at, a, c or 'c', o or 'o')
|
||||
return self.make_task(task_container, delete_at, target_path,
|
||||
is_async_delete=is_async_delete)
|
||||
|
||||
task_con_obj_list = [
|
||||
# objects in 0000 timestamp container
|
||||
self.make_task('0000', 'a/c0/o0'),
|
||||
self.make_task('0000', 'a/c0/o1'),
|
||||
make_task('0000', 'a/c0/o0'),
|
||||
make_task('0000', 'a/c0/o1'),
|
||||
# objects in 0001 timestamp container
|
||||
self.make_task('0001', 'a/c1/o0'),
|
||||
self.make_task('0001', 'a/c1/o1'),
|
||||
make_task('0001', 'a/c1/o0'),
|
||||
make_task('0001', 'a/c1/o1'),
|
||||
# objects in 0002 timestamp container
|
||||
self.make_task('0002', 'a/c2/o0'),
|
||||
self.make_task('0002', 'a/c2/o1'),
|
||||
make_task('0002', 'a/c2/o0'),
|
||||
make_task('0002', 'a/c2/o1'),
|
||||
]
|
||||
result = list(x.round_robin_order(task_con_obj_list))
|
||||
|
||||
# sorted by popping one object to delete for each target_container
|
||||
expected = [
|
||||
self.make_task('0000', 'a/c0/o0'),
|
||||
self.make_task('0001', 'a/c1/o0'),
|
||||
self.make_task('0002', 'a/c2/o0'),
|
||||
self.make_task('0000', 'a/c0/o1'),
|
||||
self.make_task('0001', 'a/c1/o1'),
|
||||
self.make_task('0002', 'a/c2/o1'),
|
||||
make_task('0000', 'a/c0/o0'),
|
||||
make_task('0001', 'a/c1/o0'),
|
||||
make_task('0002', 'a/c2/o0'),
|
||||
make_task('0000', 'a/c0/o1'),
|
||||
make_task('0001', 'a/c1/o1'),
|
||||
make_task('0002', 'a/c2/o1'),
|
||||
]
|
||||
self.assertEqual(expected, result)
|
||||
|
||||
# task containers have some task objects with invalid target paths
|
||||
task_con_obj_list = [
|
||||
# objects in 0000 timestamp container
|
||||
self.make_task('0000', 'invalid0'),
|
||||
self.make_task('0000', 'a/c0/o0'),
|
||||
self.make_task('0000', 'a/c0/o1'),
|
||||
make_task('0000', 'invalid0'),
|
||||
make_task('0000', 'a/c0/o0'),
|
||||
make_task('0000', 'a/c0/o1'),
|
||||
# objects in 0001 timestamp container
|
||||
self.make_task('0001', 'a/c1/o0'),
|
||||
self.make_task('0001', 'invalid1'),
|
||||
self.make_task('0001', 'a/c1/o1'),
|
||||
make_task('0001', 'a/c1/o0'),
|
||||
make_task('0001', 'invalid1'),
|
||||
make_task('0001', 'a/c1/o1'),
|
||||
# objects in 0002 timestamp container
|
||||
self.make_task('0002', 'a/c2/o0'),
|
||||
self.make_task('0002', 'a/c2/o1'),
|
||||
self.make_task('0002', 'invalid2'),
|
||||
make_task('0002', 'a/c2/o0'),
|
||||
make_task('0002', 'a/c2/o1'),
|
||||
make_task('0002', 'invalid2'),
|
||||
]
|
||||
result = list(x.round_robin_order(task_con_obj_list))
|
||||
|
||||
# the invalid task objects are ignored
|
||||
expected = [
|
||||
self.make_task('0000', 'a/c0/o0'),
|
||||
self.make_task('0001', 'a/c1/o0'),
|
||||
self.make_task('0002', 'a/c2/o0'),
|
||||
self.make_task('0000', 'a/c0/o1'),
|
||||
self.make_task('0001', 'a/c1/o1'),
|
||||
self.make_task('0002', 'a/c2/o1'),
|
||||
make_task('0000', 'a/c0/o0'),
|
||||
make_task('0001', 'a/c1/o0'),
|
||||
make_task('0002', 'a/c2/o0'),
|
||||
make_task('0000', 'a/c0/o1'),
|
||||
make_task('0001', 'a/c1/o1'),
|
||||
make_task('0002', 'a/c2/o1'),
|
||||
]
|
||||
self.assertEqual(expected, result)
|
||||
|
||||
@ -716,51 +744,51 @@ class TestObjectExpirer(TestCase):
|
||||
# the same timestamp container
|
||||
task_con_obj_list = [
|
||||
# objects in 0000 timestamp container
|
||||
self.make_task('0000', 'a/c0/o0'),
|
||||
self.make_task('0000', 'a/c0/o1'),
|
||||
self.make_task('0000', 'a/c2/o2'),
|
||||
self.make_task('0000', 'a/c2/o3'),
|
||||
make_task('0000', 'a/c0/o0'),
|
||||
make_task('0000', 'a/c0/o1'),
|
||||
make_task('0000', 'a/c2/o2'),
|
||||
make_task('0000', 'a/c2/o3'),
|
||||
# objects in 0001 timestamp container
|
||||
self.make_task('0001', 'a/c0/o2'),
|
||||
self.make_task('0001', 'a/c0/o3'),
|
||||
self.make_task('0001', 'a/c1/o0'),
|
||||
self.make_task('0001', 'a/c1/o1'),
|
||||
make_task('0001', 'a/c0/o2'),
|
||||
make_task('0001', 'a/c0/o3'),
|
||||
make_task('0001', 'a/c1/o0'),
|
||||
make_task('0001', 'a/c1/o1'),
|
||||
# objects in 0002 timestamp container
|
||||
self.make_task('0002', 'a/c2/o0'),
|
||||
self.make_task('0002', 'a/c2/o1'),
|
||||
make_task('0002', 'a/c2/o0'),
|
||||
make_task('0002', 'a/c2/o1'),
|
||||
]
|
||||
result = list(x.round_robin_order(task_con_obj_list))
|
||||
|
||||
# so we go around popping by *target* container, not *task* container
|
||||
expected = [
|
||||
self.make_task('0000', 'a/c0/o0'),
|
||||
self.make_task('0001', 'a/c1/o0'),
|
||||
self.make_task('0000', 'a/c2/o2'),
|
||||
self.make_task('0000', 'a/c0/o1'),
|
||||
self.make_task('0001', 'a/c1/o1'),
|
||||
self.make_task('0000', 'a/c2/o3'),
|
||||
self.make_task('0001', 'a/c0/o2'),
|
||||
self.make_task('0002', 'a/c2/o0'),
|
||||
self.make_task('0001', 'a/c0/o3'),
|
||||
self.make_task('0002', 'a/c2/o1'),
|
||||
make_task('0000', 'a/c0/o0'),
|
||||
make_task('0001', 'a/c1/o0'),
|
||||
make_task('0000', 'a/c2/o2'),
|
||||
make_task('0000', 'a/c0/o1'),
|
||||
make_task('0001', 'a/c1/o1'),
|
||||
make_task('0000', 'a/c2/o3'),
|
||||
make_task('0001', 'a/c0/o2'),
|
||||
make_task('0002', 'a/c2/o0'),
|
||||
make_task('0001', 'a/c0/o3'),
|
||||
make_task('0002', 'a/c2/o1'),
|
||||
]
|
||||
self.assertEqual(expected, result)
|
||||
|
||||
# all of the work to be done could be for different target containers
|
||||
task_con_obj_list = [
|
||||
# objects in 0000 timestamp container
|
||||
self.make_task('0000', 'a/c0/o'),
|
||||
self.make_task('0000', 'a/c1/o'),
|
||||
self.make_task('0000', 'a/c2/o'),
|
||||
self.make_task('0000', 'a/c3/o'),
|
||||
make_task('0000', 'a/c0/o'),
|
||||
make_task('0000', 'a/c1/o'),
|
||||
make_task('0000', 'a/c2/o'),
|
||||
make_task('0000', 'a/c3/o'),
|
||||
# objects in 0001 timestamp container
|
||||
self.make_task('0001', 'a/c4/o'),
|
||||
self.make_task('0001', 'a/c5/o'),
|
||||
self.make_task('0001', 'a/c6/o'),
|
||||
self.make_task('0001', 'a/c7/o'),
|
||||
make_task('0001', 'a/c4/o'),
|
||||
make_task('0001', 'a/c5/o'),
|
||||
make_task('0001', 'a/c6/o'),
|
||||
make_task('0001', 'a/c7/o'),
|
||||
# objects in 0002 timestamp container
|
||||
self.make_task('0002', 'a/c8/o'),
|
||||
self.make_task('0002', 'a/c9/o'),
|
||||
make_task('0002', 'a/c8/o'),
|
||||
make_task('0002', 'a/c9/o'),
|
||||
]
|
||||
result = list(x.round_robin_order(task_con_obj_list))
|
||||
|
||||
@ -841,10 +869,12 @@ class TestObjectExpirer(TestCase):
|
||||
side_effect=fake_ratelimiter):
|
||||
x.run_once()
|
||||
self.assertEqual(calls, [([
|
||||
self.make_task(self.past_time, target_path)
|
||||
self.make_task(self.past_time_container, self.past_time,
|
||||
target_path)
|
||||
for target_path in self.expired_target_paths[self.past_time]
|
||||
] + [
|
||||
self.make_task(self.just_past_time, target_path)
|
||||
self.make_task(self.just_past_time_container, self.just_past_time,
|
||||
target_path)
|
||||
for target_path in self.expired_target_paths[self.just_past_time]
|
||||
], 2)])
|
||||
|
||||
@ -867,7 +897,9 @@ class TestObjectExpirer(TestCase):
|
||||
divisor = 1
|
||||
|
||||
# empty container gets deleted inline
|
||||
task_account_container_list = [('.expiring_objects', self.empty_time)]
|
||||
task_account_container_list = [
|
||||
('.expiring_objects', self.empty_time_container)
|
||||
]
|
||||
with mock.patch.object(self.expirer.swift, 'delete_container') \
|
||||
as mock_delete_container:
|
||||
self.assertEqual(
|
||||
@ -875,13 +907,30 @@ class TestObjectExpirer(TestCase):
|
||||
task_account_container_list, my_index, divisor)),
|
||||
[])
|
||||
self.assertEqual(mock_delete_container.mock_calls, [
|
||||
mock.call('.expiring_objects', self.empty_time,
|
||||
mock.call('.expiring_objects', self.empty_time_container,
|
||||
acceptable_statuses=(2, 404, 409))])
|
||||
|
||||
task_account_container_list = [('.expiring_objects', self.past_time)]
|
||||
# 404 (account/container list race) gets deleted inline
|
||||
task_account_container_list = [
|
||||
('.expiring_objects', 'does-not-matter')
|
||||
]
|
||||
with mock.patch.object(self.expirer.swift, 'delete_container') \
|
||||
as mock_delete_container:
|
||||
self.assertEqual(
|
||||
list(self.expirer.iter_task_to_expire(
|
||||
task_account_container_list, my_index, divisor)),
|
||||
[])
|
||||
self.assertEqual(mock_delete_container.mock_calls, [
|
||||
mock.call('.expiring_objects', 'does-not-matter',
|
||||
acceptable_statuses=(2, 404, 409))])
|
||||
|
||||
# ready containers are processed
|
||||
task_account_container_list = [
|
||||
('.expiring_objects', self.past_time_container)]
|
||||
|
||||
expected = [
|
||||
self.make_task(self.past_time, target_path)
|
||||
self.make_task(self.past_time_container, self.past_time,
|
||||
target_path)
|
||||
for target_path in self.expired_target_paths[self.past_time]]
|
||||
|
||||
with mock.patch.object(self.expirer.swift, 'delete_container') \
|
||||
@ -895,9 +944,9 @@ class TestObjectExpirer(TestCase):
|
||||
|
||||
# the task queue has invalid task object
|
||||
invalid_aco_dict = deepcopy(self.fake_swift.aco_dict)
|
||||
invalid_aco_dict['.expiring_objects'][self.past_time].insert(
|
||||
invalid_aco_dict['.expiring_objects'][self.past_time_container].insert(
|
||||
0, self.past_time + '-invalid0')
|
||||
invalid_aco_dict['.expiring_objects'][self.past_time].insert(
|
||||
invalid_aco_dict['.expiring_objects'][self.past_time_container].insert(
|
||||
5, self.past_time + '-invalid1')
|
||||
invalid_fake_swift = FakeInternalClient(invalid_aco_dict)
|
||||
x = expirer.ObjectExpirer(self.conf, logger=self.logger,
|
||||
@ -913,7 +962,7 @@ class TestObjectExpirer(TestCase):
|
||||
async_delete_aco_dict = {
|
||||
'.expiring_objects': {
|
||||
# this task container will be checked
|
||||
self.past_time: [
|
||||
self.past_time_container: [
|
||||
# tasks ready for execution
|
||||
{'name': self.past_time + '-a0/c0/o0',
|
||||
'content_type': 'application/async-deleted'},
|
||||
@ -944,33 +993,35 @@ class TestObjectExpirer(TestCase):
|
||||
swift=async_delete_fake_swift)
|
||||
|
||||
expected = [
|
||||
self.make_task(self.past_time, target_path,
|
||||
is_async_delete=True)
|
||||
self.make_task(self.past_time_container, self.past_time,
|
||||
target_path, is_async_delete=True)
|
||||
for target_path in (
|
||||
self.expired_target_paths[self.past_time] +
|
||||
self.expired_target_paths[self.just_past_time])]
|
||||
self.expired_target_paths[self.just_past_time]
|
||||
)
|
||||
]
|
||||
|
||||
self.assertEqual(
|
||||
list(x.iter_task_to_expire(
|
||||
task_account_container_list, my_index, divisor)),
|
||||
expected)
|
||||
found = list(x.iter_task_to_expire(
|
||||
task_account_container_list, my_index, divisor))
|
||||
|
||||
self.assertEqual(expected, found)
|
||||
|
||||
def test_iter_task_to_expire_with_delay_reaping(self):
|
||||
aco_dict = {
|
||||
'.expiring_objects': {
|
||||
self.past_time: [
|
||||
self.past_time_container: [
|
||||
# tasks well past ready for execution
|
||||
{'name': self.past_time + '-a0/c0/o0'},
|
||||
{'name': self.past_time + '-a1/c1/o1'},
|
||||
{'name': self.past_time + '-a1/c2/o2'},
|
||||
],
|
||||
self.just_past_time: [
|
||||
self.just_past_time_container: [
|
||||
# tasks only just ready for execution
|
||||
{'name': self.just_past_time + '-a0/c0/o0'},
|
||||
{'name': self.just_past_time + '-a1/c1/o1'},
|
||||
{'name': self.just_past_time + '-a1/c2/o2'},
|
||||
],
|
||||
self.future_time: [
|
||||
self.future_time_container: [
|
||||
# tasks not yet ready for execution
|
||||
{'name': self.future_time + '-a0/c0/o0'},
|
||||
{'name': self.future_time + '-a1/c1/o1'},
|
||||
@ -984,7 +1035,8 @@ class TestObjectExpirer(TestCase):
|
||||
swift=fake_swift)
|
||||
# ... we expect tasks past time to yield
|
||||
expected = [
|
||||
self.make_task(self.past_time, target_path)
|
||||
self.make_task(self.past_time_container, self.past_time,
|
||||
target_path)
|
||||
for target_path in (
|
||||
swob.wsgi_to_str(tgt) for tgt in (
|
||||
'a0/c0/o0',
|
||||
@ -993,7 +1045,8 @@ class TestObjectExpirer(TestCase):
|
||||
)
|
||||
)
|
||||
] + [
|
||||
self.make_task(self.just_past_time, target_path)
|
||||
self.make_task(self.just_past_time_container, self.just_past_time,
|
||||
target_path)
|
||||
for target_path in (
|
||||
swob.wsgi_to_str(tgt) for tgt in (
|
||||
'a0/c0/o0',
|
||||
@ -1003,8 +1056,8 @@ class TestObjectExpirer(TestCase):
|
||||
)
|
||||
]
|
||||
task_account_container_list = [
|
||||
('.expiring_objects', self.past_time),
|
||||
('.expiring_objects', self.just_past_time),
|
||||
('.expiring_objects', self.past_time_container),
|
||||
('.expiring_objects', self.just_past_time_container),
|
||||
]
|
||||
observed = list(x.iter_task_to_expire(
|
||||
task_account_container_list, 0, 1))
|
||||
@ -1016,7 +1069,8 @@ class TestObjectExpirer(TestCase):
|
||||
swift=fake_swift)
|
||||
# ... and we don't expect *recent* a1 tasks or future tasks
|
||||
expected = [
|
||||
self.make_task(self.past_time, target_path)
|
||||
self.make_task(self.past_time_container, self.past_time,
|
||||
target_path)
|
||||
for target_path in (
|
||||
swob.wsgi_to_str(tgt) for tgt in (
|
||||
'a0/c0/o0',
|
||||
@ -1025,7 +1079,8 @@ class TestObjectExpirer(TestCase):
|
||||
)
|
||||
)
|
||||
] + [
|
||||
self.make_task(self.just_past_time, target_path)
|
||||
self.make_task(self.just_past_time_container, self.just_past_time,
|
||||
target_path)
|
||||
for target_path in (
|
||||
swob.wsgi_to_str(tgt) for tgt in (
|
||||
'a0/c0/o0',
|
||||
@ -1046,7 +1101,8 @@ class TestObjectExpirer(TestCase):
|
||||
# ... and we don't expect *recent* a1 tasks, excluding c2
|
||||
# or future tasks
|
||||
expected = [
|
||||
self.make_task(self.past_time, target_path)
|
||||
self.make_task(self.past_time_container, self.past_time,
|
||||
target_path)
|
||||
for target_path in (
|
||||
swob.wsgi_to_str(tgt) for tgt in (
|
||||
'a0/c0/o0',
|
||||
@ -1055,7 +1111,8 @@ class TestObjectExpirer(TestCase):
|
||||
)
|
||||
)
|
||||
] + [
|
||||
self.make_task(self.just_past_time, target_path)
|
||||
self.make_task(self.just_past_time_container, self.just_past_time,
|
||||
target_path)
|
||||
for target_path in (
|
||||
swob.wsgi_to_str(tgt) for tgt in (
|
||||
'a0/c0/o0',
|
||||
@ -1076,7 +1133,8 @@ class TestObjectExpirer(TestCase):
|
||||
# ... and we don't expect *recent* a1 tasks, excluding c2
|
||||
# or future tasks
|
||||
expected = [
|
||||
self.make_task(self.past_time, target_path)
|
||||
self.make_task(self.past_time_container, self.past_time,
|
||||
target_path)
|
||||
for target_path in (
|
||||
swob.wsgi_to_str(tgt) for tgt in (
|
||||
'a0/c0/o0',
|
||||
@ -1085,7 +1143,8 @@ class TestObjectExpirer(TestCase):
|
||||
)
|
||||
)
|
||||
] + [
|
||||
self.make_task(self.just_past_time, target_path)
|
||||
self.make_task(self.just_past_time_container, self.just_past_time,
|
||||
target_path)
|
||||
for target_path in (
|
||||
swob.wsgi_to_str(tgt) for tgt in (
|
||||
'a0/c0/o0',
|
||||
@ -1100,7 +1159,7 @@ class TestObjectExpirer(TestCase):
|
||||
def test_iter_task_to_expire_with_delay_reaping_is_async(self):
|
||||
aco_dict = {
|
||||
'.expiring_objects': {
|
||||
self.past_time: [
|
||||
self.past_time_container: [
|
||||
# tasks 86400s past ready for execution
|
||||
{'name': self.past_time + '-a0/c0/o00',
|
||||
'content_type': 'application/async-deleted'},
|
||||
@ -1115,7 +1174,7 @@ class TestObjectExpirer(TestCase):
|
||||
{'name': self.past_time + '-a1/c1/o05',
|
||||
'content_type': 'text/plain'},
|
||||
],
|
||||
self.just_past_time: [
|
||||
self.just_past_time_container: [
|
||||
# tasks only just 1s ready for execution
|
||||
{'name': self.just_past_time + '-a0/c0/o06',
|
||||
'content_type': 'application/async-deleted'},
|
||||
@ -1130,7 +1189,7 @@ class TestObjectExpirer(TestCase):
|
||||
{'name': self.just_past_time + '-a1/c1/o11',
|
||||
'content_type': 'text/plain'},
|
||||
],
|
||||
self.future_time: [
|
||||
self.future_time_container: [
|
||||
# tasks not yet ready for execution
|
||||
{'name': self.future_time + '-a0/c0/o12',
|
||||
'content_type': 'application/async-deleted'},
|
||||
@ -1153,8 +1212,8 @@ class TestObjectExpirer(TestCase):
|
||||
swift=fake_swift)
|
||||
# ... we expect all past async tasks to yield
|
||||
expected = [
|
||||
self.make_task(self.past_time, swob.wsgi_to_str(tgt),
|
||||
is_async_delete=is_async)
|
||||
self.make_task(self.past_time_container, self.past_time,
|
||||
swob.wsgi_to_str(tgt), is_async_delete=is_async)
|
||||
for (tgt, is_async) in (
|
||||
('a0/c0/o00', True),
|
||||
('a0/c0/o01', False), # a0 no delay
|
||||
@ -1164,8 +1223,8 @@ class TestObjectExpirer(TestCase):
|
||||
('a1/c1/o05', False), # c1 short delay
|
||||
)
|
||||
] + [
|
||||
self.make_task(self.just_past_time, swob.wsgi_to_str(tgt),
|
||||
is_async_delete=is_async)
|
||||
self.make_task(self.just_past_time_container, self.just_past_time,
|
||||
swob.wsgi_to_str(tgt), is_async_delete=is_async)
|
||||
for (tgt, is_async) in (
|
||||
('a0/c0/o06', True),
|
||||
('a0/c0/o07', False), # a0 no delay
|
||||
@ -1182,9 +1241,9 @@ class TestObjectExpirer(TestCase):
|
||||
swift=fake_swift)
|
||||
# ... and we still expect all past async tasks to yield
|
||||
task_account_container_list = [
|
||||
('.expiring_objects', self.past_time),
|
||||
('.expiring_objects', self.just_past_time),
|
||||
('.expiring_objects', self.future_time),
|
||||
('.expiring_objects', self.past_time_container),
|
||||
('.expiring_objects', self.just_past_time_container),
|
||||
('.expiring_objects', self.future_time_container),
|
||||
]
|
||||
observed = list(x.iter_task_to_expire(
|
||||
task_account_container_list, 0, 1))
|
||||
@ -1208,9 +1267,9 @@ class TestObjectExpirer(TestCase):
|
||||
|
||||
# iter_objects is called only for past_time, not future_time
|
||||
self.assertEqual(mock_method.call_args_list, [
|
||||
mock.call('.expiring_objects', self.empty_time),
|
||||
mock.call('.expiring_objects', self.past_time),
|
||||
mock.call('.expiring_objects', self.just_past_time)])
|
||||
mock.call('.expiring_objects', self.empty_time_container),
|
||||
mock.call('.expiring_objects', self.past_time_container),
|
||||
mock.call('.expiring_objects', self.just_past_time_container)])
|
||||
|
||||
def test_object_timestamp_break(self):
|
||||
with mock.patch.object(self.expirer, 'delete_actual_object') \
|
||||
@ -1249,10 +1308,10 @@ class TestObjectExpirer(TestCase):
|
||||
# all tasks are popped from the queue
|
||||
self.assertEqual(
|
||||
mock_method.call_args_list,
|
||||
[mock.call('.expiring_objects', self.past_time,
|
||||
[mock.call('.expiring_objects', self.past_time_container,
|
||||
self.past_time + '-' + target_path)
|
||||
for target_path in self.expired_target_paths[self.past_time]] +
|
||||
[mock.call('.expiring_objects', self.just_past_time,
|
||||
[mock.call('.expiring_objects', self.just_past_time_container,
|
||||
self.just_past_time + '-' + target_path)
|
||||
for target_path
|
||||
in self.expired_target_paths[self.just_past_time]])
|
||||
@ -1305,11 +1364,11 @@ class TestObjectExpirer(TestCase):
|
||||
|
||||
self.assertEqual(error_lines, [
|
||||
'Exception while deleting container .expiring_objects %s failed '
|
||||
'to delete container: ' % self.empty_time
|
||||
'to delete container: ' % self.empty_time_container
|
||||
] + [
|
||||
'Exception while deleting object %s %s %s '
|
||||
'failed to delete actual object: ' % (
|
||||
'.expiring_objects', self.just_past_time,
|
||||
'.expiring_objects', self.just_past_time_container,
|
||||
self.just_past_time + '-' + target_path)
|
||||
for target_path in self.expired_target_paths[self.just_past_time]
|
||||
])
|
||||
@ -1319,7 +1378,7 @@ class TestObjectExpirer(TestCase):
|
||||
'Pass completed in 0s; 5 objects expired',
|
||||
])
|
||||
self.assertEqual(mock_pop.mock_calls, [
|
||||
mock.call('.expiring_objects', self.past_time,
|
||||
mock.call('.expiring_objects', self.past_time_container,
|
||||
self.past_time + '-' + target_path)
|
||||
for target_path in self.expired_target_paths[self.past_time]
|
||||
])
|
||||
|
||||
@ -45,10 +45,9 @@ from test.debug_logger import debug_logger
|
||||
from test.unit import mocked_http_conn, \
|
||||
make_timestamp_iter, DEFAULT_TEST_EC_TYPE, skip_if_no_xattrs, \
|
||||
connect_tcp, readuntil2crlfs, patch_policies, encode_frag_archive_bodies, \
|
||||
mock_check_drive
|
||||
mock_check_drive, FakeRing
|
||||
from swift.obj import server as object_server
|
||||
from swift.obj import updater
|
||||
from swift.obj import diskfile
|
||||
from swift.obj import updater, diskfile
|
||||
from swift.common import utils, bufferedhttp, http_protocol
|
||||
from swift.common.header_key_dict import HeaderKeyDict
|
||||
from swift.common.utils import hash_path, mkdirs, normalize_timestamp, \
|
||||
@ -160,6 +159,7 @@ class TestObjectController(BaseTestCase):
|
||||
|
||||
self.ts = make_timestamp_iter()
|
||||
self.ec_policies = [p for p in POLICIES if p.policy_type == EC_POLICY]
|
||||
self.container_ring = FakeRing()
|
||||
|
||||
def tearDown(self):
|
||||
"""Tear down for testing swift.object.server.ObjectController"""
|
||||
@ -1369,22 +1369,36 @@ class TestObjectController(BaseTestCase):
|
||||
resp = req.get_response(self.object_controller)
|
||||
self.assertEqual(resp.status_int, 412)
|
||||
|
||||
def _update_delete_at_headers(self, headers, a='a', c='c', o='o',
|
||||
node_count=1):
|
||||
delete_at = headers['X-Delete-At']
|
||||
delete_at_container = utils.get_expirer_container(delete_at, 84600,
|
||||
a, c, o)
|
||||
part, nodes = self.container_ring.get_nodes(
|
||||
'.expiring_objects', delete_at_container)
|
||||
# proxy assigns each replica a node, index 0 for test stability
|
||||
nodes = nodes[:node_count]
|
||||
headers.update({
|
||||
'X-Delete-At': str(delete_at),
|
||||
'X-Delete-At-Container': delete_at_container,
|
||||
'X-Delete-At-Partition': str(part),
|
||||
'X-Delete-At-Host': ','.join('%(ip)s:%(port)s' % n for n in nodes),
|
||||
'X-Delete-At-Device': ','.join(n['device'] for n in nodes),
|
||||
})
|
||||
return headers
|
||||
|
||||
def test_PUT_if_none_match_but_expired(self):
|
||||
inital_put = next(self.ts)
|
||||
put_before_expire = next(self.ts)
|
||||
delete_at_timestamp = int(next(self.ts))
|
||||
put_after_expire = next(self.ts)
|
||||
delete_at_container = str(
|
||||
delete_at_timestamp /
|
||||
self.object_controller.expiring_objects_container_divisor *
|
||||
self.object_controller.expiring_objects_container_divisor)
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'X-Timestamp': inital_put.normal,
|
||||
headers=self._update_delete_at_headers({
|
||||
'X-Timestamp': inital_put.normal,
|
||||
'X-Delete-At': str(delete_at_timestamp),
|
||||
'X-Delete-At-Container': delete_at_container,
|
||||
'Content-Length': '4',
|
||||
'Content-Type': 'application/octet-stream'})
|
||||
'Content-Type': 'application/octet-stream'}))
|
||||
req.body = 'TEST'
|
||||
resp = req.get_response(self.object_controller)
|
||||
self.assertEqual(resp.status_int, 201)
|
||||
@ -5607,7 +5621,6 @@ class TestObjectController(BaseTestCase):
|
||||
self.conf, self.object_controller.logger)
|
||||
policy = random.choice(list(POLICIES))
|
||||
self.object_controller.expiring_objects_account = 'exp'
|
||||
self.object_controller.expiring_objects_container_divisor = 60
|
||||
|
||||
http_connect_args = []
|
||||
|
||||
@ -5637,10 +5650,8 @@ class TestObjectController(BaseTestCase):
|
||||
|
||||
return SuccessfulFakeConn()
|
||||
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c/o',
|
||||
environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'X-Timestamp': '12345',
|
||||
req_headers = {
|
||||
'X-Timestamp': '12345',
|
||||
'Content-Type': 'application/burrito',
|
||||
'Content-Length': '0',
|
||||
'X-Backend-Storage-Policy-Index': int(policy),
|
||||
@ -5648,14 +5659,11 @@ class TestObjectController(BaseTestCase):
|
||||
'X-Container-Host': '1.2.3.4:5',
|
||||
'X-Container-Device': 'sdb1',
|
||||
'X-Delete-At': 9999999999,
|
||||
'X-Delete-At-Container': '9999999960',
|
||||
'X-Delete-At-Host': "10.1.1.1:6201,10.2.2.2:6202",
|
||||
'X-Delete-At-Partition': '6237',
|
||||
'X-Delete-At-Device': 'sdp,sdq'})
|
||||
|
||||
with mock.patch.object(
|
||||
}
|
||||
self._update_delete_at_headers(req_headers, node_count=2)
|
||||
req = Request.blank('/sda1/p/a/c/o', method='PUT', headers=req_headers)
|
||||
with fake_spawn(), mock.patch.object(
|
||||
object_server, 'http_connect', fake_http_connect):
|
||||
with fake_spawn():
|
||||
resp = req.get_response(self.object_controller)
|
||||
|
||||
self.assertEqual(resp.status_int, 201)
|
||||
@ -5681,13 +5689,18 @@ class TestObjectController(BaseTestCase):
|
||||
'user-agent': 'object-server %d' % os.getpid(),
|
||||
'X-Backend-Storage-Policy-Index': int(policy),
|
||||
'x-trans-id': '-'})})
|
||||
expected_hosts = [h.split(':') for h in
|
||||
req_headers['X-Delete-At-Host'].split(',')]
|
||||
expected_devs = [d for d in
|
||||
req_headers['X-Delete-At-Device'].split(',')]
|
||||
self.assertEqual(
|
||||
http_connect_args[1],
|
||||
{'ipaddr': '10.1.1.1',
|
||||
'port': '6201',
|
||||
'path': '/exp/9999999960/9999999999-a/c/o',
|
||||
'device': 'sdp',
|
||||
'partition': '6237',
|
||||
{'ipaddr': expected_hosts[0][0],
|
||||
'port': expected_hosts[0][1],
|
||||
'path': ('/exp/%s/9999999999-a/c/o' %
|
||||
req_headers['X-Delete-At-Container']),
|
||||
'device': expected_devs[0],
|
||||
'partition': req_headers['X-Delete-At-Partition'],
|
||||
'method': 'PUT',
|
||||
'ssl': False,
|
||||
'headers': HeaderKeyDict({
|
||||
@ -5702,11 +5715,12 @@ class TestObjectController(BaseTestCase):
|
||||
'x-trans-id': '-'})})
|
||||
self.assertEqual(
|
||||
http_connect_args[2],
|
||||
{'ipaddr': '10.2.2.2',
|
||||
'port': '6202',
|
||||
'path': '/exp/9999999960/9999999999-a/c/o',
|
||||
'device': 'sdq',
|
||||
'partition': '6237',
|
||||
{'ipaddr': expected_hosts[1][0],
|
||||
'port': expected_hosts[1][1],
|
||||
'path': ('/exp/%s/9999999999-a/c/o' %
|
||||
req_headers['X-Delete-At-Container']),
|
||||
'device': expected_devs[1],
|
||||
'partition': req_headers['X-Delete-At-Partition'],
|
||||
'method': 'PUT',
|
||||
'ssl': False,
|
||||
'headers': HeaderKeyDict({
|
||||
@ -5825,26 +5839,19 @@ class TestObjectController(BaseTestCase):
|
||||
put_timestamp = next(self.ts).internal
|
||||
delete_at_timestamp = utils.normalize_delete_at_timestamp(
|
||||
next(self.ts).normal)
|
||||
delete_at_container = (
|
||||
int(delete_at_timestamp) //
|
||||
self.object_controller.expiring_objects_container_divisor *
|
||||
self.object_controller.expiring_objects_container_divisor)
|
||||
headers = {
|
||||
req_headers = {
|
||||
'Content-Type': 'text/plain',
|
||||
'X-Timestamp': put_timestamp,
|
||||
'X-Container-Host': '10.0.0.1:6201',
|
||||
'X-Container-Device': 'sda1',
|
||||
'X-Container-Partition': 'p',
|
||||
'X-Delete-At': delete_at_timestamp,
|
||||
'X-Delete-At-Container': delete_at_container,
|
||||
'X-Delete-At-Partition': 'p',
|
||||
'X-Delete-At-Host': '10.0.0.2:6202',
|
||||
'X-Delete-At-Device': 'sda1',
|
||||
'X-Backend-Storage-Policy-Index': int(policy)}
|
||||
self._update_delete_at_headers(req_headers)
|
||||
if policy.policy_type == EC_POLICY:
|
||||
headers['X-Object-Sysmeta-Ec-Frag-Index'] = '2'
|
||||
req_headers['X-Object-Sysmeta-Ec-Frag-Index'] = '2'
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c/o', method='PUT', body=b'', headers=headers)
|
||||
'/sda1/p/a/c/o', method='PUT', body=b'', headers=req_headers)
|
||||
with mocked_http_conn(
|
||||
500, 500, give_connect=capture_updates) as fake_conn:
|
||||
with fake_spawn():
|
||||
@ -5857,11 +5864,15 @@ class TestObjectController(BaseTestCase):
|
||||
delete_at_update, container_update = container_updates
|
||||
# delete_at_update
|
||||
ip, port, method, path, headers = delete_at_update
|
||||
self.assertEqual(ip, '10.0.0.2')
|
||||
self.assertEqual(port, '6202')
|
||||
expected_ip, expected_port = req_headers['X-Delete-At-Host'].split(':')
|
||||
self.assertEqual(ip, expected_ip)
|
||||
self.assertEqual(port, expected_port)
|
||||
self.assertEqual(method, 'PUT')
|
||||
self.assertEqual(path, '/sda1/p/.expiring_objects/%s/%s-a/c/o' %
|
||||
(delete_at_container, delete_at_timestamp))
|
||||
self.assertEqual(path, '/%s/%s/.expiring_objects/%s/%s-a/c/o' % (
|
||||
req_headers['X-Delete-At-Device'],
|
||||
req_headers['X-Delete-At-Partition'],
|
||||
req_headers['X-Delete-At-Container'],
|
||||
req_headers['X-Delete-At']))
|
||||
expected = {
|
||||
'X-Timestamp': put_timestamp,
|
||||
# system account storage policy is 0
|
||||
@ -6655,23 +6666,28 @@ class TestObjectController(BaseTestCase):
|
||||
given_args.extend(args)
|
||||
|
||||
self.object_controller.async_update = fake_async_update
|
||||
req_headers = {
|
||||
'X-Timestamp': '1',
|
||||
'X-Trans-Id': '1234',
|
||||
'X-Delete-At': '2',
|
||||
'X-Backend-Storage-Policy-Index': str(int(policy)),
|
||||
}
|
||||
self._update_delete_at_headers(req_headers)
|
||||
req = Request.blank(
|
||||
'/v1/a/c/o',
|
||||
environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'X-Timestamp': 1,
|
||||
'X-Trans-Id': '1234',
|
||||
'X-Delete-At-Container': '0',
|
||||
'X-Delete-At-Host': '127.0.0.1:1234',
|
||||
'X-Delete-At-Partition': '3',
|
||||
'X-Delete-At-Device': 'sdc1',
|
||||
'X-Backend-Storage-Policy-Index': int(policy)})
|
||||
headers=req_headers)
|
||||
self.object_controller.delete_at_update('PUT', 2, 'a', 'c', 'o',
|
||||
req, 'sda1', policy)
|
||||
self.assertEqual(
|
||||
given_args, [
|
||||
'PUT', '.expiring_objects', '0000000000', '0000000002-a/c/o',
|
||||
'127.0.0.1:1234',
|
||||
'3', 'sdc1', HeaderKeyDict({
|
||||
'PUT', '.expiring_objects',
|
||||
req_headers['X-Delete-At-Container'],
|
||||
'0000000002-a/c/o',
|
||||
req_headers['X-Delete-At-Host'],
|
||||
req_headers['X-Delete-At-Partition'],
|
||||
req_headers['X-Delete-At-Device'],
|
||||
HeaderKeyDict({
|
||||
# the .expiring_objects account is always policy-0
|
||||
'X-Backend-Storage-Policy-Index': 0,
|
||||
'x-size': '0',
|
||||
@ -6735,18 +6751,23 @@ class TestObjectController(BaseTestCase):
|
||||
|
||||
self.object_controller.async_update = fake_async_update
|
||||
self.object_controller.logger = self.logger
|
||||
delete_at = time()
|
||||
req_headers = {
|
||||
'X-Timestamp': 1,
|
||||
'X-Trans-Id': '1234',
|
||||
'X-Delete-At': delete_at,
|
||||
'X-Backend-Storage-Policy-Index': int(policy),
|
||||
}
|
||||
self._update_delete_at_headers(req_headers)
|
||||
req_headers.pop('X-Delete-At-Host')
|
||||
req = Request.blank(
|
||||
'/v1/a/c/o',
|
||||
environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'X-Timestamp': 1,
|
||||
'X-Trans-Id': '1234',
|
||||
'X-Delete-At-Container': '0',
|
||||
'X-Delete-At-Partition': '3',
|
||||
'X-Delete-At-Device': 'sdc1',
|
||||
'X-Backend-Storage-Policy-Index': int(policy)})
|
||||
self.object_controller.delete_at_update('PUT', 2, 'a', 'c', 'o',
|
||||
headers=req_headers)
|
||||
self.object_controller.delete_at_update('PUT', delete_at,
|
||||
'a', 'c', 'o',
|
||||
req, 'sda1', policy)
|
||||
self.assertFalse(self.logger.get_lines_for_level('warning'))
|
||||
self.assertEqual({}, self.logger.all_log_lines())
|
||||
self.assertEqual(given_args, [])
|
||||
|
||||
def test_delete_at_update_put_with_info_but_empty_host(self):
|
||||
@ -6761,12 +6782,14 @@ class TestObjectController(BaseTestCase):
|
||||
|
||||
self.object_controller.async_update = fake_async_update
|
||||
self.object_controller.logger = self.logger
|
||||
delete_at_container = utils.get_expirer_container(
|
||||
'1', 84600, 'a', 'c', 'o')
|
||||
req = Request.blank(
|
||||
'/v1/a/c/o',
|
||||
environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'X-Timestamp': 1,
|
||||
'X-Trans-Id': '1234',
|
||||
'X-Delete-At-Container': '0',
|
||||
'X-Delete-At-Container': delete_at_container,
|
||||
'X-Delete-At-Host': '',
|
||||
'X-Backend-Storage-Policy-Index': int(policy)})
|
||||
self.object_controller.delete_at_update('PUT', 2, 'a', 'c', 'o',
|
||||
@ -6975,17 +6998,13 @@ class TestObjectController(BaseTestCase):
|
||||
# Start off with an existing object that will expire
|
||||
now = time()
|
||||
delete_at_timestamp = int(now + 100)
|
||||
delete_at_container = str(
|
||||
delete_at_timestamp /
|
||||
self.object_controller.expiring_objects_container_divisor *
|
||||
self.object_controller.expiring_objects_container_divisor)
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'X-Timestamp': normalize_timestamp(now),
|
||||
headers=self._update_delete_at_headers({
|
||||
'X-Timestamp': normalize_timestamp(now),
|
||||
'X-Delete-At': str(delete_at_timestamp),
|
||||
'X-Delete-At-Container': delete_at_container,
|
||||
'Content-Length': '4',
|
||||
'Content-Type': 'application/octet-stream'})
|
||||
'Content-Type': 'application/octet-stream'}))
|
||||
req.body = 'TEST'
|
||||
resp = req.get_response(self.object_controller)
|
||||
self.assertEqual(resp.status_int, 201)
|
||||
@ -7032,17 +7051,13 @@ class TestObjectController(BaseTestCase):
|
||||
# We have an object that expires in the future
|
||||
now = time()
|
||||
delete_at_timestamp = int(now + 100)
|
||||
delete_at_container = str(
|
||||
delete_at_timestamp /
|
||||
self.object_controller.expiring_objects_container_divisor *
|
||||
self.object_controller.expiring_objects_container_divisor)
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'X-Timestamp': normalize_timestamp(now),
|
||||
headers=self._update_delete_at_headers({
|
||||
'X-Timestamp': normalize_timestamp(now),
|
||||
'X-Delete-At': str(delete_at_timestamp),
|
||||
'X-Delete-At-Container': delete_at_container,
|
||||
'Content-Length': '4',
|
||||
'Content-Type': 'application/octet-stream'})
|
||||
'Content-Type': 'application/octet-stream'}))
|
||||
req.body = b'TEST'
|
||||
resp = req.get_response(self.object_controller)
|
||||
self.assertEqual(resp.status_int, 201)
|
||||
@ -7089,19 +7104,15 @@ class TestObjectController(BaseTestCase):
|
||||
# We have an object that expires in the future
|
||||
now = time()
|
||||
delete_at_timestamp = int(now + 100)
|
||||
delete_at_container = str(
|
||||
delete_at_timestamp /
|
||||
self.object_controller.expiring_objects_container_divisor *
|
||||
self.object_controller.expiring_objects_container_divisor)
|
||||
|
||||
# PUT the object
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'X-Timestamp': normalize_timestamp(now),
|
||||
headers=self._update_delete_at_headers({
|
||||
'X-Timestamp': normalize_timestamp(now),
|
||||
'X-Delete-At': str(delete_at_timestamp),
|
||||
'X-Delete-At-Container': delete_at_container,
|
||||
'Content-Length': '4',
|
||||
'Content-Type': 'application/octet-stream'})
|
||||
'Content-Type': 'application/octet-stream'}))
|
||||
req.body = b'TEST'
|
||||
resp = req.get_response(self.object_controller)
|
||||
self.assertEqual(resp.status_int, 201)
|
||||
@ -7140,19 +7151,15 @@ class TestObjectController(BaseTestCase):
|
||||
def test_POST_with_x_backend_open_expired(self):
|
||||
now = time()
|
||||
delete_at_timestamp = int(now + 100)
|
||||
delete_at_container = str(
|
||||
delete_at_timestamp /
|
||||
self.object_controller.expiring_objects_container_divisor *
|
||||
self.object_controller.expiring_objects_container_divisor)
|
||||
|
||||
# Create the object at x-delete-at
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'X-Timestamp': normalize_timestamp(now),
|
||||
headers=self._update_delete_at_headers({
|
||||
'X-Timestamp': normalize_timestamp(now),
|
||||
'X-Delete-At': str(delete_at_timestamp),
|
||||
'X-Delete-At-Container': delete_at_container,
|
||||
'Content-Length': '4',
|
||||
'Content-Type': 'application/octet-stream'})
|
||||
'Content-Type': 'application/octet-stream'}))
|
||||
req.body = 'TEST'
|
||||
resp = req.get_response(self.object_controller)
|
||||
self.assertEqual(resp.status_int, 201)
|
||||
@ -7164,9 +7171,10 @@ class TestObjectController(BaseTestCase):
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c/o',
|
||||
environ={'REQUEST_METHOD': 'POST'},
|
||||
headers={'X-Timestamp': normalize_timestamp(the_time),
|
||||
'x-delete-at': str(new_delete_at_timestamp),
|
||||
'x-backend-open-expired': 'true'})
|
||||
headers=self._update_delete_at_headers({
|
||||
'X-Timestamp': normalize_timestamp(the_time),
|
||||
'X-Delete-At': str(new_delete_at_timestamp),
|
||||
'x-backend-open-expired': 'true'}))
|
||||
resp = req.get_response(self.object_controller)
|
||||
self.assertEqual(resp.status_int, 202)
|
||||
|
||||
@ -7208,19 +7216,15 @@ class TestObjectController(BaseTestCase):
|
||||
def test_POST_with_x_backend_replication(self):
|
||||
now = time()
|
||||
delete_at_timestamp = int(now + 100)
|
||||
delete_at_container = str(
|
||||
delete_at_timestamp /
|
||||
self.object_controller.expiring_objects_container_divisor *
|
||||
self.object_controller.expiring_objects_container_divisor)
|
||||
|
||||
# Create object with future x-delete-at
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'X-Timestamp': normalize_timestamp(now),
|
||||
headers=self._update_delete_at_headers({
|
||||
'X-Timestamp': normalize_timestamp(now),
|
||||
'X-Delete-At': str(delete_at_timestamp),
|
||||
'X-Delete-At-Container': delete_at_container,
|
||||
'Content-Length': '4',
|
||||
'Content-Type': 'application/octet-stream'})
|
||||
'Content-Type': 'application/octet-stream'}))
|
||||
req.body = 'TEST'
|
||||
resp = req.get_response(self.object_controller)
|
||||
self.assertEqual(resp.status_int, 201)
|
||||
@ -7232,9 +7236,10 @@ class TestObjectController(BaseTestCase):
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c/o',
|
||||
environ={'REQUEST_METHOD': 'POST'},
|
||||
headers={'X-Timestamp': normalize_timestamp(the_time),
|
||||
headers=self._update_delete_at_headers({
|
||||
'X-Timestamp': normalize_timestamp(the_time),
|
||||
'x-backend-replication': 'true',
|
||||
'x-delete-at': str(new_delete_at_timestamp)})
|
||||
'X-Delete-At': str(new_delete_at_timestamp)}))
|
||||
resp = req.get_response(self.object_controller)
|
||||
self.assertEqual(resp.status_int, 202)
|
||||
|
||||
@ -7244,27 +7249,24 @@ class TestObjectController(BaseTestCase):
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c/o',
|
||||
environ={'REQUEST_METHOD': 'POST'},
|
||||
headers={'X-Timestamp': normalize_timestamp(the_time),
|
||||
'x-delete-at': str(delete_at_timestamp + 101)})
|
||||
headers=self._update_delete_at_headers({
|
||||
'X-Timestamp': normalize_timestamp(the_time),
|
||||
'X-Delete-At': str(delete_at_timestamp + 101)}))
|
||||
resp = req.get_response(self.object_controller)
|
||||
self.assertEqual(resp.status_int, 202)
|
||||
|
||||
def test_POST_invalid_headers(self):
|
||||
now = time()
|
||||
delete_at_timestamp = int(now + 100)
|
||||
delete_at_container = str(
|
||||
delete_at_timestamp /
|
||||
self.object_controller.expiring_objects_container_divisor *
|
||||
self.object_controller.expiring_objects_container_divisor)
|
||||
|
||||
# Create the object at x-delete-at
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'X-Timestamp': normalize_timestamp(now),
|
||||
headers=self._update_delete_at_headers({
|
||||
'X-Timestamp': normalize_timestamp(now),
|
||||
'X-Delete-At': str(delete_at_timestamp),
|
||||
'X-Delete-At-Container': delete_at_container,
|
||||
'Content-Length': '4',
|
||||
'Content-Type': 'application/octet-stream'})
|
||||
'Content-Type': 'application/octet-stream'}))
|
||||
req.body = 'TEST'
|
||||
resp = req.get_response(self.object_controller)
|
||||
self.assertEqual(resp.status_int, 201)
|
||||
@ -7275,9 +7277,10 @@ class TestObjectController(BaseTestCase):
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c/o',
|
||||
environ={'REQUEST_METHOD': 'POST'},
|
||||
headers={'X-Timestamp': normalize_timestamp(the_time),
|
||||
headers=self._update_delete_at_headers({
|
||||
'X-Timestamp': normalize_timestamp(the_time),
|
||||
'x-backend-open-expired': 'true',
|
||||
'x-delete-at': str(delete_at_timestamp - 50)})
|
||||
'X-Delete-At': str(delete_at_timestamp - 50)}))
|
||||
resp = req.get_response(self.object_controller)
|
||||
self.assertEqual(resp.status_int, 400)
|
||||
|
||||
@ -7287,9 +7290,10 @@ class TestObjectController(BaseTestCase):
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c/o',
|
||||
environ={'REQUEST_METHOD': 'POST'},
|
||||
headers={'X-Timestamp': normalize_timestamp(the_time),
|
||||
headers=self._update_delete_at_headers({
|
||||
'X-Timestamp': normalize_timestamp(the_time),
|
||||
'x-open-expired': 'true',
|
||||
'x-delete-at': str(delete_at_timestamp + 100)})
|
||||
'X-Delete-At': str(delete_at_timestamp + 100)}))
|
||||
resp = req.get_response(self.object_controller)
|
||||
self.assertEqual(resp.status_int, 404)
|
||||
|
||||
@ -7299,17 +7303,13 @@ class TestObjectController(BaseTestCase):
|
||||
put_time = test_time
|
||||
delete_time = test_time + 1
|
||||
delete_at_timestamp = int(test_time + 10000)
|
||||
delete_at_container = str(
|
||||
delete_at_timestamp /
|
||||
self.object_controller.expiring_objects_container_divisor *
|
||||
self.object_controller.expiring_objects_container_divisor)
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'X-Timestamp': normalize_timestamp(put_time),
|
||||
headers=self._update_delete_at_headers({
|
||||
'X-Timestamp': normalize_timestamp(put_time),
|
||||
'X-Delete-At': str(delete_at_timestamp),
|
||||
'X-Delete-At-Container': delete_at_container,
|
||||
'Content-Length': '4',
|
||||
'Content-Type': 'application/octet-stream'})
|
||||
'Content-Type': 'application/octet-stream'}))
|
||||
req.body = 'TEST'
|
||||
|
||||
# Mock out async_update so we don't get any async_pending files.
|
||||
@ -7339,19 +7339,15 @@ class TestObjectController(BaseTestCase):
|
||||
put_time = test_time
|
||||
delete_time = test_time + 1
|
||||
delete_at_timestamp = int(test_time + 10000)
|
||||
delete_at_container = str(
|
||||
delete_at_timestamp /
|
||||
self.object_controller.expiring_objects_container_divisor *
|
||||
self.object_controller.expiring_objects_container_divisor)
|
||||
|
||||
def do_test(if_delete_at, expected_status):
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'X-Timestamp': normalize_timestamp(put_time),
|
||||
headers=self._update_delete_at_headers({
|
||||
'X-Timestamp': normalize_timestamp(put_time),
|
||||
'X-Delete-At': str(delete_at_timestamp),
|
||||
'X-Delete-At-Container': delete_at_container,
|
||||
'Content-Length': '4',
|
||||
'Content-Type': 'application/octet-stream'})
|
||||
'Content-Type': 'application/octet-stream'}))
|
||||
req.body = 'TEST'
|
||||
|
||||
# Mock out async_update so we don't get any async_pending files.
|
||||
@ -7398,17 +7394,13 @@ class TestObjectController(BaseTestCase):
|
||||
def test_DELETE_but_expired(self):
|
||||
test_time = time() + 10000
|
||||
delete_at_timestamp = int(test_time + 100)
|
||||
delete_at_container = str(
|
||||
delete_at_timestamp /
|
||||
self.object_controller.expiring_objects_container_divisor *
|
||||
self.object_controller.expiring_objects_container_divisor)
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'X-Timestamp': normalize_timestamp(test_time - 2000),
|
||||
headers=self._update_delete_at_headers({
|
||||
'X-Timestamp': normalize_timestamp(test_time - 2000),
|
||||
'X-Delete-At': str(delete_at_timestamp),
|
||||
'X-Delete-At-Container': delete_at_container,
|
||||
'Content-Length': '4',
|
||||
'Content-Type': 'application/octet-stream'})
|
||||
'Content-Type': 'application/octet-stream'}))
|
||||
req.body = 'TEST'
|
||||
resp = req.get_response(self.object_controller)
|
||||
self.assertEqual(resp.status_int, 201)
|
||||
@ -7428,17 +7420,13 @@ class TestObjectController(BaseTestCase):
|
||||
delete_at_timestamp = str(delete_at_time)
|
||||
expired_time = delete_at_time + 1
|
||||
expired_timestamp = normalize_timestamp(expired_time)
|
||||
delete_at_container = str(
|
||||
delete_at_time /
|
||||
self.object_controller.expiring_objects_container_divisor *
|
||||
self.object_controller.expiring_objects_container_divisor)
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'X-Timestamp': test_timestamp,
|
||||
headers=self._update_delete_at_headers({
|
||||
'X-Timestamp': test_timestamp,
|
||||
'X-Delete-At': delete_at_timestamp,
|
||||
'X-Delete-At-Container': delete_at_container,
|
||||
'Content-Length': '4',
|
||||
'Content-Type': 'application/octet-stream'})
|
||||
'Content-Type': 'application/octet-stream'}))
|
||||
req.body = 'TEST'
|
||||
resp = req.get_response(self.object_controller)
|
||||
self.assertEqual(resp.status_int, 201)
|
||||
@ -7543,17 +7531,13 @@ class TestObjectController(BaseTestCase):
|
||||
self.assertEqual(resp.status_int, 204)
|
||||
|
||||
delete_at_timestamp = int(test_time - 1)
|
||||
delete_at_container = str(
|
||||
delete_at_timestamp /
|
||||
self.object_controller.expiring_objects_container_divisor *
|
||||
self.object_controller.expiring_objects_container_divisor)
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'X-Timestamp': normalize_timestamp(test_time - 97),
|
||||
headers=self._update_delete_at_headers({
|
||||
'X-Timestamp': normalize_timestamp(test_time - 97),
|
||||
'X-Delete-At': str(delete_at_timestamp),
|
||||
'X-Delete-At-Container': delete_at_container,
|
||||
'Content-Length': '4',
|
||||
'Content-Type': 'application/octet-stream'})
|
||||
'Content-Type': 'application/octet-stream'}))
|
||||
req.body = 'TEST'
|
||||
resp = req.get_response(self.object_controller)
|
||||
self.assertEqual(resp.status_int, 201)
|
||||
@ -7574,17 +7558,13 @@ class TestObjectController(BaseTestCase):
|
||||
self.assertEqual(resp.status_int, 204)
|
||||
|
||||
delete_at_timestamp = int(test_time - 1)
|
||||
delete_at_container = str(
|
||||
delete_at_timestamp /
|
||||
self.object_controller.expiring_objects_container_divisor *
|
||||
self.object_controller.expiring_objects_container_divisor)
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'X-Timestamp': normalize_timestamp(test_time - 94),
|
||||
headers=self._update_delete_at_headers({
|
||||
'X-Timestamp': normalize_timestamp(test_time - 94),
|
||||
'X-Delete-At': str(delete_at_timestamp),
|
||||
'X-Delete-At-Container': delete_at_container,
|
||||
'Content-Length': '4',
|
||||
'Content-Type': 'application/octet-stream'})
|
||||
'Content-Type': 'application/octet-stream'}))
|
||||
req.body = 'TEST'
|
||||
resp = req.get_response(self.object_controller)
|
||||
self.assertEqual(resp.status_int, 201)
|
||||
@ -7659,17 +7639,13 @@ class TestObjectController(BaseTestCase):
|
||||
put_time = test_time
|
||||
overwrite_time = test_time + 1
|
||||
delete_at_timestamp = int(test_time + 10000)
|
||||
delete_at_container = str(
|
||||
delete_at_timestamp /
|
||||
self.object_controller.expiring_objects_container_divisor *
|
||||
self.object_controller.expiring_objects_container_divisor)
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'X-Timestamp': normalize_timestamp(put_time),
|
||||
headers=self._update_delete_at_headers({
|
||||
'X-Timestamp': normalize_timestamp(put_time),
|
||||
'X-Delete-At': str(delete_at_timestamp),
|
||||
'X-Delete-At-Container': delete_at_container,
|
||||
'Content-Length': '4',
|
||||
'Content-Type': 'application/octet-stream'})
|
||||
'Content-Type': 'application/octet-stream'}))
|
||||
req.body = 'TEST'
|
||||
|
||||
# Mock out async_update so we don't get any async_pending files.
|
||||
@ -7704,22 +7680,13 @@ class TestObjectController(BaseTestCase):
|
||||
overwrite_time = test_time + 1
|
||||
delete_at_timestamp_1 = int(test_time + 10000)
|
||||
delete_at_timestamp_2 = int(test_time + 20000)
|
||||
delete_at_container_1 = str(
|
||||
delete_at_timestamp_1 /
|
||||
self.object_controller.expiring_objects_container_divisor *
|
||||
self.object_controller.expiring_objects_container_divisor)
|
||||
delete_at_container_2 = str(
|
||||
delete_at_timestamp_2 /
|
||||
self.object_controller.expiring_objects_container_divisor *
|
||||
self.object_controller.expiring_objects_container_divisor)
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'X-Timestamp': normalize_timestamp(put_time),
|
||||
headers=self._update_delete_at_headers({
|
||||
'X-Timestamp': normalize_timestamp(put_time),
|
||||
'X-Delete-At': str(delete_at_timestamp_1),
|
||||
'X-Delete-At-Container': delete_at_container_1,
|
||||
'X-Delete-At-Host': '1.2.3.4',
|
||||
'Content-Length': '4',
|
||||
'Content-Type': 'application/octet-stream'})
|
||||
'Content-Type': 'application/octet-stream'}))
|
||||
req.body = 'TEST'
|
||||
|
||||
# Mock out async_update so we don't get any async_pending files.
|
||||
@ -7731,13 +7698,12 @@ class TestObjectController(BaseTestCase):
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c/o',
|
||||
environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'X-Timestamp': normalize_timestamp(overwrite_time),
|
||||
headers=self._update_delete_at_headers({
|
||||
'X-Timestamp': normalize_timestamp(overwrite_time),
|
||||
'X-Backend-Clean-Expiring-Object-Queue': 'false',
|
||||
'X-Delete-At': str(delete_at_timestamp_2),
|
||||
'X-Delete-At-Container': delete_at_container_2,
|
||||
'X-Delete-At-Host': '1.2.3.4',
|
||||
'Content-Length': '9',
|
||||
'Content-Type': 'application/octet-stream'})
|
||||
'Content-Type': 'application/octet-stream'}))
|
||||
req.body = 'new stuff'
|
||||
resp = req.get_response(self.object_controller)
|
||||
self.assertEqual(resp.status_int, 201)
|
||||
@ -7776,17 +7742,13 @@ class TestObjectController(BaseTestCase):
|
||||
put_time = test_time
|
||||
overwrite_time = test_time + 1
|
||||
delete_at_timestamp = int(test_time + 10000)
|
||||
delete_at_container = str(
|
||||
delete_at_timestamp /
|
||||
self.object_controller.expiring_objects_container_divisor *
|
||||
self.object_controller.expiring_objects_container_divisor)
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'X-Timestamp': normalize_timestamp(put_time),
|
||||
headers=self._update_delete_at_headers({
|
||||
'X-Timestamp': normalize_timestamp(put_time),
|
||||
'X-Delete-At': str(delete_at_timestamp),
|
||||
'X-Delete-At-Container': delete_at_container,
|
||||
'Content-Length': '4',
|
||||
'Content-Type': 'application/octet-stream'})
|
||||
'Content-Type': 'application/octet-stream'}))
|
||||
req.body = 'TEST'
|
||||
|
||||
# Mock out async_update so we don't get any async_pending files.
|
||||
|
||||
Loading…
Reference in New Issue
Block a user