openstack/swift
Code Issues Proposed changes

Merge "Use unique body content in EC test objects"

Browse Source
This commit is contained in:
Jenkins 2017-08-16 01:44:03 +00:00 committed by Gerrit Code Review
commit f9cacb3740
1 changed files with 70 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

139
test/unit/proxy/controllers/test_obj.py
View File

@ -194,6 +194,21 @@ class BaseObjectControllerMixin(object):
policy = policy or POLICIES.default policy = policy or POLICIES.default
return policy.quorum return policy.quorum
# Add a few helper methods for EC tests.
def _make_ec_archive_bodies(self, test_body, policy=None):
policy = policy or self.policy
return encode_frag_archive_bodies(policy, test_body)
def _make_ec_object_stub(self, pattern='test', policy=None,
timestamp=None):
policy = policy or self.policy
test_body = pattern * policy.ec_segment_size
test_body = test_body[:-random.randint(1, 1000)]
return make_ec_object_stub(test_body, policy, timestamp)
def _fake_ec_node_response(self, node_frags):
return fake_ec_node_response(node_frags, self.policy)
def test_iter_nodes_local_first_noops_when_no_affinity(self): def test_iter_nodes_local_first_noops_when_no_affinity(self):
# this test needs a stable node order - most don't # this test needs a stable node order - most don't
self.app.sort_nodes = lambda l, *args, **kwargs: l self.app.sort_nodes = lambda l, *args, **kwargs: l
@ -2375,18 +2390,6 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
resp = req.get_response(self.app) resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 201) self.assertEqual(resp.status_int, 201)
def _make_ec_archive_bodies(self, test_body, policy=None):
policy = policy or self.policy
return encode_frag_archive_bodies(policy, test_body)
def _make_ec_object_stub(self, test_body=None, policy=None,
timestamp=None):
policy = policy or self.policy
return make_ec_object_stub(test_body, policy, timestamp)
def _fake_ec_node_response(self, node_frags):
return fake_ec_node_response(node_frags, self.policy)
def test_GET_with_frags_swapped_around(self): def test_GET_with_frags_swapped_around(self):
segment_size = self.policy.ec_segment_size segment_size = self.policy.ec_segment_size
test_data = ('test' * segment_size)[:-657] test_data = ('test' * segment_size)[:-657]
@ -2491,8 +2494,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
len(collected_responses[obj1['etag']])) len(collected_responses[obj1['etag']]))
def test_GET_with_single_missed_overwrite_does_not_need_handoff(self): def test_GET_with_single_missed_overwrite_does_not_need_handoff(self):
obj1 = self._make_ec_object_stub() obj1 = self._make_ec_object_stub(pattern='obj1')
obj2 = self._make_ec_object_stub() obj2 = self._make_ec_object_stub(pattern='obj2')
node_frags = [ node_frags = [
{'obj': obj2, 'frag': 0}, {'obj': obj2, 'frag': 0},
@ -2541,8 +2544,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
len(frags), etag)) len(frags), etag))
def test_GET_with_many_missed_overwrite_will_need_handoff(self): def test_GET_with_many_missed_overwrite_will_need_handoff(self):
obj1 = self._make_ec_object_stub() obj1 = self._make_ec_object_stub(pattern='obj1')
obj2 = self._make_ec_object_stub() obj2 = self._make_ec_object_stub(pattern='obj2')
node_frags = [ node_frags = [
{'obj': obj2, 'frag': 0}, {'obj': obj2, 'frag': 0},
@ -2592,8 +2595,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
len(frags), etag)) len(frags), etag))
def test_GET_with_missing_and_mixed_frags_will_dig_deep_but_succeed(self): def test_GET_with_missing_and_mixed_frags_will_dig_deep_but_succeed(self):
obj1 = self._make_ec_object_stub(timestamp=self.ts()) obj1 = self._make_ec_object_stub(pattern='obj1', timestamp=self.ts())
obj2 = self._make_ec_object_stub(timestamp=self.ts()) obj2 = self._make_ec_object_stub(pattern='obj2', timestamp=self.ts())
node_frags = [ node_frags = [
{'obj': obj1, 'frag': 0}, {'obj': obj1, 'frag': 0},
@ -2655,8 +2658,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
len(frags), etag)) len(frags), etag))
def test_GET_with_missing_and_mixed_frags_will_dig_deep_but_stop(self): def test_GET_with_missing_and_mixed_frags_will_dig_deep_but_stop(self):
obj1 = self._make_ec_object_stub() obj1 = self._make_ec_object_stub(pattern='obj1')
obj2 = self._make_ec_object_stub() obj2 = self._make_ec_object_stub(pattern='obj2')
node_frags = [ node_frags = [
{'obj': obj1, 'frag': 0}, {'obj': obj1, 'frag': 0},
@ -2843,8 +2846,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
self.assertEqual({obj1['etag'], None}, collected_etags) self.assertEqual({obj1['etag'], None}, collected_etags)
def test_GET_with_missing_and_mixed_frags_may_503(self): def test_GET_with_missing_and_mixed_frags_may_503(self):
obj1 = self._make_ec_object_stub() obj1 = self._make_ec_object_stub(pattern='obj1')
obj2 = self._make_ec_object_stub() obj2 = self._make_ec_object_stub(pattern='obj2')
# we get a 503 when all the handoffs return 200 # we get a 503 when all the handoffs return 200
node_frags = [[]] * self.replicas() # primaries have no frags node_frags = [[]] * self.replicas() # primaries have no frags
node_frags = node_frags + [ # handoffs all have frags node_frags = node_frags + [ # handoffs all have frags
@ -2883,10 +2886,10 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
# all nodes have a frag but there is no one set that reaches quorum, # all nodes have a frag but there is no one set that reaches quorum,
# which means there is no backend 404 response, but proxy should still # which means there is no backend 404 response, but proxy should still
# return 404 rather than 503 # return 404 rather than 503
obj1 = self._make_ec_object_stub() obj1 = self._make_ec_object_stub(pattern='obj1')
obj2 = self._make_ec_object_stub() obj2 = self._make_ec_object_stub(pattern='obj2')
obj3 = self._make_ec_object_stub() obj3 = self._make_ec_object_stub(pattern='obj3')
obj4 = self._make_ec_object_stub() obj4 = self._make_ec_object_stub(pattern='obj4')
node_frags = [ node_frags = [
{'obj': obj1, 'frag': 0}, {'obj': obj1, 'frag': 0},
@ -3057,8 +3060,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
self.assertEqual(28, len(log)) self.assertEqual(28, len(log))
def test_GET_with_missing_durable_files_and_mixed_etags(self): def test_GET_with_missing_durable_files_and_mixed_etags(self):
obj1 = self._make_ec_object_stub() obj1 = self._make_ec_object_stub(pattern='obj1')
obj2 = self._make_ec_object_stub() obj2 = self._make_ec_object_stub(pattern='obj2')
# non-quorate durables for another object won't stop us finding the # non-quorate durables for another object won't stop us finding the
# quorate object # quorate object
@ -3147,8 +3150,10 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
# At that point (or before) the proxy knows that a durable set of # At that point (or before) the proxy knows that a durable set of
# frags for obj2 exists so will fetch them, requiring another 10 # frags for obj2 exists so will fetch them, requiring another 10
# directed requests. # directed requests.
obj2 = self._make_ec_object_stub(timestamp=self._ts_iter.next()) obj2 = self._make_ec_object_stub(pattern='obj2',
obj1 = self._make_ec_object_stub(timestamp=self._ts_iter.next()) timestamp=self._ts_iter.next())
obj1 = self._make_ec_object_stub(pattern='obj1',
timestamp=self._ts_iter.next())
node_frags = [ node_frags = [
[{'obj': obj1, 'frag': 0, 'durable': False}], # obj2 missing [{'obj': obj1, 'frag': 0, 'durable': False}], # obj2 missing
@ -3197,9 +3202,12 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
# GETs to see all the obj3 frags plus 1 more to GET a durable frag. # GETs to see all the obj3 frags plus 1 more to GET a durable frag.
# The proxy may also do one more GET if the obj2 frag is found. # The proxy may also do one more GET if the obj2 frag is found.
# i.e. 10 + 1 durable for obj3, 2 for obj1 and 1 more if obj2 found # i.e. 10 + 1 durable for obj3, 2 for obj1 and 1 more if obj2 found
obj2 = self._make_ec_object_stub(timestamp=self._ts_iter.next()) obj2 = self._make_ec_object_stub(pattern='obj2',
obj3 = self._make_ec_object_stub(timestamp=self._ts_iter.next()) timestamp=self._ts_iter.next())
obj1 = self._make_ec_object_stub(timestamp=self._ts_iter.next()) obj3 = self._make_ec_object_stub(pattern='obj3',
timestamp=self._ts_iter.next())
obj1 = self._make_ec_object_stub(pattern='obj1',
timestamp=self._ts_iter.next())
node_frags = [ node_frags = [
[{'obj': obj1, 'frag': 0, 'durable': False}, # obj1 frag [{'obj': obj1, 'frag': 0, 'durable': False}, # obj1 frag
@ -3237,8 +3245,10 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
# scenario: non-durable frags of newer obj1 obscure all frags # scenario: non-durable frags of newer obj1 obscure all frags
# of older obj2, so first 28 requests result in a non-durable set. # of older obj2, so first 28 requests result in a non-durable set.
# There are only 10 frags for obj2 and one is not durable. # There are only 10 frags for obj2 and one is not durable.
obj2 = self._make_ec_object_stub(timestamp=self._ts_iter.next()) obj2 = self._make_ec_object_stub(pattern='obj2',
obj1 = self._make_ec_object_stub(timestamp=self._ts_iter.next()) timestamp=self._ts_iter.next())
obj1 = self._make_ec_object_stub(pattern='obj1',
timestamp=self._ts_iter.next())
node_frags = [ node_frags = [
[{'obj': obj1, 'frag': 0, 'durable': False}], # obj2 missing [{'obj': obj1, 'frag': 0, 'durable': False}], # obj2 missing
@ -3287,8 +3297,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
# fragments for different content at the same timestamp then the # fragments for different content at the same timestamp then the
# object controller should handle it gracefully # object controller should handle it gracefully
ts = self.ts() # force equal timestamps for two objects ts = self.ts() # force equal timestamps for two objects
obj1 = self._make_ec_object_stub(timestamp=ts, test_body='obj1') obj1 = self._make_ec_object_stub(timestamp=ts, pattern='obj1')
obj2 = self._make_ec_object_stub(timestamp=ts, test_body='obj2') obj2 = self._make_ec_object_stub(timestamp=ts, pattern='obj2')
self.assertNotEqual(obj1['etag'], obj2['etag']) # sanity self.assertNotEqual(obj1['etag'], obj2['etag']) # sanity
node_frags = [ node_frags = [
@ -3932,18 +3942,6 @@ class TestECDuplicationObjController(
controller_cls = obj.ECObjectController controller_cls = obj.ECObjectController
def _make_ec_object_stub(self, test_body=None, policy=None,
timestamp=None):
policy = policy or self.policy
return make_ec_object_stub(test_body, policy, timestamp)
def _make_ec_archive_bodies(self, test_body, policy=None):
policy = policy or self.policy
return encode_frag_archive_bodies(policy, test_body)
def _fake_ec_node_response(self, node_frags):
return fake_ec_node_response(node_frags, self.policy)
def _test_GET_with_duplication_factor(self, node_frags, obj): def _test_GET_with_duplication_factor(self, node_frags, obj):
# This is basic tests in the healthy backends status # This is basic tests in the healthy backends status
fake_response = self._fake_ec_node_response(node_frags) fake_response = self._fake_ec_node_response(node_frags)
@ -4035,8 +4033,8 @@ class TestECDuplicationObjController(
self._test_GET_with_duplication_factor(node_frags, obj) self._test_GET_with_duplication_factor(node_frags, obj)
def test_GET_with_missing_and_mixed_frags_will_dig_deep_but_stop(self): def test_GET_with_missing_and_mixed_frags_will_dig_deep_but_stop(self):
obj1 = self._make_ec_object_stub() obj1 = self._make_ec_object_stub(pattern='obj1')
obj2 = self._make_ec_object_stub() obj2 = self._make_ec_object_stub(pattern='obj2')
# both of obj1 and obj2 has only 9 frags which is not able to decode # both of obj1 and obj2 has only 9 frags which is not able to decode
node_frags = [ node_frags = [
@ -4081,7 +4079,8 @@ class TestECDuplicationObjController(
# default node_iter will exhaust to the last of handoffs # default node_iter will exhaust to the last of handoffs
self.assertEqual(len(log), self.replicas() * 2) self.assertEqual(len(log), self.replicas() * 2)
# we have obj1, obj2, and 404 NotFound in collected_responses # we have obj1, obj2, and 404 NotFound in collected_responses
self.assertEqual(len(collected_responses), 3) self.assertEqual(sorted([obj1['etag'], obj2['etag'], None]),
sorted(collected_responses.keys()))
# ... regardless we should never need to fetch more than ec_ndata # ... regardless we should never need to fetch more than ec_ndata
# frags for any given etag # frags for any given etag
@ -4134,8 +4133,8 @@ class TestECDuplicationObjController(
self.assertEqual(len(collected_indexes), self.policy.ec_ndata - 1) self.assertEqual(len(collected_indexes), self.policy.ec_ndata - 1)
def test_GET_with_many_missed_overwrite_will_need_handoff(self): def test_GET_with_many_missed_overwrite_will_need_handoff(self):
obj1 = self._make_ec_object_stub() obj1 = self._make_ec_object_stub(pattern='obj1')
obj2 = self._make_ec_object_stub() obj2 = self._make_ec_object_stub(pattern='obj2')
# primaries # primaries
node_frags = [ node_frags = [
{'obj': obj2, 'frag': 0}, {'obj': obj2, 'frag': 0},
@ -4239,8 +4238,10 @@ class TestECDuplicationObjController(
self.assertEqual(len(collected_indexes), self.policy.ec_ndata) self.assertEqual(len(collected_indexes), self.policy.ec_ndata)
def test_GET_with_missing_and_mixed_frags_will_dig_deep_but_succeed(self): def test_GET_with_missing_and_mixed_frags_will_dig_deep_but_succeed(self):
obj1 = self._make_ec_object_stub(timestamp=self.ts()) obj1 = self._make_ec_object_stub(pattern='obj1',
obj2 = self._make_ec_object_stub(timestamp=self.ts()) timestamp=self.ts())
obj2 = self._make_ec_object_stub(pattern='obj2',
timestamp=self.ts())
# 28 nodes are here # 28 nodes are here
node_frags = [ node_frags = [
@ -4312,13 +4313,13 @@ class TestECDuplicationObjController(
# which means there is no backend 404 response, but proxy should still # which means there is no backend 404 response, but proxy should still
# return 404 rather than 503 # return 404 rather than 503
stub_objects = [ stub_objects = [
self._make_ec_object_stub('obj1' * self.policy.ec_segment_size), self._make_ec_object_stub(pattern='obj1'),
self._make_ec_object_stub('obj2' * self.policy.ec_segment_size), self._make_ec_object_stub(pattern='obj2'),
self._make_ec_object_stub('obj3' * self.policy.ec_segment_size), self._make_ec_object_stub(pattern='obj3'),
self._make_ec_object_stub('obj4' * self.policy.ec_segment_size), self._make_ec_object_stub(pattern='obj4'),
self._make_ec_object_stub('obj5' * self.policy.ec_segment_size), self._make_ec_object_stub(pattern='obj5'),
self._make_ec_object_stub('obj6' * self.policy.ec_segment_size), self._make_ec_object_stub(pattern='obj6'),
self._make_ec_object_stub('obj7' * self.policy.ec_segment_size), self._make_ec_object_stub(pattern='obj7'),
] ]
etags = collections.Counter(stub['etag'] for stub in stub_objects) etags = collections.Counter(stub['etag'] for stub in stub_objects)
self.assertEqual(len(etags), 7, etags) # sanity self.assertEqual(len(etags), 7, etags) # sanity
@ -4395,10 +4396,10 @@ class TestECDuplicationObjController(
self.assertEqual(self.replicas() * 2, len(log)) self.assertEqual(self.replicas() * 2, len(log))
def test_GET_with_missing_and_mixed_frags_may_503(self): def test_GET_with_missing_and_mixed_frags_may_503(self):
obj1 = self._make_ec_object_stub() obj1 = self._make_ec_object_stub(pattern='obj1')
obj2 = self._make_ec_object_stub() obj2 = self._make_ec_object_stub(pattern='obj2')
obj3 = self._make_ec_object_stub() obj3 = self._make_ec_object_stub(pattern='obj3')
obj4 = self._make_ec_object_stub() obj4 = self._make_ec_object_stub(pattern='obj4')
# we get a 503 when all the handoffs return 200 # we get a 503 when all the handoffs return 200
node_frags = [[]] * self.replicas() # primaries have no frags node_frags = [[]] * self.replicas() # primaries have no frags
# plus, 4 different objects and 7 indexes will b 28 node responses # plus, 4 different objects and 7 indexes will b 28 node responses
@ -4454,8 +4455,8 @@ class TestECDuplicationObjController(
# the difference from parent class is only handoff stub length # the difference from parent class is only handoff stub length
ts = self.ts() # force equal timestamps for two objects ts = self.ts() # force equal timestamps for two objects
obj1 = self._make_ec_object_stub(timestamp=ts, test_body='obj1') obj1 = self._make_ec_object_stub(timestamp=ts, pattern='obj1')
obj2 = self._make_ec_object_stub(timestamp=ts, test_body='obj2') obj2 = self._make_ec_object_stub(timestamp=ts, pattern='obj2')
self.assertNotEqual(obj1['etag'], obj2['etag']) # sanity self.assertNotEqual(obj1['etag'], obj2['etag']) # sanity
node_frags = [ node_frags = [