diff --git a/test/unit/proxy/controllers/test_obj.py b/test/unit/proxy/controllers/test_obj.py index e0f73eb25b..ffdb6e37dc 100644 --- a/test/unit/proxy/controllers/test_obj.py +++ b/test/unit/proxy/controllers/test_obj.py @@ -194,6 +194,21 @@ class BaseObjectControllerMixin(object): policy = policy or POLICIES.default return policy.quorum + # Add a few helper methods for EC tests. + def _make_ec_archive_bodies(self, test_body, policy=None): + policy = policy or self.policy + return encode_frag_archive_bodies(policy, test_body) + + def _make_ec_object_stub(self, pattern='test', policy=None, + timestamp=None): + policy = policy or self.policy + test_body = pattern * policy.ec_segment_size + test_body = test_body[:-random.randint(1, 1000)] + return make_ec_object_stub(test_body, policy, timestamp) + + def _fake_ec_node_response(self, node_frags): + return fake_ec_node_response(node_frags, self.policy) + def test_iter_nodes_local_first_noops_when_no_affinity(self): # this test needs a stable node order - most don't self.app.sort_nodes = lambda l, *args, **kwargs: l @@ -2375,18 +2390,6 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase): resp = req.get_response(self.app) self.assertEqual(resp.status_int, 201) - def _make_ec_archive_bodies(self, test_body, policy=None): - policy = policy or self.policy - return encode_frag_archive_bodies(policy, test_body) - - def _make_ec_object_stub(self, test_body=None, policy=None, - timestamp=None): - policy = policy or self.policy - return make_ec_object_stub(test_body, policy, timestamp) - - def _fake_ec_node_response(self, node_frags): - return fake_ec_node_response(node_frags, self.policy) - def test_GET_with_frags_swapped_around(self): segment_size = self.policy.ec_segment_size test_data = ('test' * segment_size)[:-657] @@ -2491,8 +2494,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase): len(collected_responses[obj1['etag']])) def test_GET_with_single_missed_overwrite_does_not_need_handoff(self): - obj1 = self._make_ec_object_stub() - obj2 = self._make_ec_object_stub() + obj1 = self._make_ec_object_stub(pattern='obj1') + obj2 = self._make_ec_object_stub(pattern='obj2') node_frags = [ {'obj': obj2, 'frag': 0}, @@ -2541,8 +2544,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase): len(frags), etag)) def test_GET_with_many_missed_overwrite_will_need_handoff(self): - obj1 = self._make_ec_object_stub() - obj2 = self._make_ec_object_stub() + obj1 = self._make_ec_object_stub(pattern='obj1') + obj2 = self._make_ec_object_stub(pattern='obj2') node_frags = [ {'obj': obj2, 'frag': 0}, @@ -2592,8 +2595,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase): len(frags), etag)) def test_GET_with_missing_and_mixed_frags_will_dig_deep_but_succeed(self): - obj1 = self._make_ec_object_stub(timestamp=self.ts()) - obj2 = self._make_ec_object_stub(timestamp=self.ts()) + obj1 = self._make_ec_object_stub(pattern='obj1', timestamp=self.ts()) + obj2 = self._make_ec_object_stub(pattern='obj2', timestamp=self.ts()) node_frags = [ {'obj': obj1, 'frag': 0}, @@ -2655,8 +2658,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase): len(frags), etag)) def test_GET_with_missing_and_mixed_frags_will_dig_deep_but_stop(self): - obj1 = self._make_ec_object_stub() - obj2 = self._make_ec_object_stub() + obj1 = self._make_ec_object_stub(pattern='obj1') + obj2 = self._make_ec_object_stub(pattern='obj2') node_frags = [ {'obj': obj1, 'frag': 0}, @@ -2843,8 +2846,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase): self.assertEqual({obj1['etag'], None}, collected_etags) def test_GET_with_missing_and_mixed_frags_may_503(self): - obj1 = self._make_ec_object_stub() - obj2 = self._make_ec_object_stub() + obj1 = self._make_ec_object_stub(pattern='obj1') + obj2 = self._make_ec_object_stub(pattern='obj2') # we get a 503 when all the handoffs return 200 node_frags = [[]] * self.replicas() # primaries have no frags node_frags = node_frags + [ # handoffs all have frags @@ -2883,10 +2886,10 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase): # all nodes have a frag but there is no one set that reaches quorum, # which means there is no backend 404 response, but proxy should still # return 404 rather than 503 - obj1 = self._make_ec_object_stub() - obj2 = self._make_ec_object_stub() - obj3 = self._make_ec_object_stub() - obj4 = self._make_ec_object_stub() + obj1 = self._make_ec_object_stub(pattern='obj1') + obj2 = self._make_ec_object_stub(pattern='obj2') + obj3 = self._make_ec_object_stub(pattern='obj3') + obj4 = self._make_ec_object_stub(pattern='obj4') node_frags = [ {'obj': obj1, 'frag': 0}, @@ -3057,8 +3060,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase): self.assertEqual(28, len(log)) def test_GET_with_missing_durable_files_and_mixed_etags(self): - obj1 = self._make_ec_object_stub() - obj2 = self._make_ec_object_stub() + obj1 = self._make_ec_object_stub(pattern='obj1') + obj2 = self._make_ec_object_stub(pattern='obj2') # non-quorate durables for another object won't stop us finding the # quorate object @@ -3147,8 +3150,10 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase): # At that point (or before) the proxy knows that a durable set of # frags for obj2 exists so will fetch them, requiring another 10 # directed requests. - obj2 = self._make_ec_object_stub(timestamp=self._ts_iter.next()) - obj1 = self._make_ec_object_stub(timestamp=self._ts_iter.next()) + obj2 = self._make_ec_object_stub(pattern='obj2', + timestamp=self._ts_iter.next()) + obj1 = self._make_ec_object_stub(pattern='obj1', + timestamp=self._ts_iter.next()) node_frags = [ [{'obj': obj1, 'frag': 0, 'durable': False}], # obj2 missing @@ -3197,9 +3202,12 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase): # GETs to see all the obj3 frags plus 1 more to GET a durable frag. # The proxy may also do one more GET if the obj2 frag is found. # i.e. 10 + 1 durable for obj3, 2 for obj1 and 1 more if obj2 found - obj2 = self._make_ec_object_stub(timestamp=self._ts_iter.next()) - obj3 = self._make_ec_object_stub(timestamp=self._ts_iter.next()) - obj1 = self._make_ec_object_stub(timestamp=self._ts_iter.next()) + obj2 = self._make_ec_object_stub(pattern='obj2', + timestamp=self._ts_iter.next()) + obj3 = self._make_ec_object_stub(pattern='obj3', + timestamp=self._ts_iter.next()) + obj1 = self._make_ec_object_stub(pattern='obj1', + timestamp=self._ts_iter.next()) node_frags = [ [{'obj': obj1, 'frag': 0, 'durable': False}, # obj1 frag @@ -3237,8 +3245,10 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase): # scenario: non-durable frags of newer obj1 obscure all frags # of older obj2, so first 28 requests result in a non-durable set. # There are only 10 frags for obj2 and one is not durable. - obj2 = self._make_ec_object_stub(timestamp=self._ts_iter.next()) - obj1 = self._make_ec_object_stub(timestamp=self._ts_iter.next()) + obj2 = self._make_ec_object_stub(pattern='obj2', + timestamp=self._ts_iter.next()) + obj1 = self._make_ec_object_stub(pattern='obj1', + timestamp=self._ts_iter.next()) node_frags = [ [{'obj': obj1, 'frag': 0, 'durable': False}], # obj2 missing @@ -3287,8 +3297,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase): # fragments for different content at the same timestamp then the # object controller should handle it gracefully ts = self.ts() # force equal timestamps for two objects - obj1 = self._make_ec_object_stub(timestamp=ts, test_body='obj1') - obj2 = self._make_ec_object_stub(timestamp=ts, test_body='obj2') + obj1 = self._make_ec_object_stub(timestamp=ts, pattern='obj1') + obj2 = self._make_ec_object_stub(timestamp=ts, pattern='obj2') self.assertNotEqual(obj1['etag'], obj2['etag']) # sanity node_frags = [ @@ -3932,18 +3942,6 @@ class TestECDuplicationObjController( controller_cls = obj.ECObjectController - def _make_ec_object_stub(self, test_body=None, policy=None, - timestamp=None): - policy = policy or self.policy - return make_ec_object_stub(test_body, policy, timestamp) - - def _make_ec_archive_bodies(self, test_body, policy=None): - policy = policy or self.policy - return encode_frag_archive_bodies(policy, test_body) - - def _fake_ec_node_response(self, node_frags): - return fake_ec_node_response(node_frags, self.policy) - def _test_GET_with_duplication_factor(self, node_frags, obj): # This is basic tests in the healthy backends status fake_response = self._fake_ec_node_response(node_frags) @@ -4035,8 +4033,8 @@ class TestECDuplicationObjController( self._test_GET_with_duplication_factor(node_frags, obj) def test_GET_with_missing_and_mixed_frags_will_dig_deep_but_stop(self): - obj1 = self._make_ec_object_stub() - obj2 = self._make_ec_object_stub() + obj1 = self._make_ec_object_stub(pattern='obj1') + obj2 = self._make_ec_object_stub(pattern='obj2') # both of obj1 and obj2 has only 9 frags which is not able to decode node_frags = [ @@ -4081,7 +4079,8 @@ class TestECDuplicationObjController( # default node_iter will exhaust to the last of handoffs self.assertEqual(len(log), self.replicas() * 2) # we have obj1, obj2, and 404 NotFound in collected_responses - self.assertEqual(len(collected_responses), 3) + self.assertEqual(sorted([obj1['etag'], obj2['etag'], None]), + sorted(collected_responses.keys())) # ... regardless we should never need to fetch more than ec_ndata # frags for any given etag @@ -4134,8 +4133,8 @@ class TestECDuplicationObjController( self.assertEqual(len(collected_indexes), self.policy.ec_ndata - 1) def test_GET_with_many_missed_overwrite_will_need_handoff(self): - obj1 = self._make_ec_object_stub() - obj2 = self._make_ec_object_stub() + obj1 = self._make_ec_object_stub(pattern='obj1') + obj2 = self._make_ec_object_stub(pattern='obj2') # primaries node_frags = [ {'obj': obj2, 'frag': 0}, @@ -4239,8 +4238,10 @@ class TestECDuplicationObjController( self.assertEqual(len(collected_indexes), self.policy.ec_ndata) def test_GET_with_missing_and_mixed_frags_will_dig_deep_but_succeed(self): - obj1 = self._make_ec_object_stub(timestamp=self.ts()) - obj2 = self._make_ec_object_stub(timestamp=self.ts()) + obj1 = self._make_ec_object_stub(pattern='obj1', + timestamp=self.ts()) + obj2 = self._make_ec_object_stub(pattern='obj2', + timestamp=self.ts()) # 28 nodes are here node_frags = [ @@ -4312,13 +4313,13 @@ class TestECDuplicationObjController( # which means there is no backend 404 response, but proxy should still # return 404 rather than 503 stub_objects = [ - self._make_ec_object_stub('obj1' * self.policy.ec_segment_size), - self._make_ec_object_stub('obj2' * self.policy.ec_segment_size), - self._make_ec_object_stub('obj3' * self.policy.ec_segment_size), - self._make_ec_object_stub('obj4' * self.policy.ec_segment_size), - self._make_ec_object_stub('obj5' * self.policy.ec_segment_size), - self._make_ec_object_stub('obj6' * self.policy.ec_segment_size), - self._make_ec_object_stub('obj7' * self.policy.ec_segment_size), + self._make_ec_object_stub(pattern='obj1'), + self._make_ec_object_stub(pattern='obj2'), + self._make_ec_object_stub(pattern='obj3'), + self._make_ec_object_stub(pattern='obj4'), + self._make_ec_object_stub(pattern='obj5'), + self._make_ec_object_stub(pattern='obj6'), + self._make_ec_object_stub(pattern='obj7'), ] etags = collections.Counter(stub['etag'] for stub in stub_objects) self.assertEqual(len(etags), 7, etags) # sanity @@ -4395,10 +4396,10 @@ class TestECDuplicationObjController( self.assertEqual(self.replicas() * 2, len(log)) def test_GET_with_missing_and_mixed_frags_may_503(self): - obj1 = self._make_ec_object_stub() - obj2 = self._make_ec_object_stub() - obj3 = self._make_ec_object_stub() - obj4 = self._make_ec_object_stub() + obj1 = self._make_ec_object_stub(pattern='obj1') + obj2 = self._make_ec_object_stub(pattern='obj2') + obj3 = self._make_ec_object_stub(pattern='obj3') + obj4 = self._make_ec_object_stub(pattern='obj4') # we get a 503 when all the handoffs return 200 node_frags = [[]] * self.replicas() # primaries have no frags # plus, 4 different objects and 7 indexes will b 28 node responses @@ -4454,8 +4455,8 @@ class TestECDuplicationObjController( # the difference from parent class is only handoff stub length ts = self.ts() # force equal timestamps for two objects - obj1 = self._make_ec_object_stub(timestamp=ts, test_body='obj1') - obj2 = self._make_ec_object_stub(timestamp=ts, test_body='obj2') + obj1 = self._make_ec_object_stub(timestamp=ts, pattern='obj1') + obj2 = self._make_ec_object_stub(timestamp=ts, pattern='obj2') self.assertNotEqual(obj1['etag'], obj2['etag']) # sanity node_frags = [