container-server: return objects of a given policy
There is a tight coupling between a root container and its shards: the shards hold the object metadata for the root container, so are really an extension of the root. When we PUT objects in to a root container, it'll redirect them, with the root's policy, to the shards. And the shards are happy to take them, even if the shard's policy is different to the root's. But when it comes to GETs, the root redirects the GET onto it's shards whom currently wont respond with objects (which they probably took) because they are of a different policy. Currently, when getting objects from the container server, the policy used is always the broker's policy. This patch corrects this behaviour by allowing the ability to override the policy index to use. If the request to the container server contains an 'X-Backend-Storage-Policy-Index' header it'll be used instead of the policy index stored in the broker. This patch adds the root container's policy as this header in the proxy container controller's `_get_from_shards` method which is used by the proxy to redirect a GET to a root to its shards. Further, a new backend response header has been added. If the container response contains an `X-Backend-Record-Type: object` header, then it means the response is a response with objects in it. In this case this patch also adds a `X-Backend-Record-Storage-Policy-Index` header so the policy index of the given objects is known, as X-Backend-Storage-Policy-Index in the response _always_ represents the policy index of the container itself. On a plus side this new container policy API allows us a way to check containers for object listing is other policies. So might come in handy for OPs/SREs. Co-Authored-By: Alistair Coles <alistairncoles@gmail.com> Change-Id: I026b699fc5f0fba619cf524093632d67ca38d32f
This commit is contained in:
Matthew Oliver
Alistair Coles
parent
b3c5a9a9e0
commit
56510ab3c3
@ -768,16 +768,22 @@ class ContainerController(BaseStorageServer):
|
||||
include_deleted=include_deleted, fill_gaps=fill_gaps,
|
||||
include_own=include_own)
|
||||
else:
|
||||
requested_policy_index = self.get_and_validate_policy_index(req)
|
||||
resp_headers = gen_resp_headers(info, is_deleted=is_deleted)
|
||||
if is_deleted:
|
||||
return HTTPNotFound(request=req, headers=resp_headers)
|
||||
resp_headers['X-Backend-Record-Type'] = 'object'
|
||||
storage_policy_index = (
|
||||
requested_policy_index if requested_policy_index is not None
|
||||
else info['storage_policy_index'])
|
||||
resp_headers['X-Backend-Record-Storage-Policy-Index'] = \
|
||||
storage_policy_index
|
||||
# Use the retired db while container is in process of sharding,
|
||||
# otherwise use current db
|
||||
src_broker = broker.get_brokers()[0]
|
||||
container_list = src_broker.list_objects_iter(
|
||||
limit, marker, end_marker, prefix, delimiter, path,
|
||||
storage_policy_index=info['storage_policy_index'],
|
||||
storage_policy_index=storage_policy_index,
|
||||
reverse=reverse, allow_reserved=req.allow_reserved_names)
|
||||
return self.create_listing(req, out_content_type, info, resp_headers,
|
||||
broker.metadata, container_list, container)
|
||||
|
||||
@ -2285,7 +2285,9 @@ class Controller(object):
|
||||
:param req: original Request instance.
|
||||
:param account: account in which `container` is stored.
|
||||
:param container: container from which listing should be fetched.
|
||||
:param headers: headers to be included with the request
|
||||
:param headers: extra headers to be included with the listing
|
||||
sub-request; these update the headers copied from the original
|
||||
request.
|
||||
:param params: query string parameters to be used.
|
||||
:return: a tuple of (deserialized json data structure, swob Response)
|
||||
"""
|
||||
|
||||
@ -318,6 +318,17 @@ class ContainerController(Controller):
|
||||
# shard may return the root's shard range.
|
||||
shard_listing_history = req.environ.setdefault(
|
||||
'swift.shard_listing_history', [])
|
||||
policy_key = 'X-Backend-Storage-Policy-Index'
|
||||
if not (shard_listing_history or policy_key in req.headers):
|
||||
# We're handling the original request to the root container: set
|
||||
# the root policy index in the request, unless it is already set,
|
||||
# so that shards will return listings for that policy index.
|
||||
# Note: we only get here if the root responded with shard ranges,
|
||||
# or if the shard ranges were cached and the cached root container
|
||||
# info has sharding_state==sharded; in both cases we can assume
|
||||
# that the response is "modern enough" to include
|
||||
# 'X-Backend-Storage-Policy-Index'.
|
||||
req.headers[policy_key] = resp.headers[policy_key]
|
||||
shard_listing_history.append((self.account_name, self.container_name))
|
||||
shard_ranges = [ShardRange.from_dict(data)
|
||||
for data in json.loads(resp.body)]
|
||||
|
||||
@ -16,6 +16,7 @@ import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import unittest
|
||||
import uuid
|
||||
|
||||
from nose import SkipTest
|
||||
@ -23,6 +24,7 @@ import six
|
||||
from six.moves.urllib.parse import quote
|
||||
|
||||
from swift.common import direct_client, utils
|
||||
from swift.common.internal_client import UnexpectedResponse
|
||||
from swift.common.manager import Manager
|
||||
from swift.common.memcached import MemcacheRing
|
||||
from swift.common.utils import ShardRange, parse_db_filename, get_db_files, \
|
||||
@ -39,7 +41,7 @@ from test import annotate_failure
|
||||
from test.probe import PROXY_BASE_URL
|
||||
from test.probe.brain import BrainSplitter
|
||||
from test.probe.common import ReplProbeTest, get_server_number, \
|
||||
wait_for_server_to_hangup
|
||||
wait_for_server_to_hangup, ENABLED_POLICIES
|
||||
import mock
|
||||
|
||||
|
||||
@ -2215,6 +2217,72 @@ class TestContainerSharding(BaseTestContainerSharding):
|
||||
self.assert_container_delete_fails()
|
||||
self.assert_container_post_ok('revived')
|
||||
|
||||
def _do_test_sharded_can_get_objects_different_policy(self,
|
||||
policy_idx,
|
||||
new_policy_idx):
|
||||
# create sharded container
|
||||
client.delete_container(self.url, self.token, self.container_name)
|
||||
self.brain.put_container(policy_index=int(policy_idx))
|
||||
all_obj_names = self._make_object_names(self.max_shard_size)
|
||||
self.put_objects(all_obj_names)
|
||||
client.post_container(self.url, self.admin_token, self.container_name,
|
||||
headers={'X-Container-Sharding': 'on'})
|
||||
for n in self.brain.node_numbers:
|
||||
self.sharders.once(
|
||||
number=n, additional_args='--partitions=%s' % self.brain.part)
|
||||
# empty and delete
|
||||
self.delete_objects(all_obj_names)
|
||||
shard_ranges = self.get_container_shard_ranges()
|
||||
self.run_sharders(shard_ranges)
|
||||
client.delete_container(self.url, self.token, self.container_name)
|
||||
|
||||
# re-create with new_policy_idx
|
||||
self.brain.put_container(policy_index=int(new_policy_idx))
|
||||
|
||||
# we re-use shard ranges
|
||||
new_shard_ranges = self.get_container_shard_ranges()
|
||||
self.assertEqual(shard_ranges, new_shard_ranges)
|
||||
self.put_objects(all_obj_names)
|
||||
|
||||
# The shard is still on the old policy index, but the root spi
|
||||
# is passed to shard container server and is used to pull objects
|
||||
# of that index out.
|
||||
self.assert_container_listing(all_obj_names)
|
||||
# although a head request is getting object count for the shard spi
|
||||
self.assert_container_object_count(0)
|
||||
|
||||
# we can force the listing to use the old policy index in which case we
|
||||
# expect no objects to be listed
|
||||
try:
|
||||
resp = self.internal_client.make_request(
|
||||
'GET',
|
||||
path=self.internal_client.make_path(
|
||||
self.account, self.container_name),
|
||||
headers={'X-Backend-Storage-Policy-Index': str(policy_idx)},
|
||||
acceptable_statuses=(2,),
|
||||
params={'format': 'json'}
|
||||
)
|
||||
except UnexpectedResponse as exc:
|
||||
self.fail('Listing failed with %s' % exc.resp.status)
|
||||
|
||||
self.assertEqual([], json.loads(b''.join(resp.app_iter)))
|
||||
|
||||
@unittest.skipIf(len(ENABLED_POLICIES) < 2, "Need more than one policy")
|
||||
def test_sharded_can_get_objects_different_policy(self):
|
||||
policy_idx = self.policy.idx
|
||||
new_policy_idx = [pol.idx for pol in ENABLED_POLICIES
|
||||
if pol != self.policy.idx][0]
|
||||
self._do_test_sharded_can_get_objects_different_policy(
|
||||
policy_idx, new_policy_idx)
|
||||
|
||||
@unittest.skipIf(len(ENABLED_POLICIES) < 2, "Need more than one policy")
|
||||
def test_sharded_can_get_objects_different_policy_reversed(self):
|
||||
policy_idx = [pol.idx for pol in ENABLED_POLICIES
|
||||
if pol != self.policy][0]
|
||||
new_policy_idx = self.policy.idx
|
||||
self._do_test_sharded_can_get_objects_different_policy(
|
||||
policy_idx, new_policy_idx)
|
||||
|
||||
def test_object_update_redirection(self):
|
||||
all_obj_names = self._make_object_names(self.max_shard_size)
|
||||
self.put_objects(all_obj_names)
|
||||
|
||||
@ -3415,6 +3415,12 @@ class TestContainerController(unittest.TestCase):
|
||||
self.assertIn('X-Backend-Record-Type', resp.headers)
|
||||
self.assertEqual(
|
||||
'object', resp.headers.pop('X-Backend-Record-Type'))
|
||||
self.assertEqual(
|
||||
str(POLICIES.default.idx),
|
||||
resp.headers.pop('X-Backend-Storage-Policy-Index'))
|
||||
self.assertEqual(
|
||||
str(POLICIES.default.idx),
|
||||
resp.headers.pop('X-Backend-Record-Storage-Policy-Index'))
|
||||
resp.headers.pop('Content-Length')
|
||||
return resp
|
||||
|
||||
@ -3430,6 +3436,11 @@ class TestContainerController(unittest.TestCase):
|
||||
self.assertIn('X-Backend-Record-Type', resp.headers)
|
||||
self.assertEqual(
|
||||
'shard', resp.headers.pop('X-Backend-Record-Type'))
|
||||
self.assertEqual(
|
||||
str(POLICIES.default.idx),
|
||||
resp.headers.pop('X-Backend-Storage-Policy-Index'))
|
||||
self.assertNotIn('X-Backend-Record-Storage-Policy-Index',
|
||||
resp.headers)
|
||||
resp.headers.pop('Content-Length')
|
||||
return resp
|
||||
|
||||
@ -4239,6 +4250,119 @@ class TestContainerController(unittest.TestCase):
|
||||
resp = req.get_response(self.controller)
|
||||
self.assertEqual(resp.status_int, 406)
|
||||
|
||||
@patch_policies([
|
||||
StoragePolicy(0, name='nulo', is_default=True),
|
||||
StoragePolicy(1, name='unu'),
|
||||
StoragePolicy(2, name='du'),
|
||||
])
|
||||
def test_GET_objects_of_different_policies(self):
|
||||
# make a container
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT',
|
||||
'HTTP_X_TIMESTAMP': '0'})
|
||||
resp = req.get_response(self.controller)
|
||||
resp_policy_idx = resp.headers['X-Backend-Storage-Policy-Index']
|
||||
self.assertEqual(resp_policy_idx, str(POLICIES.default.idx))
|
||||
|
||||
pol_def_objs = ['obj_default_%d' % i for i in range(11)]
|
||||
pol_1_objs = ['obj_1_%d' % i for i in range(10)]
|
||||
|
||||
# fill the container
|
||||
for obj in pol_def_objs:
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c/%s' % obj,
|
||||
environ={
|
||||
'REQUEST_METHOD': 'PUT',
|
||||
'HTTP_X_TIMESTAMP': '1',
|
||||
'HTTP_X_CONTENT_TYPE': 'text/plain',
|
||||
'HTTP_X_ETAG': 'x',
|
||||
'HTTP_X_SIZE': 0})
|
||||
self._update_object_put_headers(req)
|
||||
resp = req.get_response(self.controller)
|
||||
self.assertEqual(resp.status_int, 201)
|
||||
|
||||
for obj in pol_1_objs:
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c/%s' % obj,
|
||||
environ={
|
||||
'REQUEST_METHOD': 'PUT',
|
||||
'HTTP_X_TIMESTAMP': '1',
|
||||
'HTTP_X_CONTENT_TYPE': 'text/plain',
|
||||
'HTTP_X_ETAG': 'x',
|
||||
'HTTP_X_SIZE': 0,
|
||||
'HTTP_X_BACKEND_STORAGE_POLICY_INDEX': 1})
|
||||
resp = req.get_response(self.controller)
|
||||
self.assertEqual(resp.status_int, 201)
|
||||
|
||||
expected_pol_def_objs = [o.encode('utf8') for o in pol_def_objs]
|
||||
expected_pol_1_objs = [o.encode('utf8') for o in pol_1_objs]
|
||||
|
||||
# By default the container server will return objects belonging to
|
||||
# the brokers storage policy
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'})
|
||||
resp = req.get_response(self.controller)
|
||||
self.assertEqual(resp.status_int, 200)
|
||||
result = [o for o in resp.body.split(b'\n') if o]
|
||||
self.assertEqual(len(result), 11)
|
||||
self.assertEqual(sorted(result), sorted(expected_pol_def_objs))
|
||||
self.assertIn('X-Backend-Storage-Policy-Index', resp.headers)
|
||||
self.assertEqual('0', resp.headers['X-Backend-Storage-Policy-Index'])
|
||||
self.assertEqual('0',
|
||||
resp.headers['X-Backend-Record-Storage-Policy-Index'])
|
||||
|
||||
# If we specify the policy 0 idx we should get the same
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'})
|
||||
req.headers['X-Backend-Storage-Policy-Index'] = POLICIES.default.idx
|
||||
resp = req.get_response(self.controller)
|
||||
self.assertEqual(resp.status_int, 200)
|
||||
result = [o for o in resp.body.split(b'\n') if o]
|
||||
self.assertEqual(len(result), 11)
|
||||
self.assertEqual(sorted(result), sorted(expected_pol_def_objs))
|
||||
self.assertIn('X-Backend-Storage-Policy-Index', resp.headers)
|
||||
self.assertEqual('0', resp.headers['X-Backend-Storage-Policy-Index'])
|
||||
self.assertEqual('0',
|
||||
resp.headers['X-Backend-Record-Storage-Policy-Index'])
|
||||
|
||||
# And if we specify a different idx we'll get objects for that policy
|
||||
# and the X-Backend-Record-Storage-Policy-Index letting us know the
|
||||
# policy for which these objects came from, if it differs from the
|
||||
# policy stored in the DB.
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'})
|
||||
req.headers['X-Backend-Storage-Policy-Index'] = 1
|
||||
resp = req.get_response(self.controller)
|
||||
self.assertEqual(resp.status_int, 200)
|
||||
result = [o for o in resp.body.split(b'\n') if o]
|
||||
self.assertEqual(len(result), 10)
|
||||
self.assertEqual(sorted(result), sorted(expected_pol_1_objs))
|
||||
self.assertIn('X-Backend-Storage-Policy-Index', resp.headers)
|
||||
self.assertEqual('0', resp.headers['X-Backend-Storage-Policy-Index'])
|
||||
self.assertEqual('1',
|
||||
resp.headers['X-Backend-Record-Storage-Policy-Index'])
|
||||
|
||||
# And an index that the broker doesn't have any objects for
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'})
|
||||
req.headers['X-Backend-Storage-Policy-Index'] = 2
|
||||
resp = req.get_response(self.controller)
|
||||
self.assertEqual(resp.status_int, 204)
|
||||
result = [o for o in resp.body.split(b'\n') if o]
|
||||
self.assertEqual(len(result), 0)
|
||||
self.assertFalse(result)
|
||||
self.assertIn('X-Backend-Storage-Policy-Index', resp.headers)
|
||||
self.assertEqual('0', resp.headers['X-Backend-Storage-Policy-Index'])
|
||||
self.assertEqual('2',
|
||||
resp.headers['X-Backend-Record-Storage-Policy-Index'])
|
||||
|
||||
# And an index that doesn't exist in POLICIES
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'})
|
||||
req.headers['X-Backend-Storage-Policy-Index'] = 3
|
||||
resp = req.get_response(self.controller)
|
||||
self.assertEqual(resp.status_int, 400)
|
||||
|
||||
def test_GET_limit(self):
|
||||
# make a container
|
||||
req = Request.blank(
|
||||
|
||||
@ -610,25 +610,30 @@ class TestContainerController(TestRingBase):
|
||||
('a/c', {'X-Backend-Record-Type': 'auto'},
|
||||
dict(states='listing')), # 200
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[0].name)),
|
||||
{'X-Backend-Record-Type': 'auto'},
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='', end_marker='ham\x00', limit=str(limit),
|
||||
states='listing')), # 200
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[1].name)),
|
||||
{'X-Backend-Record-Type': 'auto'},
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='h', end_marker='pie\x00', states='listing',
|
||||
limit=str(limit - len(sr_objs[0])))), # 200
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[2].name)),
|
||||
{'X-Backend-Record-Type': 'auto'},
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='p', end_marker='\xe2\x98\x83\x00', states='listing',
|
||||
limit=str(limit - len(sr_objs[0] + sr_objs[1])))), # 200
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[3].name)),
|
||||
{'X-Backend-Record-Type': 'auto'},
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='\xd1\xb0', end_marker='\xf0\x9f\x8c\xb4\x00',
|
||||
states='listing',
|
||||
limit=str(limit - len(sr_objs[0] + sr_objs[1]
|
||||
+ sr_objs[2])))), # 200
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[4].name)),
|
||||
{'X-Backend-Record-Type': 'auto'},
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='\xe2\xa8\x83', end_marker='', states='listing',
|
||||
limit=str(limit - len(sr_objs[0] + sr_objs[1] + sr_objs[2]
|
||||
+ sr_objs[3])))), # 200
|
||||
@ -653,13 +658,19 @@ class TestContainerController(TestRingBase):
|
||||
# path, headers, params
|
||||
('a/c', {'X-Backend-Record-Type': 'auto'},
|
||||
dict(states='listing')), # 200
|
||||
(shard_ranges[0].name, {'X-Backend-Record-Type': 'auto'},
|
||||
(shard_ranges[0].name,
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='', end_marker='ham\x00', limit=str(limit),
|
||||
states='listing')), # 200
|
||||
(shard_ranges[1].name, {'X-Backend-Record-Type': 'auto'},
|
||||
(shard_ranges[1].name,
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='h', end_marker='pie\x00', states='listing',
|
||||
limit=str(limit - len(sr_objs[0])))), # 200
|
||||
(root_range.name, {'X-Backend-Record-Type': 'object'},
|
||||
(root_range.name,
|
||||
{'X-Backend-Record-Type': 'object',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='p', end_marker='',
|
||||
limit=str(limit - len(sr_objs[0] + sr_objs[1])))) # 200
|
||||
]
|
||||
@ -685,27 +696,32 @@ class TestContainerController(TestRingBase):
|
||||
('a/c', {'X-Backend-Record-Type': 'auto'},
|
||||
dict(states='listing', reverse='true', limit='')),
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[4].name)),
|
||||
{'X-Backend-Record-Type': 'auto'},
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='', end_marker='\xf0\x9f\x8c\xb4', states='listing',
|
||||
reverse='true', limit=str(limit))), # 200
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[3].name)),
|
||||
{'X-Backend-Record-Type': 'auto'},
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='\xf0\x9f\x8c\xb5', end_marker='\xe2\x98\x83',
|
||||
states='listing', reverse='true',
|
||||
limit=str(limit - len(sr_objs[4])))), # 200
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[2].name)),
|
||||
{'X-Backend-Record-Type': 'auto'},
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='\xe2\x98\x84', end_marker='pie', states='listing',
|
||||
reverse='true',
|
||||
limit=str(limit - len(sr_objs[4] + sr_objs[3])))), # 200
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[1].name)),
|
||||
{'X-Backend-Record-Type': 'auto'},
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='q', end_marker='ham', states='listing',
|
||||
reverse='true',
|
||||
limit=str(limit - len(sr_objs[4] + sr_objs[3]
|
||||
+ sr_objs[2])))), # 200
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[0].name)),
|
||||
{'X-Backend-Record-Type': 'auto'},
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='i', end_marker='', states='listing', reverse='true',
|
||||
limit=str(limit - len(sr_objs[4] + sr_objs[3] + sr_objs[2]
|
||||
+ sr_objs[1])))), # 200
|
||||
@ -735,15 +751,18 @@ class TestContainerController(TestRingBase):
|
||||
('a/c', {'X-Backend-Record-Type': 'auto'},
|
||||
dict(limit=str(limit), states='listing')), # 200
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[0].name)),
|
||||
{'X-Backend-Record-Type': 'auto'}, # 200
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'}, # 200
|
||||
dict(marker='', end_marker='ham\x00', states='listing',
|
||||
limit=str(limit))),
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[1].name)),
|
||||
{'X-Backend-Record-Type': 'auto'}, # 200
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'}, # 200
|
||||
dict(marker='h', end_marker='pie\x00', states='listing',
|
||||
limit=str(limit - len(sr_objs[0])))),
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[2].name)),
|
||||
{'X-Backend-Record-Type': 'auto'}, # 200
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'}, # 200
|
||||
dict(marker='p', end_marker='\xe2\x98\x83\x00', states='listing',
|
||||
limit=str(limit - len(sr_objs[0] + sr_objs[1])))),
|
||||
]
|
||||
@ -771,15 +790,18 @@ class TestContainerController(TestRingBase):
|
||||
('a/c', {'X-Backend-Record-Type': 'auto'},
|
||||
dict(marker=marker, states='listing')), # 200
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[3].name)),
|
||||
{'X-Backend-Record-Type': 'auto'}, # 200
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'}, # 200
|
||||
dict(marker=marker, end_marker='\xf0\x9f\x8c\xb4\x00',
|
||||
states='listing', limit=str(limit))),
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[3].name)),
|
||||
{'X-Backend-Record-Type': 'auto'}, # 200
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'}, # 200
|
||||
dict(marker=marker, end_marker='\xf0\x9f\x8c\xb4\x00',
|
||||
states='listing', limit=str(limit))),
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[4].name)),
|
||||
{'X-Backend-Record-Type': 'auto'}, # 200
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'}, # 200
|
||||
dict(marker='\xe2\xa8\x83', end_marker='', states='listing',
|
||||
limit=str(limit - len(sr_objs[3][2:])))),
|
||||
]
|
||||
@ -809,28 +831,34 @@ class TestContainerController(TestRingBase):
|
||||
('a/c', {'X-Backend-Record-Type': 'auto'},
|
||||
dict(end_marker=end_marker, states='listing')), # 200
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[0].name)),
|
||||
{'X-Backend-Record-Type': 'auto'}, # 200
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'}, # 200
|
||||
dict(marker='', end_marker='ham\x00', states='listing',
|
||||
limit=str(limit))),
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[1].name)),
|
||||
{'X-Backend-Record-Type': 'auto'}, # 404
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'}, # 404
|
||||
dict(marker='h', end_marker='pie\x00', states='listing',
|
||||
limit=str(limit - len(sr_objs[0])))),
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[1].name)),
|
||||
{'X-Backend-Record-Type': 'auto'}, # 200
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'}, # 200
|
||||
dict(marker='h', end_marker='pie\x00', states='listing',
|
||||
limit=str(limit - len(sr_objs[0])))),
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[2].name)),
|
||||
{'X-Backend-Record-Type': 'auto'}, # 200
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'}, # 200
|
||||
dict(marker='p', end_marker='\xe2\x98\x83\x00', states='listing',
|
||||
limit=str(limit - len(sr_objs[0] + sr_objs[1])))),
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[3].name)),
|
||||
{'X-Backend-Record-Type': 'auto'}, # 404
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'}, # 404
|
||||
dict(marker='\xd1\xb0', end_marker=end_marker, states='listing',
|
||||
limit=str(limit - len(sr_objs[0] + sr_objs[1]
|
||||
+ sr_objs[2])))),
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[3].name)),
|
||||
{'X-Backend-Record-Type': 'auto'}, # 200
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'}, # 200
|
||||
dict(marker='\xd1\xb0', end_marker=end_marker, states='listing',
|
||||
limit=str(limit - len(sr_objs[0] + sr_objs[1]
|
||||
+ sr_objs[2])))),
|
||||
@ -856,7 +884,8 @@ class TestContainerController(TestRingBase):
|
||||
('a/c', {'X-Backend-Record-Type': 'auto'},
|
||||
dict(prefix=prefix, states='listing')), # 200
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[1].name)),
|
||||
{'X-Backend-Record-Type': 'auto'}, # 404
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'}, # 404
|
||||
dict(prefix=prefix, marker='', end_marker='pie\x00',
|
||||
states='listing', limit=str(limit))),
|
||||
]
|
||||
@ -877,7 +906,8 @@ class TestContainerController(TestRingBase):
|
||||
dict(states='listing', limit=str(limit),
|
||||
marker=marker, end_marker=end_marker)), # 200
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[3].name)),
|
||||
{'X-Backend-Record-Type': 'auto'}, # 200
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'}, # 200
|
||||
dict(marker=marker, end_marker=end_marker, states='listing',
|
||||
limit=str(limit))),
|
||||
]
|
||||
@ -898,7 +928,8 @@ class TestContainerController(TestRingBase):
|
||||
dict(marker=end_marker, reverse='true', end_marker=marker,
|
||||
limit=str(limit), states='listing',)), # 200
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[3].name)),
|
||||
{'X-Backend-Record-Type': 'auto'}, # 200
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'}, # 200
|
||||
dict(marker=end_marker, end_marker=marker, states='listing',
|
||||
limit=str(limit), reverse='true')),
|
||||
]
|
||||
@ -1720,28 +1751,160 @@ class TestContainerController(TestRingBase):
|
||||
('a/c', {'X-Backend-Record-Type': 'auto'},
|
||||
dict(states='listing')), # 200
|
||||
# get first shard objects
|
||||
(shard_ranges[0].name, {'X-Backend-Record-Type': 'auto'},
|
||||
(shard_ranges[0].name,
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='', end_marker='ham\x00', states='listing',
|
||||
limit=str(limit))), # 200
|
||||
# get second shard sub-shard ranges
|
||||
(shard_ranges[1].name, {'X-Backend-Record-Type': 'auto'},
|
||||
(shard_ranges[1].name,
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='h', end_marker='pie\x00', states='listing',
|
||||
limit=str(limit - len(sr_objs[0])))),
|
||||
# get first sub-shard objects
|
||||
(sub_shard_ranges[0].name, {'X-Backend-Record-Type': 'auto'},
|
||||
(sub_shard_ranges[0].name,
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='h', end_marker='juice\x00', states='listing',
|
||||
limit=str(limit - len(sr_objs[0])))),
|
||||
# get second sub-shard objects
|
||||
(sub_shard_ranges[1].name, {'X-Backend-Record-Type': 'auto'},
|
||||
(sub_shard_ranges[1].name,
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='j', end_marker='lemon\x00', states='listing',
|
||||
limit=str(limit - len(sr_objs[0] + sub_sr_objs[0])))),
|
||||
# get remainder of first shard objects
|
||||
(shard_ranges[1].name, {'X-Backend-Record-Type': 'object'},
|
||||
(shard_ranges[1].name,
|
||||
{'X-Backend-Record-Type': 'object',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='l', end_marker='pie\x00',
|
||||
limit=str(limit - len(sr_objs[0] + sub_sr_objs[0] +
|
||||
sub_sr_objs[1])))), # 200
|
||||
# get third shard objects
|
||||
(shard_ranges[2].name, {'X-Backend-Record-Type': 'auto'},
|
||||
(shard_ranges[2].name,
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='p', end_marker='', states='listing',
|
||||
limit=str(limit - len(sr_objs[0] + sr_objs[1])))) # 200
|
||||
]
|
||||
expected_objects = (
|
||||
sr_objs[0] + sub_sr_objs[0] + sub_sr_objs[1] +
|
||||
sr_objs[1][len(sub_sr_objs[0] + sub_sr_objs[1]):] + sr_objs[2])
|
||||
resp = self._check_GET_shard_listing(
|
||||
mock_responses, expected_objects, expected_requests)
|
||||
# root object count will overridden by actual length of listing
|
||||
self.check_response(resp, root_resp_hdrs)
|
||||
|
||||
@patch_policies([
|
||||
StoragePolicy(0, 'zero', True, object_ring=FakeRing()),
|
||||
StoragePolicy(1, 'one', False, object_ring=FakeRing())
|
||||
])
|
||||
def test_GET_sharded_container_sharding_shard_mixed_policies(self):
|
||||
# scenario: one shard is in process of sharding, shards have different
|
||||
# policy than root, expect listing to always request root policy index
|
||||
shard_bounds = (('', 'ham'), ('ham', 'pie'), ('pie', ''))
|
||||
shard_ranges = [
|
||||
ShardRange('.shards_a/c_' + upper, Timestamp.now(), lower, upper)
|
||||
for lower, upper in shard_bounds]
|
||||
sr_dicts = [dict(sr) for sr in shard_ranges]
|
||||
sr_objs = [self._make_shard_objects(sr) for sr in shard_ranges]
|
||||
shard_resp_hdrs = [
|
||||
{'X-Backend-Sharding-State': 'unsharded',
|
||||
'X-Container-Object-Count': len(sr_objs[i]),
|
||||
'X-Container-Bytes-Used':
|
||||
sum([obj['bytes'] for obj in sr_objs[i]]),
|
||||
'X-Container-Meta-Flavour': 'flavour%d' % i,
|
||||
'X-Backend-Storage-Policy-Index': 1,
|
||||
'X-Backend-Record-Storage-Policy-Index': 0}
|
||||
for i in range(3)]
|
||||
shard_1_shard_resp_hdrs = dict(shard_resp_hdrs[1])
|
||||
shard_1_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard'
|
||||
|
||||
# second shard is sharding and has cleaved two out of three sub shards
|
||||
shard_resp_hdrs[1]['X-Backend-Sharding-State'] = 'sharding'
|
||||
sub_shard_bounds = (('ham', 'juice'), ('juice', 'lemon'))
|
||||
sub_shard_ranges = [
|
||||
ShardRange('a/c_sub_' + upper, Timestamp.now(), lower, upper)
|
||||
for lower, upper in sub_shard_bounds]
|
||||
sub_sr_dicts = [dict(sr) for sr in sub_shard_ranges]
|
||||
sub_sr_objs = [self._make_shard_objects(sr) for sr in sub_shard_ranges]
|
||||
sub_shard_resp_hdrs = [
|
||||
{'X-Backend-Sharding-State': 'unsharded',
|
||||
'X-Container-Object-Count': len(sub_sr_objs[i]),
|
||||
'X-Container-Bytes-Used':
|
||||
sum([obj['bytes'] for obj in sub_sr_objs[i]]),
|
||||
'X-Container-Meta-Flavour': 'flavour%d' % i,
|
||||
'X-Backend-Storage-Policy-Index': 1,
|
||||
'X-Backend-Record-Storage-Policy-Index': 0}
|
||||
for i in range(2)]
|
||||
|
||||
all_objects = []
|
||||
for objects in sr_objs:
|
||||
all_objects.extend(objects)
|
||||
size_all_objects = sum([obj['bytes'] for obj in all_objects])
|
||||
num_all_objects = len(all_objects)
|
||||
limit = CONTAINER_LISTING_LIMIT
|
||||
root_resp_hdrs = {'X-Backend-Sharding-State': 'sharded',
|
||||
'X-Backend-Timestamp': '99',
|
||||
'X-Container-Object-Count': num_all_objects,
|
||||
'X-Container-Bytes-Used': size_all_objects,
|
||||
'X-Container-Meta-Flavour': 'peach',
|
||||
'X-Backend-Storage-Policy-Index': 0}
|
||||
root_shard_resp_hdrs = dict(root_resp_hdrs)
|
||||
root_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard'
|
||||
|
||||
mock_responses = [
|
||||
# status, body, headers
|
||||
(200, sr_dicts, root_shard_resp_hdrs),
|
||||
(200, sr_objs[0], shard_resp_hdrs[0]),
|
||||
(200, sub_sr_dicts + [sr_dicts[1]], shard_1_shard_resp_hdrs),
|
||||
(200, sub_sr_objs[0], sub_shard_resp_hdrs[0]),
|
||||
(200, sub_sr_objs[1], sub_shard_resp_hdrs[1]),
|
||||
(200, sr_objs[1][len(sub_sr_objs[0] + sub_sr_objs[1]):],
|
||||
shard_resp_hdrs[1]),
|
||||
(200, sr_objs[2], shard_resp_hdrs[2])
|
||||
]
|
||||
# NB marker always advances to last object name
|
||||
expected_requests = [
|
||||
# get root shard ranges
|
||||
('a/c', {'X-Backend-Record-Type': 'auto'},
|
||||
dict(states='listing')), # 200
|
||||
# get first shard objects
|
||||
(shard_ranges[0].name,
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='', end_marker='ham\x00', states='listing',
|
||||
limit=str(limit))), # 200
|
||||
# get second shard sub-shard ranges
|
||||
(shard_ranges[1].name,
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='h', end_marker='pie\x00', states='listing',
|
||||
limit=str(limit - len(sr_objs[0])))),
|
||||
# get first sub-shard objects
|
||||
(sub_shard_ranges[0].name,
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='h', end_marker='juice\x00', states='listing',
|
||||
limit=str(limit - len(sr_objs[0])))),
|
||||
# get second sub-shard objects
|
||||
(sub_shard_ranges[1].name,
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='j', end_marker='lemon\x00', states='listing',
|
||||
limit=str(limit - len(sr_objs[0] + sub_sr_objs[0])))),
|
||||
# get remainder of second shard objects
|
||||
(shard_ranges[1].name,
|
||||
{'X-Backend-Record-Type': 'object',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='l', end_marker='pie\x00',
|
||||
limit=str(limit - len(sr_objs[0] + sub_sr_objs[0] +
|
||||
sub_sr_objs[1])))), # 200
|
||||
# get third shard objects
|
||||
(shard_ranges[2].name,
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='p', end_marker='', states='listing',
|
||||
limit=str(limit - len(sr_objs[0] + sr_objs[1])))) # 200
|
||||
]
|
||||
@ -1927,6 +2090,88 @@ class TestContainerController(TestRingBase):
|
||||
('delete', 'shard-listing/a/c', None, None)],
|
||||
self.memcache.calls)
|
||||
|
||||
def test_get_from_shards_add_root_spi(self):
|
||||
self._setup_shard_range_stubs()
|
||||
shard_resp = mock.MagicMock(status_int=204, headers={})
|
||||
|
||||
def mock_get_container_listing(self_, req, *args, **kargs):
|
||||
captured_hdrs.update(req.headers)
|
||||
return None, shard_resp
|
||||
|
||||
# header in response -> header added to request
|
||||
captured_hdrs = {}
|
||||
req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'GET'})
|
||||
resp = mock.MagicMock(body=self._stub_shards_dump,
|
||||
headers=self.root_resp_hdrs,
|
||||
request=req)
|
||||
resp.headers['X-Backend-Storage-Policy-Index'] = '0'
|
||||
with mock.patch('swift.proxy.controllers.container.'
|
||||
'ContainerController._get_container_listing',
|
||||
mock_get_container_listing):
|
||||
controller_cls, d = self.app.get_controller(req)
|
||||
controller = controller_cls(self.app, **d)
|
||||
controller._get_from_shards(req, resp)
|
||||
|
||||
self.assertIn('X-Backend-Storage-Policy-Index', captured_hdrs)
|
||||
self.assertEqual(
|
||||
captured_hdrs['X-Backend-Storage-Policy-Index'], '0')
|
||||
|
||||
captured_hdrs = {}
|
||||
req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'GET'})
|
||||
resp = mock.MagicMock(body=self._stub_shards_dump,
|
||||
headers=self.root_resp_hdrs,
|
||||
request=req)
|
||||
resp.headers['X-Backend-Storage-Policy-Index'] = '1'
|
||||
with mock.patch('swift.proxy.controllers.container.'
|
||||
'ContainerController._get_container_listing',
|
||||
mock_get_container_listing):
|
||||
controller_cls, d = self.app.get_controller(req)
|
||||
controller = controller_cls(self.app, **d)
|
||||
controller._get_from_shards(req, resp)
|
||||
|
||||
self.assertIn('X-Backend-Storage-Policy-Index', captured_hdrs)
|
||||
self.assertEqual(
|
||||
captured_hdrs['X-Backend-Storage-Policy-Index'], '1')
|
||||
|
||||
# header not added to request if not root request
|
||||
captured_hdrs = {}
|
||||
req = Request.blank('/v1/a/c',
|
||||
environ={
|
||||
'REQUEST_METHOD': 'GET',
|
||||
'swift.shard_listing_history': [('a', 'c')]}
|
||||
)
|
||||
resp = mock.MagicMock(body=self._stub_shards_dump,
|
||||
headers=self.root_resp_hdrs,
|
||||
request=req)
|
||||
resp.headers['X-Backend-Storage-Policy-Index'] = '0'
|
||||
with mock.patch('swift.proxy.controllers.container.'
|
||||
'ContainerController._get_container_listing',
|
||||
mock_get_container_listing):
|
||||
controller_cls, d = self.app.get_controller(req)
|
||||
controller = controller_cls(self.app, **d)
|
||||
controller._get_from_shards(req, resp)
|
||||
|
||||
self.assertNotIn('X-Backend-Storage-Policy-Index', captured_hdrs)
|
||||
|
||||
# existing X-Backend-Storage-Policy-Index in request is respected
|
||||
captured_hdrs = {}
|
||||
req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'GET'})
|
||||
req.headers['X-Backend-Storage-Policy-Index'] = '0'
|
||||
resp = mock.MagicMock(body=self._stub_shards_dump,
|
||||
headers=self.root_resp_hdrs,
|
||||
request=req)
|
||||
resp.headers['X-Backend-Storage-Policy-Index'] = '1'
|
||||
with mock.patch('swift.proxy.controllers.container.'
|
||||
'ContainerController._get_container_listing',
|
||||
mock_get_container_listing):
|
||||
controller_cls, d = self.app.get_controller(req)
|
||||
controller = controller_cls(self.app, **d)
|
||||
controller._get_from_shards(req, resp)
|
||||
|
||||
self.assertIn('X-Backend-Storage-Policy-Index', captured_hdrs)
|
||||
self.assertEqual(
|
||||
captured_hdrs['X-Backend-Storage-Policy-Index'], '0')
|
||||
|
||||
def test_GET_shard_ranges(self):
|
||||
self._setup_shard_range_stubs()
|
||||
# expect shard ranges cache time to be default value of 600
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user