diff --git a/swift/proxy/controllers/base.py b/swift/proxy/controllers/base.py index f19f160089..d59608cef0 100644 --- a/swift/proxy/controllers/base.py +++ b/swift/proxy/controllers/base.py @@ -896,8 +896,10 @@ def get_namespaces_from_cache(req, cache_key, skip_chance): :param req: a :class:`swift.common.swob.Request` object. :param cache_key: the cache key for both infocache and memcache. :param skip_chance: the probability of skipping the memcache look-up. - :return: a tuple of - (:class:`swift.common.utils.NamespaceBoundList`, cache state) + :return: a tuple of (value, cache state). Value is an instance of + :class:`swift.common.utils.NamespaceBoundList` if a non-empty list is + found in memcache. Otherwise value is ``None``, for example if memcache + look-up was skipped, or no value was found, or an empty list was found. """ # try get namespaces from infocache first infocache = req.environ.setdefault('swift.infocache', {}) diff --git a/swift/proxy/controllers/container.py b/swift/proxy/controllers/container.py index e8e0ba5a50..6226b7a703 100644 --- a/swift/proxy/controllers/container.py +++ b/swift/proxy/controllers/container.py @@ -108,15 +108,14 @@ class ContainerController(Controller): req.swift_entity_path, concurrency) return resp - def _make_namespaces_response_body(self, req, ns_bound_list): + def _filter_complete_listing(self, req, namespaces): """ - Filter namespaces according to request constraints and return a - serialised list of namespaces. + Filter complete list of namespaces to return only those specified by + the request constraints. - :param req: the request object. - :param ns_bound_list: an instance of - :class:`~swift.common.utils.NamespaceBoundList`. - :return: a serialised list of namespaces. + :param req: a :class:`~swift.common.swob.Request`. + :param namespaces: a list of :class:`~swift.common.utils.Namespace`. + :return: a list of :class:`~swift.common.utils.Namespace`. """ marker = get_param(req, 'marker', '') end_marker = get_param(req, 'end_marker') @@ -124,141 +123,150 @@ class ContainerController(Controller): reverse = config_true_value(get_param(req, 'reverse')) if reverse: marker, end_marker = end_marker, marker - namespaces = ns_bound_list.get_namespaces() namespaces = filter_namespaces( namespaces, includes, marker, end_marker) if reverse: namespaces.reverse() - return json.dumps([dict(ns) for ns in namespaces]).encode('ascii') + return namespaces - def _get_shard_ranges_from_cache(self, req, headers): + def _get_listing_namespaces_from_cache(self, req, headers): """ Try to fetch shard namespace data from cache and, if successful, return - a response. Also return the cache state. - - The response body will be a list of dicts each of which describes - a Namespace (i.e. includes the keys ``lower``, ``upper`` and ``name``). + a list of Namespaces. Also return the cache state. :param req: an instance of ``swob.Request``. - :param headers: Headers to be sent with request. - :return: a tuple comprising (an instance of ``swob.Response``or - ``None`` if no namespaces were found in cache, the cache state). + :return: a tuple comprising (a list instance of ``Namespace`` objects + or ``None`` if no namespaces were found in cache, the cache state). """ cache_key = get_cache_key(self.account_name, self.container_name, shard='listing') skip_chance = self.app.container_listing_shard_ranges_skip_cache ns_bound_list, cache_state = get_namespaces_from_cache( req, cache_key, skip_chance) - if ns_bound_list: - # shard ranges can be returned from cache - resp_body = self._make_namespaces_response_body(req, ns_bound_list) - self.logger.debug('Found %d shards in cache for %s', - len(ns_bound_list.bounds), req.path_qs) - headers.update({'x-backend-record-type': 'shard', - 'x-backend-cached-results': 'true'}) - # mimic GetOrHeadHandler.get_working_response... - # note: server sets charset with content_type but proxy - # GETorHEAD_base does not, so don't set it here either - resp = Response(request=req, body=resp_body) - update_headers(resp, headers) - resp.last_modified = Timestamp(headers['x-put-timestamp']).ceil() - resp.environ['swift_x_timestamp'] = headers.get('x-timestamp') - resp.accept_ranges = 'bytes' - resp.content_type = 'application/json' - else: - resp = None + if not ns_bound_list: + return None, None, cache_state - return resp, cache_state + # Namespaces found in cache so there is no need to go to backend, + # but we need to build response headers: mimic + # GetOrHeadHandler.get_working_response... + # note: server sets charset with content_type but proxy + # GETorHEAD_base does not, so don't set it here either + namespaces = ns_bound_list.get_namespaces() + self.logger.debug('Found %d shards in cache for %s', + len(namespaces), req.path_qs) + headers.update({'x-backend-record-type': 'shard', + 'x-backend-record-shard-format': 'namespace', + 'x-backend-cached-results': 'true'}) + resp = Response(request=req) + update_headers(resp, headers) + resp.last_modified = Timestamp(headers['x-put-timestamp']).ceil() + resp.environ['swift_x_timestamp'] = headers.get('x-timestamp') + resp.accept_ranges = 'bytes' + resp.content_type = 'application/json' + namespaces = self._filter_complete_listing(req, namespaces) + return resp, namespaces, cache_state - def _store_shard_ranges_in_cache(self, req, resp): + def _set_listing_namespaces_in_cache(self, req, namespaces): """ - Parse shard ranges returned from backend, store them in both infocache - and memcache. + Store a list of namespaces in both infocache and memcache. + + Note: the returned list of namespaces may not be identical to the given + list. Any gaps in the given namespaces will be 'lost' as a result of + compacting the list of namespaces to a NamespaceBoundList for caching. + That is ok. When the cached NamespaceBoundList is transformed back to + Namespaces to perform a listing, the Namespace before each gap will + have expanded to include the gap, which means that the backend GET to + that shard will have an end_marker beyond that shard's upper bound, and + equal to the next available shard's lower. At worst, some misplaced + objects, in the gap above the shard's upper, may be included in the + shard's response. :param req: the request object. - :param resp: the response object for the shard range listing. - :return: an instance of - :class:`~swift.common.utils.NamespaceBoundList`. + :param namespaces: a list of :class:`~swift.common.utils.Namespace` + objects. + :return: a list of :class:`~swift.common.utils.Namespace` objects. """ - # Note: Any gaps in the response's shard ranges will be 'lost' as a - # result of compacting the list of shard ranges to a - # NamespaceBoundList. That is ok. When the cached NamespaceBoundList is - # transformed back to shard range Namespaces to perform a listing, the - # Namespace before each gap will have expanded to include the gap, - # which means that the backend GET to that shard will have an - # end_marker beyond that shard's upper bound, and equal to the next - # available shard's lower. At worst, some misplaced objects, in the gap - # above the shard's upper, may be included in the shard's response. - data = self._parse_listing_response(req, resp) - backend_shard_ranges = self._parse_namespaces(req, data, resp) - if backend_shard_ranges is None: - return None + cache_key = get_cache_key(self.account_name, self.container_name, + shard='listing') + ns_bound_list = NamespaceBoundList.parse(namespaces) + # cache in infocache even if no namespaces returned; this + # is unexpected but use that result for this request + set_cache_state = set_namespaces_in_cache( + req, cache_key, ns_bound_list, + self.app.recheck_listing_shard_ranges) + if set_cache_state == 'set': + self.logger.info( + 'Caching listing namespaces for %s (%d namespaces)', + cache_key, len(ns_bound_list.bounds)) + # return the de-gapped namespaces + return ns_bound_list.get_namespaces() - ns_bound_list = NamespaceBoundList.parse(backend_shard_ranges) - if resp.headers.get('x-backend-sharding-state') == 'sharded': - # cache in infocache even if no shard ranges returned; this - # is unexpected but use that result for this request - cache_key = get_cache_key( - self.account_name, self.container_name, shard='listing') - set_cache_state = set_namespaces_in_cache( - req, cache_key, ns_bound_list, - self.app.recheck_listing_shard_ranges) - if set_cache_state == 'set': - self.logger.info( - 'Caching listing namespaces for %s (%d namespaces)', - cache_key, len(ns_bound_list.bounds)) - return ns_bound_list - - def _get_shard_ranges_from_backend(self, req): + def _get_listing_namespaces_from_backend(self, req, cache_enabled): """ - Make a backend request for shard ranges and return a response. - - The response body will be a list of dicts each of which describes - a Namespace (i.e. includes the keys ``lower``, ``upper`` and ``name``). - If the response headers indicate that the response body contains a - complete list of shard ranges for a sharded container then the response - body will be transformed to a ``NamespaceBoundsList`` and cached. + Fetch shard namespace data from the backend and, if successful, return + a list of Namespaces. :param req: an instance of ``swob.Request``. - :return: an instance of ``swob.Response``. + :param cache_enabled: a boolean which should be True if memcache is + available to cache the returned data, False otherwise. + :return: a list instance of ``Namespace`` objects or ``None`` if no + namespace data was returned from the backend. """ - # Note: We instruct the backend server to ignore name constraints in - # request params if returning shard ranges so that the response can - # potentially be cached, but we only cache it if the container state is - # 'sharded'. We don't attempt to cache shard ranges for a 'sharding' - # container as they may include the container itself as a 'gap filler' - # for shard ranges that have not yet cleaved; listings from 'gap - # filler' shard ranges are likely to become stale as the container - # continues to cleave objects to its shards and caching them is - # therefore more likely to result in stale or incomplete listings on - # subsequent container GETs. - req.headers['x-backend-override-shard-name-filter'] = 'sharded' + # Instruct the backend server to 'automatically' return namespaces + # of shards in a 'listing' state if the container is sharded, and + # that the more compact 'namespace' format is sufficient. Older + # container servers may still respond with the 'full' shard range + # format. + req.headers['X-Backend-Record-Type'] = 'auto' + req.headers['X-Backend-Record-Shard-Format'] = 'namespace' + # 'x-backend-include-deleted' is not expected in 'auto' requests to + # the proxy (it's not supported for objects and is used by the + # sharder when explicitly fetching 'shard' record type), but we + # explicitly set it to false here just in case. A newer container + # server would ignore it when returning namespaces, but an older + # container server would include unwanted deleted shard range. + req.headers['X-Backend-Include-Deleted'] = 'false' + params = req.params + params['states'] = 'listing' + req.params = params + if cache_enabled: + # Instruct the backend server to ignore name constraints in + # request params if returning namespaces so that the response + # can potentially be cached, but only if the container state is + # 'sharded'. We don't attempt to cache namespaces for a + # 'sharding' container as they may include the container itself + # as a 'gap filler' for shards that have not yet cleaved; + # listings from 'gap filler' namespaces are likely to become + # stale as the container continues to cleave objects to its + # shards and caching them is therefore more likely to result in + # stale or incomplete listings on subsequent container GETs. + req.headers['x-backend-override-shard-name-filter'] = 'sharded' resp = self._GETorHEAD_from_backend(req) - - sharding_state = resp.headers.get( - 'x-backend-sharding-state', '').lower() resp_record_type = resp.headers.get( 'x-backend-record-type', '').lower() + sharding_state = resp.headers.get( + 'x-backend-sharding-state', '').lower() complete_listing = config_true_value(resp.headers.pop( 'x-backend-override-shard-name-filter', False)) - # given that we sent 'x-backend-override-shard-name-filter=sharded' we - # should only receive back 'x-backend-override-shard-name-filter=true' - # if the sharding state is 'sharded', but check them both anyway... - if (resp_record_type == 'shard' and - sharding_state == 'sharded' and - complete_listing): - # note: old container servers return a list of shard ranges, newer - # ones return a list of namespaces. If we ever need to know we can - # look for a 'x-backend-record-shard-format' header from newer - # container servers. - ns_bound_list = self._store_shard_ranges_in_cache(req, resp) - if ns_bound_list: - resp.body = self._make_namespaces_response_body( - req, ns_bound_list) - return resp + if resp_record_type == 'shard': + data = self._parse_listing_response(req, resp) + namespaces = self._parse_namespaces(req, data, resp) + # given that we sent + # 'x-backend-override-shard-name-filter=sharded' we should only + # receive back 'x-backend-override-shard-name-filter=true' if + # the sharding state is 'sharded', but check them both + # anyway... + if (namespaces and + sharding_state == 'sharded' and + complete_listing): + namespaces = self._set_listing_namespaces_in_cache( + req, namespaces) + namespaces = self._filter_complete_listing(req, namespaces) + else: + namespaces = None + return resp, namespaces - def _record_shard_listing_cache_metrics( - self, cache_state, resp, resp_record_type, info): + def _record_shard_listing_cache_metrics(self, cache_state, resp, info): """ Record a single cache operation by shard listing into its corresponding metrics. @@ -267,21 +275,19 @@ class ContainerController(Controller): infocache_hit, memcache hit, miss, error, skip, force_skip and disabled. :param resp: the response from either backend or cache hit. - :param resp_record_type: indicates the type of response record, e.g. - 'shard' for shard range listing, 'object' for object listing. :param info: the cached container info. """ should_record = False if is_success(resp.status_int): - if resp_record_type == 'shard': - # Here we either got shard ranges by hitting the cache, or we - # got shard ranges from backend successfully for cache_state + if resp.headers.get('X-Backend-Record-Type', '') == 'shard': + # Here we either got namespaces by hitting the cache, or we + # got namespaces from backend successfully for cache_state # other than cache hit. Note: it's possible that later we find - # that shard ranges can't be parsed. + # that namespaces can't be parsed. should_record = True elif (info and is_success(info['status']) and info.get('sharding_state') == 'sharded'): - # The shard listing request failed when getting shard ranges from + # The shard listing request failed when getting namespaces from # backend. # Note: In the absence of 'info' we cannot assume the container is # sharded, so we don't increment the metric if 'info' is None. Even @@ -298,34 +304,55 @@ class ContainerController(Controller): self.logger, self.server_type.lower(), 'shard_listing', cache_state, resp) - def _GET_using_cache(self, req, info): - # It may be possible to fulfil the request from cache: we only reach - # here if request record_type is 'shard' or 'auto', so if the container - # state is 'sharded' then look for cached shard ranges. However, if - # X-Newest is true then we always fetch from the backend servers. - headers = headers_from_container_info(info) - if config_true_value(req.headers.get('x-newest', False)): - cache_state = 'force_skip' - self.logger.debug( - 'Skipping shard cache lookup (x-newest) for %s', req.path_qs) - elif (headers and info and is_success(info['status']) and - info.get('sharding_state') == 'sharded'): - # container is sharded so we may have the shard ranges cached; only - # use cached values if all required backend headers available. - resp, cache_state = self._get_shard_ranges_from_cache(req, headers) - if resp: - return resp, cache_state + def _GET_auto(self, req): + # This is an object listing but the backend may be sharded. + # Only lookup container info from cache and skip the backend HEAD, + # since we are going to GET the backend container anyway. + info = get_container_info( + req.environ, self.app, swift_source=None, cache_only=True) + memcache = cache_from_env(req.environ, True) + cache_enabled = self.app.recheck_listing_shard_ranges > 0 and memcache + resp = namespaces = None + if cache_enabled: + # if the container is sharded we may look for namespaces in cache + headers = headers_from_container_info(info) + if config_true_value(req.headers.get('x-newest', False)): + cache_state = 'force_skip' + self.logger.debug( + 'Skipping shard cache lookup (x-newest) for %s', + req.path_qs) + elif (headers and is_success(info['status']) and + info.get('sharding_state') == 'sharded'): + # container is sharded so we may have the namespaces cached, + # but only use cached namespaces if all required response + # headers are also available from cache. + resp, namespaces, cache_state = \ + self._get_listing_namespaces_from_cache(req, headers) + else: + # container metadata didn't support a cache lookup, this could + # be the case that container metadata was not in cache and we + # don't know if the container was sharded, or the case that the + # sharding state in metadata indicates the container was + # unsharded. + cache_state = 'bypass' else: - # container metadata didn't support a cache lookup, this could be - # the case that container metadata was not in cache and we don't - # know if the container was sharded, or the case that the sharding - # state in metadata indicates the container was unsharded. - cache_state = 'bypass' - # The request was not fulfilled from cache so send to backend server. - return self._get_shard_ranges_from_backend(req), cache_state + cache_state = 'disabled' - def GETorHEAD(self, req): - """Handler for HTTP GET/HEAD requests.""" + if not namespaces: + resp, namespaces = self._get_listing_namespaces_from_backend( + req, cache_enabled) + self._record_shard_listing_cache_metrics(cache_state, resp, info) + + if namespaces is not None: + # we got namespaces, so the container must be sharded; now build + # the listing from shards + # NB: the filtered namespaces list may be empty but we still need + # to build a response body with an empty list of objects + resp = self._get_from_shards(req, resp, namespaces) + + return resp + + def _get_or_head_pre_check(self, req): ai = self.account_info(self.account_name, req) auto_account = self.account_name.startswith( self.app.auto_create_account_prefix) @@ -339,78 +366,9 @@ class ContainerController(Controller): # Don't cache this. The lack of account will be cached, and that # is sufficient. return HTTPNotFound(request=req) + return None - # The read-modify-write of params here is because the Request.params - # getter dynamically generates a dict of params from the query string; - # the setter must be called for new params to update the query string. - params = req.params - params['format'] = 'json' - # x-backend-record-type may be sent via internal client e.g. from the - # sharder, or by the proxy itself when making a recursive request, or - # in probe tests. If the header is present then the only values that - # the proxy respects are 'object' or 'shard'. However, the proxy may - # use the value 'auto' when making requests to container server. - orig_record_type = req.headers.get('X-Backend-Record-Type', '').lower() - if orig_record_type in ('object', 'shard'): - record_type = orig_record_type - else: - record_type = 'auto' - req.headers['X-Backend-Record-Type'] = 'auto' - req.headers['X-Backend-Record-Shard-Format'] = 'namespace' - params['states'] = 'listing' - req.params = params - - if (req.method == 'GET' - and get_param(req, 'states') == 'listing' - and record_type != 'object'): - may_get_listing_shards = True - # Only lookup container info from cache and skip the backend HEAD, - # since we are going to GET the backend container anyway. - info = get_container_info( - req.environ, self.app, swift_source=None, cache_only=True) - else: - info = None - may_get_listing_shards = False - - memcache = cache_from_env(req.environ, True) - sr_cache_state = None - if (may_get_listing_shards and - self.app.recheck_listing_shard_ranges > 0 - and memcache - and not config_true_value( - req.headers.get('x-backend-include-deleted', False))): - # This GET might be served from cache or might populate cache. - # 'x-backend-include-deleted' is not usually expected in requests - # to the proxy (it is used from sharder to container servers) but - # it is included in the conditions just in case because we don't - # cache deleted shard ranges. - resp, sr_cache_state = self._GET_using_cache(req, info) - else: - resp = self._GETorHEAD_from_backend(req) - if may_get_listing_shards and ( - not self.app.recheck_listing_shard_ranges or not memcache): - sr_cache_state = 'disabled' - - resp_record_type = resp.headers.get('X-Backend-Record-Type', '') - if sr_cache_state: - self._record_shard_listing_cache_metrics( - sr_cache_state, resp, resp_record_type, info) - - if all((req.method == "GET", record_type == 'auto', - resp_record_type.lower() == 'shard')): - data = self._parse_listing_response(req, resp) - namespaces = self._parse_namespaces(req, data, resp) - if namespaces is not None: - # we got namespaces, so the container must be sharded; now - # build the listing from shards - # NB: the filtered namespaces list may be empty but we still - # need to build a response body with an empty list of shards - resp = self._get_from_shards(req, resp, namespaces) - - if orig_record_type not in ('object', 'shard'): - resp.headers.pop('X-Backend-Record-Type', None) - resp.headers.pop('X-Backend-Record-Shard-Format', None) - + def _get_or_head_post_check(self, req, resp): if not config_true_value( resp.headers.get('X-Backend-Cached-Results')): # Cache container metadata. We just made a request to a storage @@ -419,6 +377,7 @@ class ContainerController(Controller): self.app.recheck_container_existence) set_info_cache(req.environ, self.account_name, self.container_name, resp) + if 'swift.authorize' in req.environ: req.acl = wsgi_to_str(resp.headers.get('x-container-read')) aresp = req.environ['swift.authorize'](req) @@ -437,6 +396,51 @@ class ContainerController(Controller): 'False')) return resp + @public + @delay_denial + @cors_validation + def GET(self, req): + """Handler for HTTP GET requests.""" + # early checks for request validity + validate_container_params(req) + aresp = self._get_or_head_pre_check(req) + if aresp: + return aresp + + # Always request json format from the backend. listing_formats + # middleware will take care of what the client gets back. + # The read-modify-write of params here is because the + # Request.params getter dynamically generates a dict of params from + # the query string; the setter must be called for new params to + # update the query string. + params = req.params + params['format'] = 'json' + req.params = params + + # x-backend-record-type may be sent via internal client e.g. from + # the sharder or in probe tests + record_type = req.headers.get('X-Backend-Record-Type', '').lower() + if record_type in ('object', 'shard'): + # Go direct to the backend for HEADs, and GETs that *explicitly* + # specify a record type. We won't be reading/writing namespaces in + # cache nor building listings from shards. This path is used by + # the sharder, manage_shard_ranges and other tools that fetch shard + # ranges, and by the proxy itself when explicitly requesting + # objects while recursively building a listing from shards. + # Note: shard record type could be namespace or full format + resp = self._GETorHEAD_from_backend(req) + else: + # Requests that do not explicitly specify a record type, or specify + # 'auto', default to returning an object listing. The listing may + # be built from shards and may involve reading/writing namespaces + # in cache. This path is used for client requests and by the proxy + # itself while recursively building a listing from shards. + resp = self._GET_auto(req) + resp.headers.pop('X-Backend-Record-Type', None) + resp.headers.pop('X-Backend-Record-Shard-Format', None) + + return self._get_or_head_post_check(req, resp) + def _get_from_shards(self, req, resp, namespaces): """ Construct an object listing using shards described by the list of @@ -527,6 +531,8 @@ class ContainerController(Controller): shard_listing_history): # directed back to same container - force GET of objects headers['X-Backend-Record-Type'] = 'object' + else: + headers['X-Backend-Record-Type'] = 'auto' if config_true_value(req.headers.get('x-newest', False)): headers['X-Newest'] = 'true' @@ -611,21 +617,16 @@ class ContainerController(Controller): [o['bytes'] for o in objects]) return resp - @public - @delay_denial - @cors_validation - def GET(self, req): - """Handler for HTTP GET requests.""" - # early checks for request validity - validate_container_params(req) - return self.GETorHEAD(req) - @public @delay_denial @cors_validation def HEAD(self, req): """Handler for HTTP HEAD requests.""" - return self.GETorHEAD(req) + aresp = self._get_or_head_pre_check(req) + if aresp: + return aresp + resp = self._GETorHEAD_from_backend(req) + return self._get_or_head_post_check(req, resp) @public @cors_validation diff --git a/test/probe/test_sharder.py b/test/probe/test_sharder.py index 14a83746e6..cb2727e97f 100644 --- a/test/probe/test_sharder.py +++ b/test/probe/test_sharder.py @@ -2998,17 +2998,14 @@ class TestShardedAPI(BaseTestContainerSharding): params={'states': 'updating'}) self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges) - # XXX the states=listing param provokes the proxy to cache the backend - # values and then respond to the client with the cached *namespaces* !! - # shard_ranges = self.get_container_shard_ranges( - # params={'states': 'listing'}) - # self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges) + shard_ranges = self.get_container_shard_ranges( + params={'states': 'listing'}) + self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges) - # XXX ditto... - # shard_ranges = self.get_container_shard_ranges( - # headers={'X-Newest': 'true'}, - # params={'states': 'listing'}) - # self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges) + shard_ranges = self.get_container_shard_ranges( + headers={'X-Newest': 'true'}, + params={'states': 'listing'}) + self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges) # this is what the sharder requests... shard_ranges = self.get_container_shard_ranges( diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index f80ee1128f..bd692c8ea1 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -7904,6 +7904,15 @@ class TestNamespaceBoundList(unittest.TestCase): self.end_ns = utils.Namespace('a/z-', 'z', '') self.lowerbounds = [start, atof, ftol, ltor, rtoz, end] + def test_eq(self): + this = utils.NamespaceBoundList(self.lowerbounds) + that = utils.NamespaceBoundList(self.lowerbounds) + self.assertEqual(this, that) + that = utils.NamespaceBoundList(self.lowerbounds[:1]) + self.assertNotEqual(this, that) + self.assertNotEqual(this, None) + self.assertNotEqual(this, self.lowerbounds) + def test_get_namespace(self): namespace_list = utils.NamespaceBoundList(self.lowerbounds) self.assertEqual(namespace_list.bounds, self.lowerbounds) diff --git a/test/unit/proxy/controllers/test_container.py b/test/unit/proxy/controllers/test_container.py index 1d27579cf2..9bed448ed3 100644 --- a/test/unit/proxy/controllers/test_container.py +++ b/test/unit/proxy/controllers/test_container.py @@ -649,12 +649,18 @@ class TestGetShardedContainer(BaseTestContainerController): info = get_container_info(resp.request.environ, self.app) self.assertEqual(headers_to_container_info(info_hdrs), info) - def create_server_response_data(self, bounds, states=None): + def create_server_namespace_dict(self, name, lower, upper): + # return a dict representation of an instance of the type the backend + # server returns for shard format = 'namespace' + return dict(Namespace(name, lower, upper)) + + def create_server_response_data(self, bounds, states=None, + name_prefix='.shards_a/c_'): if not isinstance(bounds[0], (list, tuple)): bounds = [(l, u) for l, u in zip(bounds[:-1], bounds[1:])] # some tests use bounds with '/' char, so replace this before using the # upper bound to synthesize a valid container name - namespaces = [Namespace('.shards_a/c_%s' % upper.replace('/', '-'), + namespaces = [Namespace(name_prefix + upper.replace('/', '-'), lower, upper) for lower, upper in bounds] ns_dicts = [dict(ns) for ns in namespaces] @@ -1120,6 +1126,10 @@ class TestGetShardedContainer(BaseTestContainerController): # root object count will overridden by actual length of listing self.check_listing_response(resp, root_resp_hdrs, expected_objects=expected_objects) + self.assertIn('swift.cache', resp.request.environ) + cached_keys = set(k for k in resp.request.environ['swift.cache'].store + if k.startswith('shard-listing')) + self.assertEqual({'shard-listing-v2/a/c'}, cached_keys) resp = self._check_GET_shard_listing( mock_responses, expected_objects, expected_requests, memcache=True, @@ -1172,6 +1182,10 @@ class TestGetShardedContainer(BaseTestContainerController): # root object count will overridden by actual length of listing self.check_listing_response(resp, root_resp_hdrs, expected_objects=expected_objects) + self.assertIn('swift.cache', resp.request.environ) + cached_keys = set(k for k in resp.request.environ['swift.cache'].store + if k.startswith('shard-listing')) + self.assertEqual({'shard-listing-v2/a/c'}, cached_keys) # GET all objects in reverse and *blank* limit mock_responses = [ @@ -1232,6 +1246,10 @@ class TestGetShardedContainer(BaseTestContainerController): # root object count will overridden by actual length of listing self.check_listing_response(resp, root_resp_hdrs, expected_objects=expected_objects) + self.assertIn('swift.cache', resp.request.environ) + cached_keys = set(k for k in resp.request.environ['swift.cache'].store + if k.startswith('shard-listing')) + self.assertEqual({'shard-listing-v2/a/c'}, cached_keys) # GET with limit param limit = len(sr_objs[0]) + len(sr_objs[1]) + 1 @@ -1273,6 +1291,10 @@ class TestGetShardedContainer(BaseTestContainerController): mock_responses, expected_objects, expected_requests, query_string='?limit=%s' % limit, memcache=True) self.check_listing_response(resp, root_resp_hdrs) + self.assertIn('swift.cache', resp.request.environ) + cached_keys = set(k for k in resp.request.environ['swift.cache'].store + if k.startswith('shard-listing')) + self.assertEqual({'shard-listing-v2/a/c'}, cached_keys) # GET with marker marker = bytes_to_wsgi(sr_objs[3][2]['name'].encode('utf8')) @@ -1319,6 +1341,10 @@ class TestGetShardedContainer(BaseTestContainerController): mock_responses, expected_objects, expected_requests, query_string='?marker=%s' % marker, memcache=True) self.check_listing_response(resp, root_resp_hdrs) + self.assertIn('swift.cache', resp.request.environ) + cached_keys = set(k for k in resp.request.environ['swift.cache'].store + if k.startswith('shard-listing')) + self.assertEqual({'shard-listing-v2/a/c'}, cached_keys) # GET with end marker end_marker = bytes_to_wsgi(sr_objs[3][6]['name'].encode('utf8')) @@ -1385,6 +1411,10 @@ class TestGetShardedContainer(BaseTestContainerController): mock_responses, expected_objects, expected_requests, query_string='?end_marker=%s' % end_marker, memcache=True) self.check_listing_response(resp, root_resp_hdrs) + self.assertIn('swift.cache', resp.request.environ) + cached_keys = set(k for k in resp.request.environ['swift.cache'].store + if k.startswith('shard-listing')) + self.assertEqual({'shard-listing-v2/a/c'}, cached_keys) # GET with prefix prefix = 'hat' @@ -1414,6 +1444,10 @@ class TestGetShardedContainer(BaseTestContainerController): mock_responses, expected_objects, expected_requests, query_string='?prefix=%s' % prefix, memcache=True) self.check_listing_response(resp, root_resp_hdrs) + self.assertIn('swift.cache', resp.request.environ) + cached_keys = set(k for k in resp.request.environ['swift.cache'].store + if k.startswith('shard-listing')) + self.assertEqual({'shard-listing-v2/a/c'}, cached_keys) # marker and end_marker and limit limit = 2 @@ -1463,6 +1497,10 @@ class TestGetShardedContainer(BaseTestContainerController): query_string='?marker=%s&end_marker=%s&limit=%s&reverse=true' % (end_marker, marker, limit), reverse=True, memcache=True) self.check_listing_response(resp, root_resp_hdrs) + self.assertIn('swift.cache', resp.request.environ) + cached_keys = set(k for k in resp.request.environ['swift.cache'].store + if k.startswith('shard-listing')) + self.assertEqual({'shard-listing-v2/a/c'}, cached_keys) def _do_test_GET_sharded_container_with_deleted_shards(self, shard_specs): # verify that if a shard fails to return its listing component then the @@ -1537,7 +1575,7 @@ class TestGetShardedContainer(BaseTestContainerController): ['Aborting listing from shards due to bad response: %s' % ([200, 200, 503],)], errors[-1:]) - def test_GET_sharded_container_with_marker_beyond_end_marker(self): + def test_GET_sharded_container_marker_beyond_end_marker_memcache(self): # verify that if request params result in the filtered namespaces list # being empty the response body still has an empty object list shard_bounds = (('a', 'b'), ('b', 'c'), ('c', '')) @@ -1562,7 +1600,7 @@ class TestGetShardedContainer(BaseTestContainerController): query_string='?marker=bb&end_marker=aa', memcache=True) self.check_listing_response(resp, root_resp_hdrs) - def test_GET_sharded_container_with_delimiter(self): + def test_GET_sharded_container_with_delimiter_no_memcache(self): shard_bounds = (('', 'ha/ppy'), ('ha/ppy', 'ha/ptic'), ('ha/ptic', 'ham'), ('ham', 'pie'), ('pie', '')) namespaces, ns_dicts, _ = self.create_server_response_data( @@ -1620,7 +1658,7 @@ class TestGetShardedContainer(BaseTestContainerController): query_string='?delimiter=/') self.check_listing_response(resp, root_resp_hdrs) - def test_GET_sharded_container_with_delimiter_and_reverse(self): + def test_GET_sharded_container_with_delimiter_reverse_no_memcache(self): shard_bounds = ('', 'ha.d', 'ha/ppy', 'ha/ptic', 'ham', 'pie', '') namespaces, ns_dicts, _ = self.create_server_response_data( shard_bounds) @@ -1687,7 +1725,7 @@ class TestGetShardedContainer(BaseTestContainerController): query_string='?delimiter=/&reverse=on', reverse=True) self.check_listing_response(resp, root_resp_hdrs) - def test_GET_sharded_container_shard_redirects_to_root(self): + def test_GET_sharded_container_shard_redirects_to_root_no_memcache(self): # check that if the root redirects listing to a shard, but the shard # returns the root shard (e.g. it was the final shard to shrink into # the root) objects are requested from the root, rather than a loop. @@ -1715,11 +1753,11 @@ class TestGetShardedContainer(BaseTestContainerController): root_resp_hdrs, root_shard_resp_hdrs = self._make_root_resp_hdrs( num_all_objects, size_all_objects) - root_sr = ShardRange('a/c', Timestamp.now(), '', '') + root_sr_dict = self.create_server_namespace_dict('a/c', '', '') mock_responses = [ # status, body, headers (200, [dict(namespaces[0])], root_shard_resp_hdrs), # from root - (200, [dict(root_sr)], shard_resp_hdrs), # from shard + (200, [root_sr_dict], shard_resp_hdrs), # from shard (200, all_objects, root_resp_hdrs), # from root ] expected_requests = [ @@ -1847,7 +1885,7 @@ class TestGetShardedContainer(BaseTestContainerController): [('a', 'c'), ('.shards_a', 'c_b'), ('.shards_a', 'c_')], resp.request.environ.get('swift.shard_listing_history')) - def test_GET_sharded_container_overlapping_shards(self): + def test_GET_sharded_container_overlapping_shards_no_memcache(self): # verify ordered listing even if unexpected overlapping shard ranges shard_bounds = (('', 'ham'), ('', 'pie'), ('lemon', '')) shard_states = (ShardRange.CLEAVED, ShardRange.ACTIVE, @@ -1986,7 +2024,7 @@ class TestGetShardedContainer(BaseTestContainerController): self.check_listing_response(resp, root_resp_hdrs) self.assertNotIn('swift.cache', resp.request.environ) - def test_GET_sharding_container_gap_in_shards_memcache(self): + def test_GET_sharding_container_gap_in_shards_with_memcache(self): # verify ordered listing even if unexpected gap between shard ranges; # root is sharding so shard ranges are not cached shard_bounds = (('', 'ham'), ('onion', 'pie'), ('rhubarb', '')) @@ -2034,10 +2072,11 @@ class TestGetShardedContainer(BaseTestContainerController): self.check_listing_response(resp, root_resp_hdrs, exp_sharding_state='sharding') self.assertIn('swift.cache', resp.request.environ) - self.assertNotIn('shard-listing-v2/a/c', - resp.request.environ['swift.cache'].store) + cached_keys = set(k for k in resp.request.environ['swift.cache'].store + if k.startswith('shard-listing')) + self.assertFalse(cached_keys) - def test_GET_sharded_container_gap_in_shards_memcache(self): + def test_GET_sharded_container_gap_in_shards_with_memcache(self): # verify ordered listing even if unexpected gap between shard ranges shard_bounds = (('', 'ham'), ('onion', 'pie'), ('rhubarb', '')) namespaces, ns_dicts, sr_objs = self.create_server_response_data( @@ -2084,8 +2123,9 @@ class TestGetShardedContainer(BaseTestContainerController): # root object count will be overridden by actual length of listing self.check_listing_response(resp, root_resp_hdrs) self.assertIn('swift.cache', resp.request.environ) - self.assertIn('shard-listing-v2/a/c', - resp.request.environ['swift.cache'].store) + cached_keys = set(k for k in resp.request.environ['swift.cache'].store + if k.startswith('shard-listing')) + self.assertEqual({'shard-listing-v2/a/c'}, cached_keys) # NB compact bounds in cache do not reveal the gap in shard ranges self.assertEqual( [['', '.shards_a/c_ham'], @@ -2093,7 +2133,7 @@ class TestGetShardedContainer(BaseTestContainerController): ['rhubarb', '.shards_a/c_']], resp.request.environ['swift.cache'].store['shard-listing-v2/a/c']) - def test_GET_sharded_container_empty_shard(self): + def test_GET_sharded_container_empty_shard_no_memcache(self): # verify ordered listing when a shard is empty shard_bounds = (('', 'ham'), ('ham', 'pie'), ('pie', '')) namespaces, ns_dicts, sr_objs = self.create_server_response_data( @@ -2318,28 +2358,33 @@ class TestGetShardedContainer(BaseTestContainerController): mock_responses, all_objects, expected_requests, expected_status=503) - def test_GET_sharded_container_shard_errors(self): + def test_GET_sharded_container_shard_errors_no_memcache(self): self._check_GET_sharded_container_shard_error(404) self._check_GET_sharded_container_shard_error(500) - def test_GET_sharded_container_sharding_shard(self): + def test_GET_sharded_container_sharding_shard_no_memcache(self): # one shard is in process of sharding shard_bounds = (('', 'ham'), ('ham', 'pie'), ('pie', '')) namespaces, ns_dicts, sr_objs = self.create_server_response_data( shard_bounds) - shard_resp_hdrs = self._make_shard_resp_hdrs(sr_objs) - shard_1_shard_resp_hdrs = dict(shard_resp_hdrs[1]) + # headers returned with obj listing from shard containers... + shard_obj_resp_hdrs = self._make_shard_resp_hdrs(sr_objs) + # modify second shard's obj listing resp - this one is sharding... + shard_obj_resp_hdrs[1]['X-Backend-Sharding-State'] = 'sharding' + # ...and will return shards in 'response' to auto record-type... + shard_1_shard_resp_hdrs = dict(shard_obj_resp_hdrs[1]) shard_1_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard' shard_1_shard_resp_hdrs.update(self.RESP_SHARD_FORMAT_HEADERS) # second shard is sharding and has cleaved two out of three sub shards - shard_resp_hdrs[1]['X-Backend-Sharding-State'] = 'sharding' sub_shard_bounds = (('ham', 'juice'), ('juice', 'lemon')) - sub_shard_ranges = [ - ShardRange('a/c_sub_' + upper, Timestamp.now(), lower, upper) - for lower, upper in sub_shard_bounds] - sub_sr_dicts = [dict(sr) for sr in sub_shard_ranges] - sub_sr_objs = [self._make_shard_objects(sr) for sr in sub_shard_ranges] + sub_namespaces, sub_ns_dicts, sub_sr_objs = \ + self.create_server_response_data(sub_shard_bounds, + name_prefix='a/c_sub_') + filler_sr_dict = self.create_server_namespace_dict( + namespaces[1].name, lower=sub_ns_dicts[-1]['upper'], + upper=namespaces[1].upper) + sub_sr_objs = [self._make_shard_objects(sr) for sr in sub_namespaces] sub_shard_resp_hdrs = [ {'X-Backend-Sharding-State': 'unsharded', 'X-Container-Object-Count': len(sub_sr_objs[i]), @@ -2361,6 +2406,7 @@ class TestGetShardedContainer(BaseTestContainerController): 'X-Container-Bytes-Used': size_all_objects, 'X-Container-Meta-Flavour': 'peach', 'X-Backend-Storage-Policy-Index': 0} + # headers returned with root response to auto record-type listing... root_shard_resp_hdrs = dict(root_resp_hdrs) root_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard' root_shard_resp_hdrs.update(self.RESP_SHARD_FORMAT_HEADERS) @@ -2368,13 +2414,13 @@ class TestGetShardedContainer(BaseTestContainerController): mock_responses = [ # status, body, headers (200, ns_dicts, root_shard_resp_hdrs), - (200, sr_objs[0], shard_resp_hdrs[0]), - (200, sub_sr_dicts + [ns_dicts[1]], shard_1_shard_resp_hdrs), + (200, sr_objs[0], shard_obj_resp_hdrs[0]), + (200, sub_ns_dicts + [filler_sr_dict], shard_1_shard_resp_hdrs), (200, sub_sr_objs[0], sub_shard_resp_hdrs[0]), (200, sub_sr_objs[1], sub_shard_resp_hdrs[1]), (200, sr_objs[1][len(sub_sr_objs[0] + sub_sr_objs[1]):], - shard_resp_hdrs[1]), - (200, sr_objs[2], shard_resp_hdrs[2]) + shard_obj_resp_hdrs[1]), + (200, sr_objs[2], shard_obj_resp_hdrs[2]) ] # NB marker always advances to last object name expected_requests = [ @@ -2394,13 +2440,13 @@ class TestGetShardedContainer(BaseTestContainerController): dict(marker='h', end_marker='pie\x00', states='listing', limit=str(limit - len(sr_objs[0])))), # get first sub-shard objects - (sub_shard_ranges[0].name, + (sub_namespaces[0].name, {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='h', end_marker='juice\x00', states='listing', limit=str(limit - len(sr_objs[0])))), # get second sub-shard objects - (sub_shard_ranges[1].name, + (sub_namespaces[1].name, {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='j', end_marker='lemon\x00', states='listing', @@ -2427,6 +2473,247 @@ class TestGetShardedContainer(BaseTestContainerController): # root object count will overridden by actual length of listing self.check_listing_response(resp, root_resp_hdrs) + def test_GET_sharded_container_sharding_shard_with_memcache(self): + # one shard is in process of sharding + shard_bounds = (('', 'ham'), ('ham', 'pie'), ('pie', '')) + namespaces, ns_dicts, sr_objs = self.create_server_response_data( + shard_bounds) + # headers returned with obj listing from shard containers... + shard_obj_resp_hdrs = self._make_shard_resp_hdrs(sr_objs) + # modify second shard's obj listing resp - this one is sharding... + shard_obj_resp_hdrs[1]['X-Backend-Sharding-State'] = 'sharding' + # ...and will return shards in 'response' to auto record-type... + shard_1_shard_resp_hdrs = dict(shard_obj_resp_hdrs[1]) + shard_1_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard' + shard_1_shard_resp_hdrs.update(self.RESP_SHARD_FORMAT_HEADERS) + + # second shard is sharding and has cleaved two out of three sub shards + sub_shard_bounds = (('ham', 'juice'), ('juice', 'lemon')) + sub_namespaces, sub_ns_dicts, sub_sr_objs = \ + self.create_server_response_data(sub_shard_bounds, + name_prefix='a/c_sub_') + filler_sr_dict = self.create_server_namespace_dict( + namespaces[1].name, lower=sub_ns_dicts[-1]['upper'], + upper=namespaces[1].upper) + sub_shard_resp_hdrs = [ + {'X-Backend-Sharding-State': 'unsharded', + 'X-Container-Object-Count': len(sub_sr_objs[i]), + 'X-Container-Bytes-Used': + sum([obj['bytes'] for obj in sub_sr_objs[i]]), + 'X-Container-Meta-Flavour': 'flavour%d' % i, + 'X-Backend-Storage-Policy-Index': 0} + for i in range(2)] + + all_objects = [] + for objects in sr_objs: + all_objects.extend(objects) + size_all_objects = sum([obj['bytes'] for obj in all_objects]) + num_all_objects = len(all_objects) + limit = CONTAINER_LISTING_LIMIT + root_resp_hdrs = {'X-Backend-Sharding-State': 'sharded', + 'X-Backend-Timestamp': '99', + 'X-Container-Object-Count': num_all_objects, + 'X-Container-Bytes-Used': size_all_objects, + 'X-Container-Meta-Flavour': 'peach', + 'X-Backend-Storage-Policy-Index': 0} + # headers returned with root response to auto record-type listing... + root_shard_resp_hdrs = dict(root_resp_hdrs) + root_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard' + root_shard_resp_hdrs['X-Backend-Override-Shard-Name-Filter'] = 'true' + root_shard_resp_hdrs.update(self.RESP_SHARD_FORMAT_HEADERS) + + mock_responses = [ + # status, body, headers + (200, ns_dicts, root_shard_resp_hdrs), + (200, sr_objs[0], shard_obj_resp_hdrs[0]), + (200, sub_ns_dicts + [filler_sr_dict], shard_1_shard_resp_hdrs), + (200, sub_sr_objs[0], sub_shard_resp_hdrs[0]), + (200, sub_sr_objs[1], sub_shard_resp_hdrs[1]), + (200, sr_objs[1][len(sub_sr_objs[0] + sub_sr_objs[1]):], + shard_obj_resp_hdrs[1]), + (200, sr_objs[2], shard_obj_resp_hdrs[2]) + ] + # NB marker always advances to last object name + expected_requests = [ + # get root shard ranges + ('a/c', {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded'}, + dict(states='listing')), # 200 + # get first shard objects + (namespaces[0].name, + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', + 'X-Backend-Storage-Policy-Index': '0'}, + dict(marker='', end_marker='ham\x00', states='listing', + limit=str(limit))), # 200 + # get second shard sub-shard ranges + (namespaces[1].name, + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', + 'X-Backend-Storage-Policy-Index': '0'}, + dict(marker='h', end_marker='pie\x00', states='listing', + limit=str(limit - len(sr_objs[0])))), + # get first sub-shard objects + (sub_namespaces[0].name, + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', + 'X-Backend-Storage-Policy-Index': '0'}, + dict(marker='h', end_marker='juice\x00', states='listing', + limit=str(limit - len(sr_objs[0])))), + # get second sub-shard objects + (sub_namespaces[1].name, + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', + 'X-Backend-Storage-Policy-Index': '0'}, + dict(marker='j', end_marker='lemon\x00', states='listing', + limit=str(limit - len(sr_objs[0] + sub_sr_objs[0])))), + # get remainder of first shard objects (filler shard range) + (namespaces[1].name, + {'X-Backend-Record-Type': 'object', + 'X-Backend-Storage-Policy-Index': '0'}, + dict(marker='l', end_marker='pie\x00', + limit=str(limit - len(sr_objs[0] + sub_sr_objs[0] + + sub_sr_objs[1])))), # 200 + # get third shard objects + (namespaces[2].name, + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', + 'X-Backend-Storage-Policy-Index': '0'}, + dict(marker='p', end_marker='', states='listing', + limit=str(limit - len(sr_objs[0] + sr_objs[1])))) # 200 + ] + expected_objects = ( + sr_objs[0] + sub_sr_objs[0] + sub_sr_objs[1] + + sr_objs[1][len(sub_sr_objs[0] + sub_sr_objs[1]):] + sr_objs[2]) + resp = self._check_GET_shard_listing( + mock_responses, expected_objects, expected_requests, memcache=True) + # root object count will overridden by actual length of listing + self.check_listing_response(resp, root_resp_hdrs) + self.assertIn('swift.cache', resp.request.environ) + # sub-shards are not cached because the shard is still 'sharding'... + cached_keys = set(k for k in resp.request.environ['swift.cache'].store + if k.startswith('shard-listing')) + self.assertEqual({'shard-listing-v2/a/c'}, cached_keys) + + def test_GET_sharded_container_sharded_shard_with_memcache(self): + # one shard is sharded but still in shard listing returned by root + shard_bounds = (('', 'ham'), ('ham', 'pie'), ('pie', '')) + namespaces, ns_dicts, sr_objs = self.create_server_response_data( + shard_bounds) + shard_resp_hdrs = self._make_shard_resp_hdrs(sr_objs) + shard_1_shard_resp_hdrs = dict(shard_resp_hdrs[1]) + shard_1_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard' + shard_1_shard_resp_hdrs['X-Backend-Sharding-State'] = 'sharded' + shard_1_shard_resp_hdrs[ + 'X-Backend-Override-Shard-Name-Filter'] = 'true' + shard_1_shard_resp_hdrs.update(self.RESP_SHARD_FORMAT_HEADERS) + + # second shard is sharded and has cleaved three sub shards + sub_shard_bounds = (('ham', 'juice'), ('juice', 'lemon'), + ('lemon', 'pie')) + sub_namespaces, sub_ns_dicts, sub_sr_objs = \ + self.create_server_response_data(sub_shard_bounds, + name_prefix='a/c_sub_') + sub_shard_resp_hdrs = [ + {'X-Backend-Sharding-State': 'unsharded', + 'X-Container-Object-Count': len(sub_sr_objs[i]), + 'X-Container-Bytes-Used': + sum([obj['bytes'] for obj in sub_sr_objs[i]]), + 'X-Container-Meta-Flavour': 'flavour%d' % i, + 'X-Backend-Storage-Policy-Index': 0} + for i in range(3)] + + all_objects = [] + for objects in sr_objs: + all_objects.extend(objects) + size_all_objects = sum([obj['bytes'] for obj in all_objects]) + num_all_objects = len(all_objects) + limit = CONTAINER_LISTING_LIMIT + root_resp_hdrs = {'X-Backend-Sharding-State': 'sharded', + 'X-Backend-Timestamp': '99', + 'X-Container-Object-Count': num_all_objects, + 'X-Container-Bytes-Used': size_all_objects, + 'X-Container-Meta-Flavour': 'peach', + 'X-Backend-Storage-Policy-Index': 0} + root_shard_resp_hdrs = dict(root_resp_hdrs) + root_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard' + root_shard_resp_hdrs['X-Backend-Override-Shard-Name-Filter'] = 'true' + root_shard_resp_hdrs.update(self.RESP_SHARD_FORMAT_HEADERS) + + mock_responses = [ + # status, body, headers + (200, ns_dicts, root_shard_resp_hdrs), + (200, sr_objs[0], shard_resp_hdrs[0]), + (200, sub_ns_dicts, shard_1_shard_resp_hdrs), + (200, sub_sr_objs[0], sub_shard_resp_hdrs[0]), + (200, sub_sr_objs[1], sub_shard_resp_hdrs[1]), + (200, sub_sr_objs[2], sub_shard_resp_hdrs[2]), + (200, sr_objs[2], shard_resp_hdrs[2]) + ] + # NB marker always advances to last object name + expected_requests = [ + # get root shard ranges + ('a/c', {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded'}, + dict(states='listing')), # 200 + # get first shard objects + (namespaces[0].name, + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', + 'X-Backend-Storage-Policy-Index': '0'}, + dict(marker='', end_marker='ham\x00', states='listing', + limit=str(limit))), # 200 + # get second shard sub-shard ranges + (namespaces[1].name, + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', + 'X-Backend-Storage-Policy-Index': '0'}, + dict(marker='h', end_marker='pie\x00', states='listing', + limit=str(limit - len(sr_objs[0])))), + # get first sub-shard objects + (sub_namespaces[0].name, + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', + 'X-Backend-Storage-Policy-Index': '0'}, + dict(marker='h', end_marker='juice\x00', states='listing', + limit=str(limit - len(sr_objs[0])))), + # get second sub-shard objects + (sub_namespaces[1].name, + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', + 'X-Backend-Storage-Policy-Index': '0'}, + dict(marker='j', end_marker='lemon\x00', states='listing', + limit=str(limit - len(sr_objs[0] + sub_sr_objs[0])))), + # get third sub-shard objects + (sub_namespaces[2].name, + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', + 'X-Backend-Storage-Policy-Index': '0'}, + dict(marker='l', end_marker='pie\x00', states='listing', + limit=str(limit - len(sr_objs[0] + sub_sr_objs[0] + + sub_sr_objs[1])))), + # get third shard objects + (namespaces[2].name, + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', + 'X-Backend-Storage-Policy-Index': '0'}, + dict(marker='p', end_marker='', states='listing', + limit=str(limit - len(sr_objs[0] + sr_objs[1])))) # 200 + ] + expected_objects = ( + sr_objs[0] + sub_sr_objs[0] + sub_sr_objs[1] + sub_sr_objs[2] + + sr_objs[2]) + resp = self._check_GET_shard_listing( + mock_responses, expected_objects, expected_requests, memcache=True) + # root object count will overridden by actual length of listing + self.check_listing_response(resp, root_resp_hdrs) + self.assertIn('swift.cache', resp.request.environ) + cached_keys = set(k for k in resp.request.environ['swift.cache'].store + if k.startswith('shard-listing')) + self.assertEqual( + {'shard-listing-v2/a/c', 'shard-listing-v2/.shards_a/c_pie'}, + cached_keys) + @patch_policies([ StoragePolicy(0, 'zero', True, object_ring=FakeRing()), StoragePolicy(1, 'one', False, object_ring=FakeRing()) @@ -2437,22 +2724,23 @@ class TestGetShardedContainer(BaseTestContainerController): shard_bounds = (('', 'ham'), ('ham', 'pie'), ('pie', '')) namespaces, ns_dicts, sr_objs = self.create_server_response_data( shard_bounds) - shard_resp_hdrs = self._make_shard_resp_hdrs( + shard_obj_resp_hdrs = self._make_shard_resp_hdrs( sr_objs, extra_hdrs={ 'X-Backend-Storage-Policy-Index': 1, 'X-Backend-Record-Storage-Policy-Index': 0}) - shard_1_shard_resp_hdrs = dict(shard_resp_hdrs[1]) + # second shard is sharding and has cleaved two out of three sub shards + shard_obj_resp_hdrs[1]['X-Backend-Sharding-State'] = 'sharding' + shard_1_shard_resp_hdrs = dict(shard_obj_resp_hdrs[1]) shard_1_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard' shard_1_shard_resp_hdrs.update(self.RESP_SHARD_FORMAT_HEADERS) - # second shard is sharding and has cleaved two out of three sub shards - shard_resp_hdrs[1]['X-Backend-Sharding-State'] = 'sharding' sub_shard_bounds = (('ham', 'juice'), ('juice', 'lemon')) - sub_shard_ranges = [ - ShardRange('a/c_sub_' + upper, Timestamp.now(), lower, upper) - for lower, upper in sub_shard_bounds] - sub_sr_dicts = [dict(sr) for sr in sub_shard_ranges] - sub_sr_objs = [self._make_shard_objects(sr) for sr in sub_shard_ranges] + sub_namespaces, sub_ns_dicts, sub_sr_objs = \ + self.create_server_response_data(sub_shard_bounds, + name_prefix='a/c_sub_') + filler_sr_dict = self.create_server_namespace_dict( + namespaces[1].name, lower=sub_ns_dicts[-1]['upper'], + upper=namespaces[1].upper) sub_shard_resp_hdrs = self._make_shard_resp_hdrs( sub_sr_objs, extra_hdrs={ 'X-Backend-Storage-Policy-Index': 1, @@ -2477,13 +2765,13 @@ class TestGetShardedContainer(BaseTestContainerController): mock_responses = [ # status, body, headers (200, ns_dicts, root_shard_resp_hdrs), - (200, sr_objs[0], shard_resp_hdrs[0]), - (200, sub_sr_dicts + [ns_dicts[1]], shard_1_shard_resp_hdrs), + (200, sr_objs[0], shard_obj_resp_hdrs[0]), + (200, sub_ns_dicts + [filler_sr_dict], shard_1_shard_resp_hdrs), (200, sub_sr_objs[0], sub_shard_resp_hdrs[0]), (200, sub_sr_objs[1], sub_shard_resp_hdrs[1]), (200, sr_objs[1][len(sub_sr_objs[0] + sub_sr_objs[1]):], - shard_resp_hdrs[1]), - (200, sr_objs[2], shard_resp_hdrs[2]) + shard_obj_resp_hdrs[1]), + (200, sr_objs[2], shard_obj_resp_hdrs[2]) ] # NB marker always advances to last object name expected_requests = [ @@ -2503,13 +2791,13 @@ class TestGetShardedContainer(BaseTestContainerController): dict(marker='h', end_marker='pie\x00', states='listing', limit=str(limit - len(sr_objs[0])))), # get first sub-shard objects - (sub_shard_ranges[0].name, + (sub_namespaces[0].name, {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='h', end_marker='juice\x00', states='listing', limit=str(limit - len(sr_objs[0])))), # get second sub-shard objects - (sub_shard_ranges[1].name, + (sub_namespaces[1].name, {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='j', end_marker='lemon\x00', states='listing', @@ -2594,6 +2882,95 @@ class TestGetShardedContainer(BaseTestContainerController): do_test(0) do_test(None) + def test_GET_record_type_shard(self): + # explicit request for namespaces + memcache = FakeMemcache() + shard_bounds = ('', 'pie') + namespaces, ns_dicts, sr_objs = self.create_server_response_data( + shard_bounds) + _, root_shard_resp_hdrs = self._make_root_resp_hdrs(2, 4) + + body = json.dumps(ns_dicts).encode('ascii') + req = Request.blank('/v1/a/c', {'swift.cache': memcache}) + req.headers['X-Backend-Record-Type'] = 'shard' + req.headers['X-Backend-Record-Shard-Format'] = 'namespace' + with mocked_http_conn(200, body_iter=[body], + headers=root_shard_resp_hdrs) as fake_conn: + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 200) + self.assertEqual(1, len(fake_conn.requests)) + exp_backend_hdrs = { + 'X-Backend-Record-Type': 'shard', + 'X-Backend-Record-Shard-Format': 'namespace', + 'Host': mock.ANY, 'X-Trans-Id': mock.ANY, 'X-Timestamp': mock.ANY, + 'Connection': 'close', 'User-Agent': mock.ANY, + 'Referer': mock.ANY} + self.assertEqual(exp_backend_hdrs, fake_conn.requests[0]['headers']) + self.assertNotIn('state=', fake_conn.requests[0]['qs']) + # NB: no namespaces cached + self.assertEqual([mock.call.set('container/a/c', mock.ANY, time=60)], + memcache.calls) + self.assertEqual(ns_dicts, json.loads(resp.body)) + + def test_GET_record_type_shard_with_listing_state(self): + # explicit request for namespaces specifying list state + memcache = FakeMemcache() + shard_bounds = ('', 'pie') + namespaces, ns_dicts, sr_objs = self.create_server_response_data( + shard_bounds) + _, root_shard_resp_hdrs = self._make_root_resp_hdrs(2, 4) + + body = json.dumps(ns_dicts).encode('ascii') + req = Request.blank('/v1/a/c?state=listing', {'swift.cache': memcache}) + req.headers['X-Backend-Record-Type'] = 'shard' + req.headers['X-Backend-Record-Shard-Format'] = 'namespace' + with mocked_http_conn(200, body_iter=[body], + headers=root_shard_resp_hdrs) as fake_conn: + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 200) + self.assertEqual(1, len(fake_conn.requests)) + exp_backend_hdrs = { + 'X-Backend-Record-Type': 'shard', + 'X-Backend-Record-Shard-Format': 'namespace', + 'Host': mock.ANY, 'X-Trans-Id': mock.ANY, 'X-Timestamp': mock.ANY, + 'Connection': 'close', 'User-Agent': mock.ANY, + 'Referer': mock.ANY} + self.assertEqual(exp_backend_hdrs, fake_conn.requests[0]['headers']) + self.assertIn('state=listing', fake_conn.requests[0]['qs']) + # NB: no namespaces cached + self.assertEqual([mock.call.set('container/a/c', mock.ANY, time=60)], + memcache.calls) + self.assertEqual(ns_dicts, json.loads(resp.body)) + + def test_GET_record_type_object(self): + # explicit request for objects + memcache = FakeMemcache() + shard_bounds = ('', 'pie') + namespaces, ns_dicts, sr_objs = self.create_server_response_data( + shard_bounds) + all_objs = sum(sr_objs, []) + root_resp_hdrs, _ = self._make_root_resp_hdrs( + len(all_objs), 4, extra_hdrs={'X-Backend-Record-Type': 'object'}) + body = json.dumps(sr_objs[0]).encode('ascii') + req = Request.blank('/v1/a/c', {'swift.cache': memcache}) + req.headers['X-Backend-Record-Type'] = 'object' + with mocked_http_conn(200, body_iter=[body], + headers=root_resp_hdrs) as fake_conn: + resp = req.get_response(self.app) + + self.assertEqual(resp.status_int, 200) + self.assertEqual(1, len(fake_conn.requests)) + exp_backend_hdrs = { + 'X-Backend-Record-Type': 'object', + 'Host': mock.ANY, 'X-Trans-Id': mock.ANY, 'X-Timestamp': mock.ANY, + 'Connection': 'close', 'User-Agent': mock.ANY, + 'Referer': 'GET http://localhost/v1/a/c?format=json'} + self.assertEqual(exp_backend_hdrs, fake_conn.requests[0]['headers']) + self.assertEqual([mock.call.set('container/a/c', mock.ANY, time=60)], + memcache.calls) + self.assertEqual(all_objs, json.loads(resp.body)) + self.assertEqual('object', resp.headers.get('X-Backend-Record-Type')) + @patch_policies([StoragePolicy(0, 'zero', True, object_ring=FakeRing())]) class TestGetShardedContainerLegacy(TestGetShardedContainer): @@ -2606,13 +2983,20 @@ class TestGetShardedContainerLegacy(TestGetShardedContainer): # old container servers did not return this header RESP_SHARD_FORMAT_HEADERS = {} - def create_server_response_data(self, bounds, states=None): + def create_server_namespace_dict(self, name, lower, upper): + # return a dict representation of an instance of the type the backend + # server returns for shard format = 'namespace' + return dict(ShardRange(name, Timestamp.now(), lower, upper, + state=ShardRange.ACTIVE)) + + def create_server_response_data(self, bounds, states=None, + name_prefix='.shards_a/c_'): if not isinstance(bounds[0], (list, tuple)): bounds = [(l, u) for l, u in zip(bounds[:-1], bounds[1:])] if not states: states = [] shard_ranges = [ - ShardRange('.shards_a/c_%s' % bound[1].replace('/', '-'), + ShardRange(name_prefix + bound[1].replace('/', '-'), Timestamp.now(), bound[0], bound[1], state=state) for bound, state in zip_longest( bounds, states, fillvalue=ShardRange.FOUND)] @@ -2622,34 +3006,31 @@ class TestGetShardedContainerLegacy(TestGetShardedContainer): return shard_ranges, sr_dicts, sr_objs -class TestGetPathNamespaceCaching(BaseTestContainerController): - # These tests are verifying the content and caching of the backend - # namespace responses so we're not interested in gathering objects from the - # shards. We therefore mock _get_from_shards so that the response actually - # contains a fake listing and also capture the namespace listing passed to - # _get_from_shards. This avoids faking all the object listing responses - # from shards, and facilitates making assertions about the namespaces - # passed to _get_from_shards. - bogus_listing = [{'name': 'x'}, {'name': 'y'}] - bogus_listing_body = json.dumps(bogus_listing).encode('ascii') - +class BaseTestContainerControllerGetPath(BaseTestContainerController): def setUp(self): - super(TestGetPathNamespaceCaching, self).setUp() - self._setup_shard_range_stubs() - self.get_from_shards_lists = [] - - def _fake_get_from_shards(self, req, resp, namespaces): - self.get_from_shards_lists.append(namespaces) - resp.body = self.bogus_listing_body - return resp + super(BaseTestContainerControllerGetPath, self).setUp() + self.memcache = FakeMemcache() + shard_bounds = (('', 'ham'), ('ham', 'pie'), ('pie', '')) + self.ns_dicts = [{'name': '.shards_a/c_%s' % upper, + 'lower': lower, + 'upper': upper} + for lower, upper in shard_bounds] + self.root_resp_hdrs = { + 'Accept-Ranges': 'bytes', + 'Content-Type': 'application/json', + 'Last-Modified': 'Thu, 01 Jan 1970 00:00:03 GMT', + 'X-Backend-Timestamp': '2', + 'X-Backend-Put-Timestamp': '3', + 'X-Backend-Delete-Timestamp': '0', + 'X-Backend-Status-Changed-At': '0', + 'X-Timestamp': '2', + 'X-Put-Timestamp': '3', + 'X-Container-Object-Count': '6', + 'X-Container-Bytes-Used': '12', + 'X-Backend-Storage-Policy-Index': '0'} def _call_app(self, req): - self.get_from_shards_lists = [] - with mock.patch( - 'swift.proxy.controllers.container.' - 'ContainerController._get_from_shards', - side_effect=self._fake_get_from_shards): - return req.get_response(self.app) + return req.get_response(self.app) def _build_request(self, headers, params, infocache=None): # helper to make a GET request with caches set in environ @@ -2734,66 +3115,80 @@ class TestGetPathNamespaceCaching(BaseTestContainerController): self.assertIn(k, backend_hdrs) self.assertEqual(v, backend_hdrs.get(k)) - def _setup_shard_range_stubs(self): - self.memcache = FakeMemcache() - shard_bounds = (('', 'ham'), ('ham', 'pie'), ('pie', '')) - self.ns_dicts = [{'name': '.shards_a/c_%s' % upper, - 'lower': lower, - 'upper': upper} - for lower, upper in shard_bounds] + +class TestGetPathNamespaceCaching(BaseTestContainerControllerGetPath): + # These tests are verifying the content and caching of the backend + # namespace responses so we're not interested in gathering objects from the + # shards. We therefore mock _get_from_shards so that the response actually + # contains a fake listing and also capture the namespace listing passed to + # _get_from_shards. This avoids faking all the object listing responses + # from shards, and facilitates making assertions about the namespaces + # passed to _get_from_shards. + RESP_SHARD_FORMAT_HEADERS = {'X-Backend-Record-Shard-Format': 'namespace'} + bogus_listing = [{'name': 'x'}, {'name': 'y'}] + bogus_listing_body = json.dumps(bogus_listing).encode('ascii') + + def setUp(self): + super(TestGetPathNamespaceCaching, self).setUp() self.namespaces = [Namespace(**ns) for ns in self.ns_dicts] self.ns_bound_list = NamespaceBoundList.parse(self.namespaces) - self.sr_dicts = [dict(Namespace(**ns)) for ns in self.ns_dicts] - self._stub_shards_dump = json.dumps(self.sr_dicts).encode('ascii') - self.root_resp_hdrs = { - 'Accept-Ranges': 'bytes', - 'Content-Type': 'application/json', - 'Last-Modified': 'Thu, 01 Jan 1970 00:00:03 GMT', - 'X-Backend-Timestamp': '2', - 'X-Backend-Put-Timestamp': '3', - 'X-Backend-Delete-Timestamp': '0', - 'X-Backend-Status-Changed-At': '0', - 'X-Timestamp': '2', - 'X-Put-Timestamp': '3', - 'X-Container-Object-Count': '6', - 'X-Container-Bytes-Used': '12', - 'X-Backend-Storage-Policy-Index': '0'} + self._setup_namespace_stubs() + self.get_from_shards_lists = [] - def _do_test_caching(self, record_type, exp_recheck_listing, - exp_record_type, exp_listing, - extra_backend_req_hdrs=None): + def _fake_get_from_shards(self, req, resp, namespaces): + self.get_from_shards_lists.append(namespaces) + resp.body = self.bogus_listing_body + return resp + + def _call_app(self, req): + # override base class method to mock get_from_shards + self.get_from_shards_lists = [] + with mock.patch( + 'swift.proxy.controllers.container.' + 'ContainerController._get_from_shards', + side_effect=self._fake_get_from_shards): + return req.get_response(self.app) + + def _setup_namespace_stubs(self): + self._stub_namespaces = self.ns_dicts + self._stub_namespaces_dump = json.dumps( + self._stub_namespaces).encode('ascii') + + def _do_test_GET_namespace_caching(self, record_type, exp_recheck_listing, + extra_backend_req_hdrs=None): # this test gets shard ranges into cache and then reads from cache exp_backend_req_hdrs = { - 'X-Backend-Record-Type': record_type, + 'X-Backend-Record-Type': 'auto', + 'X-Backend-Record-Shard-Format': 'namespace', + 'X-Backend-Include-Deleted': 'false', 'X-Backend-Override-Shard-Name-Filter': 'sharded'} if extra_backend_req_hdrs: exp_backend_req_hdrs.update(extra_backend_req_hdrs) sharding_state = 'sharded' - exp_resp_headers = { + exp_noncache_resp_hdrs = { 'X-Backend-Recheck-Container-Existence': '60', - 'X-Backend-Sharding-State': sharding_state} - if exp_record_type: - exp_resp_headers['X-Backend-Record-Type'] = exp_record_type - exp_cache_resp_headers = { + 'X-Backend-Sharding-State': 'sharded'} + exp_cache_resp_hdrs = { 'X-Backend-Cached-Results': 'true', 'X-Backend-Sharding-State': sharding_state} - if exp_record_type: - exp_cache_resp_headers['X-Backend-Record-Type'] = exp_record_type self.memcache.delete_all() # container is sharded but proxy does not have that state cached; - # expect a backend request and expect shard ranges to be cached + # expect a backend request and expect namespaces to be cached self.memcache.clear_calls() self.logger.clear() req = self._build_request({'X-Backend-Record-Type': record_type}, {'states': 'listing'}, {}) + backend_resp_hdrs = {'X-Backend-Record-Type': 'shard', + 'X-Backend-Sharding-State': 'sharded', + 'X-Backend-Override-Shard-Name-Filter': 'true'} + backend_resp_hdrs.update(self.RESP_SHARD_FORMAT_HEADERS) + backend_req, resp = self._capture_backend_request( - req, 200, self._stub_shards_dump, - {'X-Backend-Record-Type': 'shard', - 'X-Backend-Sharding-State': sharding_state, - 'X-Backend-Override-Shard-Name-Filter': 'true'}) + req, 200, self._stub_namespaces_dump, backend_resp_hdrs) + self._check_backend_req( req, backend_req, extra_hdrs=exp_backend_req_hdrs) - self._check_response(resp, exp_listing, exp_resp_headers) + self._check_response(resp, self.bogus_listing, exp_noncache_resp_hdrs) cache_key = 'shard-listing-v2/a/c' self.assertEqual( @@ -2802,7 +3197,7 @@ class TestGetPathNamespaceCaching(BaseTestContainerController): time=exp_recheck_listing, raise_on_error=True), mock.call.set('container/a/c', mock.ANY, time=60)], self.memcache.calls) - self.assertEqual(sharding_state, + self.assertEqual('sharded', self.memcache.calls[2][1][1]['sharding_state']) self.assertIn('swift.infocache', req.environ) self.assertIn(cache_key, req.environ['swift.infocache']) @@ -2822,13 +3217,10 @@ class TestGetPathNamespaceCaching(BaseTestContainerController): req = self._build_request({'X-Backend-Record-Type': record_type}, {'states': 'listing'}, {}) backend_req, resp = self._capture_backend_request( - req, 200, self._stub_shards_dump, - {'X-Backend-Record-Type': 'shard', - 'X-Backend-Sharding-State': sharding_state, - 'X-Backend-Override-Shard-Name-Filter': 'true'}) + req, 200, self._stub_namespaces_dump, backend_resp_hdrs) self._check_backend_req( req, backend_req, extra_hdrs=exp_backend_req_hdrs) - self._check_response(resp, exp_listing, exp_resp_headers) + self._check_response(resp, self.bogus_listing, exp_noncache_resp_hdrs) self.assertEqual( [mock.call.get('container/a/c'), mock.call.get(cache_key, raise_on_error=True), @@ -2854,8 +3246,8 @@ class TestGetPathNamespaceCaching(BaseTestContainerController): self.logger.clear() req = self._build_request({'X-Backend-Record-Type': record_type}, {'states': 'listing'}, {}) - resp = req.get_response(self.app) - self._check_response(resp, exp_listing, exp_cache_resp_headers) + resp = self._call_app(req) + self._check_response(resp, self.bogus_listing, exp_cache_resp_hdrs) self.assertEqual( [mock.call.get('container/a/c'), mock.call.get(cache_key, raise_on_error=True)], @@ -2878,13 +3270,10 @@ class TestGetPathNamespaceCaching(BaseTestContainerController): {'states': 'listing'}, {}) with mock.patch('random.random', return_value=0.05): backend_req, resp = self._capture_backend_request( - req, 200, self._stub_shards_dump, - {'X-Backend-Record-Type': 'shard', - 'X-Backend-Sharding-State': sharding_state, - 'X-Backend-Override-Shard-Name-Filter': 'true'}) + req, 200, self._stub_namespaces_dump, backend_resp_hdrs) self._check_backend_req( req, backend_req, extra_hdrs=exp_backend_req_hdrs) - self._check_response(resp, exp_listing, exp_resp_headers) + self._check_response(resp, self.bogus_listing, exp_noncache_resp_hdrs) self.assertEqual( [mock.call.get('container/a/c'), mock.call.set(cache_key, self.ns_bound_list.bounds, @@ -2909,8 +3298,8 @@ class TestGetPathNamespaceCaching(BaseTestContainerController): req = self._build_request({'X-Backend-Record-Type': record_type}, {'states': 'listing'}, {}) with mock.patch('random.random', return_value=0.11): - resp = req.get_response(self.app) - self._check_response(resp, exp_listing, exp_cache_resp_headers) + resp = self._call_app(req) + self._check_response(resp, self.bogus_listing, exp_cache_resp_hdrs) self.assertEqual( [mock.call.get('container/a/c'), mock.call.get(cache_key, raise_on_error=True)], @@ -2933,8 +3322,8 @@ class TestGetPathNamespaceCaching(BaseTestContainerController): {'states': 'listing'}, infocache=req.environ['swift.infocache']) with mock.patch('random.random', return_value=0.11): - resp = req.get_response(self.app) - self._check_response(resp, exp_listing, exp_cache_resp_headers) + resp = self._call_app(req) + self._check_response(resp, self.bogus_listing, exp_cache_resp_hdrs) self.assertEqual([], self.memcache.calls) self.assertIn('swift.infocache', req.environ) self.assertIn(cache_key, req.environ['swift.infocache']) @@ -2949,7 +3338,7 @@ class TestGetPathNamespaceCaching(BaseTestContainerController): # put this back the way we found it for later subtests self.app.container_listing_shard_ranges_skip_cache = 0.0 - # delete the container; check that shard ranges are evicted from cache + # delete the container; check that namespaces are evicted from cache self.memcache.clear_calls() infocache = {} req = Request.blank('/v1/a/c', method='DELETE') @@ -2962,8 +3351,21 @@ class TestGetPathNamespaceCaching(BaseTestContainerController): mock.call.delete(cache_key)], self.memcache.calls) + def test_GET_namespace_caching(self): + # no record type defaults to 'auto' in backend requests; + # expect shard ranges cache time to be default value of 600 + self._do_test_GET_namespace_caching('', 600) + # expect shard ranges cache time to be configured value of 120 + self.app.recheck_listing_shard_ranges = 120 + self._do_test_GET_namespace_caching('', 120) + # explicitly requesting record type 'auto' + self._do_test_GET_namespace_caching('auto', 120) + # nonsense record type defaults to 'auto' + self._do_test_GET_namespace_caching('banana', 120) + def test_get_from_shards_add_root_spi(self): - shard_resp = mock.MagicMock(status_int=204, headers={}) + shard_resp = mock.MagicMock(status_int=204, + headers={'x-backend-record-type': 'shard'}) def mock_get_container_listing(self_, req, *args, **kargs): captured_hdrs.update(req.headers) @@ -2972,8 +3374,12 @@ class TestGetPathNamespaceCaching(BaseTestContainerController): # header in response -> header added to request captured_hdrs = {} req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'GET'}) + resp_hdrs = dict(self.root_resp_hdrs) + resp_hdrs['x-backend-record-type'] = 'shard' + resp_hdrs.update(self.RESP_SHARD_FORMAT_HEADERS) + resp = mock.MagicMock(status_int=200, - headers=self.root_resp_hdrs, + headers=resp_hdrs, request=req) resp.headers['X-Backend-Storage-Policy-Index'] = '0' with mock.patch('swift.proxy.controllers.container.' @@ -2990,7 +3396,7 @@ class TestGetPathNamespaceCaching(BaseTestContainerController): captured_hdrs = {} req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'GET'}) resp = mock.MagicMock(status_int=200, - headers=self.root_resp_hdrs, + headers=resp_hdrs, request=req) resp.headers['X-Backend-Storage-Policy-Index'] = '1' with mock.patch('swift.proxy.controllers.container.' @@ -3029,7 +3435,7 @@ class TestGetPathNamespaceCaching(BaseTestContainerController): req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'GET'}) req.headers['X-Backend-Storage-Policy-Index'] = '0' resp = mock.MagicMock(status_int=200, - headers=self.root_resp_hdrs, + headers=resp_hdrs, request=req) resp.headers['X-Backend-Storage-Policy-Index'] = '1' with mock.patch('swift.proxy.controllers.container.' @@ -3043,37 +3449,23 @@ class TestGetPathNamespaceCaching(BaseTestContainerController): self.assertEqual( captured_hdrs['X-Backend-Storage-Policy-Index'], '0') - def test_GET_shard_ranges(self): - # expect shard ranges cache time to be default value of 600 - self._do_test_caching('shard', 600, 'shard', self.ns_dicts) - # expect shard ranges cache time to be configured value of 120 - self.app.recheck_listing_shard_ranges = 120 - self._do_test_caching('shard', 120, 'shard', self.ns_dicts) - - with mock.patch('swift.proxy.controllers.container.' - 'ContainerController._get_from_shards', - self._fake_get_from_shards): - self.app.recheck_listing_shard_ranges = 600 - extra_req_headers = {'X-Backend-Record-Shard-Format': 'namespace'} - self._do_test_caching('auto', 600, None, self.bogus_listing, - extra_backend_req_hdrs=extra_req_headers) - - def test_GET_shard_ranges_404_response(self): + def test_GET_namespaces_404_response(self): # pre-warm cache with container info but not shard ranges so that the # backend request tries to get a cacheable listing, but backend 404's - self.memcache.delete_all() info = headers_to_container_info(self.root_resp_hdrs) info['status'] = 200 info['sharding_state'] = 'sharded' self.memcache.set('container/a/c', info) self.memcache.clear_calls() - req = self._build_request({'X-Backend-Record-Type': 'shard'}, + req = self._build_request({'X-Backend-Record-Type': ''}, {'states': 'listing'}, {}) backend_req, resp = self._capture_backend_request( req, 404, b'', {}, num_resp=2 * self.CONTAINER_REPLICAS) self._check_backend_req( req, backend_req, - extra_hdrs={'X-Backend-Record-Type': 'shard', + extra_hdrs={'X-Backend-Record-Type': 'auto', + 'X-Backend-Record-Shard-Format': 'namespace', + 'X-Backend-Include-Deleted': 'false', 'X-Backend-Override-Shard-Name-Filter': 'sharded'}) self.assertNotIn('X-Backend-Cached-Results', resp.headers) # Note: container metadata is updated in cache but shard ranges are not @@ -3090,10 +3482,7 @@ class TestGetPathNamespaceCaching(BaseTestContainerController): 'container.shard_listing.cache.miss.404': 1}, self.logger.statsd_client.get_increment_counts()) - def test_GET_shard_ranges_read_from_cache_error(self): - self.memcache = FakeMemcache() - self.memcache.delete_all() - self.logger.clear() + def test_GET_namespaces_read_from_cache_error(self): info = headers_to_container_info(self.root_resp_hdrs) info['status'] = 200 info['sharding_state'] = 'sharded' @@ -3101,13 +3490,15 @@ class TestGetPathNamespaceCaching(BaseTestContainerController): self.memcache.clear_calls() self.memcache.error_on_get = [False, True] - req = self._build_request({'X-Backend-Record-Type': 'shard'}, + req = self._build_request({'X-Backend-Record-Type': ''}, {'states': 'listing'}, {}) backend_req, resp = self._capture_backend_request( req, 404, b'', {}, num_resp=2 * self.CONTAINER_REPLICAS) self._check_backend_req( req, backend_req, - extra_hdrs={'X-Backend-Record-Type': 'shard', + extra_hdrs={'X-Backend-Record-Type': 'auto', + 'X-Backend-Record-Shard-Format': 'namespace', + 'X-Backend-Include-Deleted': 'false', 'X-Backend-Override-Shard-Name-Filter': 'sharded'}) self.assertNotIn('X-Backend-Cached-Results', resp.headers) self.assertEqual( @@ -3122,7 +3513,39 @@ class TestGetPathNamespaceCaching(BaseTestContainerController): 'container.shard_listing.cache.error.404': 1}, self.logger.statsd_client.get_increment_counts()) - def _do_test_GET_shard_ranges_read_from_cache(self, params, record_type): + def test_GET_namespaces_read_from_cache_empty_list(self): + info = headers_to_container_info(self.root_resp_hdrs) + info['status'] = 200 + info['sharding_state'] = 'sharded' + self.memcache.set('container/a/c', info) + # note: an empty list in cache is unexpected and is treated as a miss + self.memcache.set('shard-listing-v2/a/c', []) + self.memcache.clear_calls() + + req = self._build_request({'X-Backend-Record-Type': ''}, + {'states': 'listing'}, {}) + backend_req, resp = self._capture_backend_request( + req, 404, b'', {}, num_resp=2 * self.CONTAINER_REPLICAS) + self._check_backend_req( + req, backend_req, + extra_hdrs={'X-Backend-Record-Type': 'auto', + 'X-Backend-Record-Shard-Format': 'namespace', + 'X-Backend-Include-Deleted': 'false', + 'X-Backend-Override-Shard-Name-Filter': 'sharded'}) + self.assertNotIn('X-Backend-Cached-Results', resp.headers) + self.assertEqual( + [mock.call.get('container/a/c'), + mock.call.get('shard-listing-v2/a/c', raise_on_error=True), + mock.call.set('container/a/c', mock.ANY, time=6.0)], + self.memcache.calls) + self.assertEqual(404, self.memcache.calls[2][1][1]['status']) + self.assertEqual(b'', resp.body) + self.assertEqual(404, resp.status_int) + self.assertEqual({'container.info.cache.hit': 1, + 'container.shard_listing.cache.miss.404': 1}, + self.logger.statsd_client.get_increment_counts()) + + def _do_test_GET_namespaces_read_from_cache(self, params, record_type): # pre-warm cache with container metadata and shard ranges and verify # that shard range listing are read from cache when appropriate self.memcache.delete_all() @@ -3136,7 +3559,7 @@ class TestGetPathNamespaceCaching(BaseTestContainerController): req_hdrs = {'X-Backend-Record-Type': record_type} req = self._build_request(req_hdrs, params, {}) - resp = req.get_response(self.app) + resp = self._call_app(req) self.assertEqual( [mock.call.get('container/a/c'), mock.call.get('shard-listing-v2/a/c', raise_on_error=True)], @@ -3146,70 +3569,47 @@ class TestGetPathNamespaceCaching(BaseTestContainerController): self.logger.statsd_client.get_increment_counts()) return resp - def test_GET_shard_ranges_read_from_cache(self): - exp_hdrs = {'X-Backend-Cached-Results': 'true', - 'X-Backend-Record-Type': 'shard', - 'X-Backend-Override-Shard-Name-Filter': 'true', - 'X-Backend-Sharding-State': 'sharded'} + def test_GET_namespaces_read_from_cache(self): + exp_resp_hdrs = {'X-Backend-Cached-Results': 'true', + 'X-Backend-Override-Shard-Name-Filter': 'true', + 'X-Backend-Sharding-State': 'sharded'} - resp = self._do_test_GET_shard_ranges_read_from_cache( - {'states': 'listing'}, 'shard') - self._check_response(resp, self.ns_dicts, exp_hdrs) + resp = self._do_test_GET_namespaces_read_from_cache( + {'states': 'listing'}, 'auto') + self._check_response(resp, self.bogus_listing, exp_resp_hdrs) + self.assertEqual([self.namespaces], self.get_from_shards_lists) - resp = self._do_test_GET_shard_ranges_read_from_cache( - {'states': 'listing', 'reverse': 'true'}, 'shard') - exp_shards = list(self.ns_dicts) + # no record type defaults to auto + resp = self._do_test_GET_namespaces_read_from_cache( + {'states': 'listing'}, '') + self._check_response(resp, self.bogus_listing, exp_resp_hdrs) + self.assertEqual([self.namespaces], self.get_from_shards_lists) + + resp = self._do_test_GET_namespaces_read_from_cache( + {'states': 'listing', 'reverse': 'true'}, 'auto') + self._check_response(resp, self.bogus_listing, exp_resp_hdrs) + exp_shards = list(self.namespaces) exp_shards.reverse() - self._check_response(resp, exp_shards, exp_hdrs) + self.assertEqual([exp_shards], self.get_from_shards_lists) - resp = self._do_test_GET_shard_ranges_read_from_cache( - {'states': 'listing', 'marker': 'jam'}, 'shard') - self._check_response(resp, self.ns_dicts[1:], exp_hdrs) + resp = self._do_test_GET_namespaces_read_from_cache( + {'states': 'listing', 'marker': 'jam'}, 'auto') + self._check_response(resp, self.bogus_listing, exp_resp_hdrs) + self.assertEqual([self.namespaces[1:]], self.get_from_shards_lists) - resp = self._do_test_GET_shard_ranges_read_from_cache( + resp = self._do_test_GET_namespaces_read_from_cache( {'states': 'listing', 'marker': 'jam', 'end_marker': 'kale'}, - 'shard') - self._check_response(resp, self.ns_dicts[1:2], exp_hdrs) + 'auto') + self._check_response(resp, self.bogus_listing, exp_resp_hdrs) + self.assertEqual([self.namespaces[1:2]], self.get_from_shards_lists) - resp = self._do_test_GET_shard_ranges_read_from_cache( - {'states': 'listing', 'includes': 'egg'}, 'shard') - self._check_response(resp, self.ns_dicts[:1], exp_hdrs) + resp = self._do_test_GET_namespaces_read_from_cache( + {'states': 'listing', 'includes': 'egg'}, 'auto') + self._check_response(resp, self.bogus_listing, exp_resp_hdrs) + self.assertEqual([self.namespaces[:1]], self.get_from_shards_lists) - # request with record-type=auto does not expect record-type in response - del exp_hdrs['X-Backend-Record-Type'] - with mock.patch('swift.proxy.controllers.container.' - 'ContainerController._get_from_shards', - side_effect=self._fake_get_from_shards): - resp = self._do_test_GET_shard_ranges_read_from_cache( - {'states': 'listing', 'reverse': 'true'}, 'auto') - exp_shards = list(self.namespaces) - exp_shards.reverse() - self._check_response(resp, self.bogus_listing, exp_hdrs) - self.assertEqual([exp_shards], self.get_from_shards_lists) - - self.get_from_shards_lists = [] - resp = self._do_test_GET_shard_ranges_read_from_cache( - {'states': 'listing', 'marker': 'jam'}, 'auto') - self._check_response(resp, self.bogus_listing, exp_hdrs) - self.assertEqual([self.namespaces[1:]], self.get_from_shards_lists) - - self.get_from_shards_lists = [] - resp = self._do_test_GET_shard_ranges_read_from_cache( - {'states': 'listing', 'marker': 'jam', 'end_marker': 'kale'}, - 'auto') - self._check_response(resp, self.bogus_listing, exp_hdrs) - self.assertEqual([self.namespaces[1:2]], - self.get_from_shards_lists) - - self.get_from_shards_lists = [] - resp = self._do_test_GET_shard_ranges_read_from_cache( - {'states': 'listing', 'includes': 'egg'}, 'auto') - self._check_response(resp, self.bogus_listing, exp_hdrs) - self.assertEqual([self.namespaces[:1]], self.get_from_shards_lists) - - def _do_test_GET_shard_ranges_write_to_cache( - self, params, record_type, extra_backend_req_hdrs=None): - # verify that shard range listing are written to cache when appropriate + def _do_test_GET_namespaces_write_to_cache(self, params, record_type): + # verify that namespace listing is written to cache when appropriate self.logger.clear() self.memcache.delete_all() self.memcache.clear_calls() @@ -3220,16 +3620,16 @@ class TestGetPathNamespaceCaching(BaseTestContainerController): resp_hdrs = {'X-Backend-Record-Type': 'shard', 'X-Backend-Override-Shard-Name-Filter': 'true', 'X-Backend-Sharding-State': 'sharded'} + resp_hdrs.update(self.RESP_SHARD_FORMAT_HEADERS) backend_req, resp = self._capture_backend_request( - req, 200, self._stub_shards_dump, resp_hdrs) - extra_hdrs = {'X-Backend-Record-Type': record_type, - 'X-Backend-Override-Shard-Name-Filter': 'sharded'} - if extra_backend_req_hdrs: - extra_hdrs.update(extra_backend_req_hdrs) + req, 200, self._stub_namespaces_dump, resp_hdrs) self._check_backend_req( req, backend_req, extra_params=params, - extra_hdrs=extra_hdrs) + extra_hdrs={'X-Backend-Record-Type': 'auto', + 'X-Backend-Record-Shard-Format': 'namespace', + 'X-Backend-Include-Deleted': 'false', + 'X-Backend-Override-Shard-Name-Filter': 'sharded'}) expected_hdrs = {'X-Backend-Recheck-Container-Existence': '60'} expected_hdrs.update(resp_hdrs) self.assertEqual( @@ -3239,8 +3639,9 @@ class TestGetPathNamespaceCaching(BaseTestContainerController): mock.call.set('container/a/c', mock.ANY, time=60)], self.memcache.calls) info_lines = self.logger.get_lines_for_level('info') - self.assertIn('Caching listing namespaces for shard-listing-v2/a/c ' - '(3 namespaces)', info_lines) + self.assertIn( + 'Caching listing namespaces for shard-listing-v2/a/c ' + '(3 namespaces)', info_lines) # shards were cached self.assertEqual('sharded', self.memcache.calls[2][1][1]['sharding_state']) @@ -3249,95 +3650,73 @@ class TestGetPathNamespaceCaching(BaseTestContainerController): self.logger.statsd_client.get_increment_counts()) return resp - def test_GET_shard_ranges_write_to_cache(self): - exp_hdrs = {'X-Backend-Recheck-Container-Existence': '60', - 'X-Backend-Record-Type': 'shard', - 'X-Backend-Override-Shard-Name-Filter': 'true', - 'X-Backend-Sharding-State': 'sharded'} + def test_GET_namespaces_write_to_cache(self): + exp_resp_hdrs = {'X-Backend-Recheck-Container-Existence': '60', + 'X-Backend-Override-Shard-Name-Filter': 'true', + 'X-Backend-Sharding-State': 'sharded'} - resp = self._do_test_GET_shard_ranges_write_to_cache( - {'states': 'listing'}, 'shard') - self._check_response(resp, self.ns_dicts, exp_hdrs) + resp = self._do_test_GET_namespaces_write_to_cache( + {'states': 'listing'}, 'auto') + self._check_response(resp, self.bogus_listing, exp_resp_hdrs) + self.assertEqual([self.namespaces], self.get_from_shards_lists) - resp = self._do_test_GET_shard_ranges_write_to_cache( - {'states': 'listing', 'reverse': 'true'}, 'shard') - exp_shards = list(self.ns_dicts) + # no record type defaults to auto + resp = self._do_test_GET_namespaces_write_to_cache( + {'states': 'listing'}, '') + self._check_response(resp, self.bogus_listing, exp_resp_hdrs) + self.assertEqual([self.namespaces], self.get_from_shards_lists) + + resp = self._do_test_GET_namespaces_write_to_cache( + {'states': 'listing', 'reverse': 'true'}, 'auto') + self._check_response(resp, self.bogus_listing, exp_resp_hdrs) + exp_shards = list(self.namespaces) exp_shards.reverse() - self._check_response(resp, exp_shards, exp_hdrs) + self.assertEqual([exp_shards], self.get_from_shards_lists) - resp = self._do_test_GET_shard_ranges_write_to_cache( - {'states': 'listing', 'marker': 'jam'}, 'shard') - self._check_response(resp, self.ns_dicts[1:], exp_hdrs) + resp = self._do_test_GET_namespaces_write_to_cache( + {'states': 'listing', 'marker': 'jam'}, 'auto') + self._check_response(resp, self.bogus_listing, exp_resp_hdrs) + self.assertEqual([self.namespaces[1:]], self.get_from_shards_lists) - resp = self._do_test_GET_shard_ranges_write_to_cache( + resp = self._do_test_GET_namespaces_write_to_cache( {'states': 'listing', 'marker': 'jam', 'end_marker': 'kale'}, - 'shard') - self._check_response(resp, self.ns_dicts[1:2], exp_hdrs) + 'auto') + self._check_response(resp, self.bogus_listing, exp_resp_hdrs) + self.assertEqual([self.namespaces[1:2]], self.get_from_shards_lists) - resp = self._do_test_GET_shard_ranges_write_to_cache( - {'states': 'listing', 'includes': 'egg'}, 'shard') - self._check_response(resp, self.ns_dicts[:1], exp_hdrs) + resp = self._do_test_GET_namespaces_write_to_cache( + {'states': 'listing', 'includes': 'egg'}, 'auto') + self._check_response(resp, self.bogus_listing, exp_resp_hdrs) + self.assertEqual([self.namespaces[:1]], self.get_from_shards_lists) - # request with record-type=auto does not expect record-type in response - del exp_hdrs['X-Backend-Record-Type'] - extra_req_headers = {'X-Backend-Record-Shard-Format': 'namespace'} - with mock.patch('swift.proxy.controllers.container.' - 'ContainerController._get_from_shards', - side_effect=self._fake_get_from_shards): - resp = self._do_test_GET_shard_ranges_write_to_cache( - {'states': 'listing', 'reverse': 'true'}, 'auto', - extra_backend_req_hdrs=extra_req_headers) - exp_shards = list(self.namespaces) - exp_shards.reverse() - self.assertEqual([exp_shards], self.get_from_shards_lists) - - self.get_from_shards_lists = [] - resp = self._do_test_GET_shard_ranges_write_to_cache( - {'states': 'listing', 'marker': 'jam'}, 'auto', - extra_backend_req_hdrs=extra_req_headers) - - self._check_response(resp, self.bogus_listing, exp_hdrs) - self.assertEqual([self.namespaces[1:]], self.get_from_shards_lists) - - self.get_from_shards_lists = [] - resp = self._do_test_GET_shard_ranges_write_to_cache( - {'states': 'listing', 'marker': 'jam', 'end_marker': 'kale'}, - 'auto', extra_backend_req_hdrs=extra_req_headers) - self._check_response(resp, self.bogus_listing, exp_hdrs) - self.assertEqual([self.namespaces[1:2]], - self.get_from_shards_lists) - - self.get_from_shards_lists = [] - resp = self._do_test_GET_shard_ranges_write_to_cache( - {'states': 'listing', 'includes': 'egg'}, 'auto', - extra_backend_req_hdrs=extra_req_headers) - self._check_response(resp, self.bogus_listing, exp_hdrs) - self.assertEqual([self.namespaces[:1]], self.get_from_shards_lists) - - def test_GET_shard_ranges_write_to_cache_with_x_newest(self): + def test_GET_namespaces_write_to_cache_with_x_newest(self): # when x-newest is sent, verify that there is no cache lookup to check # sharding state but then backend requests are made requesting complete - # shard list which can be cached - self.memcache.delete_all() - self.memcache.clear_calls() - req_hdrs = {'X-Backend-Record-Type': 'shard', + # namespace list which can be cached + req_hdrs = {'X-Backend-Record-Type': 'auto', 'X-Newest': 'true'} params = {'states': 'listing'} req = self._build_request(req_hdrs, params, {}) resp_hdrs = {'X-Backend-Record-Type': 'shard', 'X-Backend-Override-Shard-Name-Filter': 'true', 'X-Backend-Sharding-State': 'sharded'} + resp_hdrs.update(self.RESP_SHARD_FORMAT_HEADERS) + backend_req, resp = self._capture_backend_request( - req, 200, self._stub_shards_dump, resp_hdrs, + req, 200, self._stub_namespaces_dump, resp_hdrs, num_resp=2 * self.CONTAINER_REPLICAS) self._check_backend_req( req, backend_req, - extra_hdrs={'X-Backend-Record-Type': 'shard', + extra_hdrs={'X-Backend-Record-Type': 'auto', + 'X-Backend-Record-Shard-Format': 'namespace', + 'X-Backend-Include-Deleted': 'false', 'X-Newest': 'true', 'X-Backend-Override-Shard-Name-Filter': 'sharded'}) - expected_hdrs = {'X-Backend-Recheck-Container-Existence': '60'} - expected_hdrs.update(resp_hdrs) - self._check_response(resp, self.ns_dicts, expected_hdrs) + exp_resp_hdrs = {'X-Backend-Recheck-Container-Existence': '60', + 'X-Backend-Override-Shard-Name-Filter': 'true', + 'X-Backend-Sharding-State': 'sharded'} + self._check_response(resp, self.bogus_listing, exp_resp_hdrs) + self.assertEqual([self.namespaces], self.get_from_shards_lists) self.assertEqual( [mock.call.get('container/a/c'), mock.call.set('shard-listing-v2/a/c', self.ns_bound_list.bounds, @@ -3350,7 +3729,7 @@ class TestGetPathNamespaceCaching(BaseTestContainerController): 'container.shard_listing.cache.force_skip.200': 1}, self.logger.statsd_client.get_increment_counts()) - def _do_test_GET_shard_ranges_no_cache_write(self, resp_hdrs): + def _do_test_GET_namespaces_no_cache_write(self, resp_hdrs): # verify that there is a cache lookup to check container info but then # a backend request is made requesting complete shard list, but do not # expect shard ranges to be cached; check that marker, end_marker etc @@ -3358,25 +3737,31 @@ class TestGetPathNamespaceCaching(BaseTestContainerController): self.logger.clear() self.memcache.clear_calls() req = self._build_request( - {'X-Backend-Record-Type': 'shard'}, + {'X-Backend-Record-Type': ''}, # no record type defaults to auto {'states': 'listing', 'marker': 'egg', 'end_marker': 'jam', 'reverse': 'true'}, {}) - resp_shards = self.sr_dicts[:2] - resp_shards.reverse() + resp_namespaces = self._stub_namespaces[:2] + resp_namespaces.reverse() backend_req, resp = self._capture_backend_request( - req, 200, json.dumps(resp_shards).encode('ascii'), + req, 200, json.dumps(resp_namespaces).encode('ascii'), resp_hdrs) self._check_backend_req( req, backend_req, extra_params={'marker': 'egg', 'end_marker': 'jam', 'reverse': 'true'}, - extra_hdrs={'X-Backend-Record-Type': 'shard', + extra_hdrs={'X-Backend-Record-Type': 'auto', + 'X-Backend-Record-Shard-Format': 'namespace', + 'X-Backend-Include-Deleted': 'false', 'X-Backend-Override-Shard-Name-Filter': 'sharded'}) - expected_shards = self.sr_dicts[:2] + exp_resp_hdrs = {'X-Backend-Recheck-Container-Existence': '60', + 'X-Backend-Override-Shard-Name-Filter': 'true'} + if 'X-Backend-Sharding-State' in resp_hdrs: + exp_resp_hdrs['X-Backend-Sharding-State'] = \ + resp_hdrs['X-Backend-Sharding-State'] + self._check_response(resp, self.bogus_listing, exp_resp_hdrs) + expected_shards = self.namespaces[:2] expected_shards.reverse() - expected_hdrs = {'X-Backend-Recheck-Container-Existence': '60'} - expected_hdrs.update(resp_hdrs) - self._check_response(resp, expected_shards, expected_hdrs) + self.assertEqual([expected_shards], self.get_from_shards_lists) # container metadata is looked up in memcache for sharding state # container metadata is set in memcache self.assertEqual( @@ -3387,19 +3772,20 @@ class TestGetPathNamespaceCaching(BaseTestContainerController): self.memcache.calls[1][1][1]['sharding_state']) self.memcache.delete_all() - def test_GET_shard_ranges_no_cache_write_with_cached_container_info(self): + def test_GET_namespaces_no_cache_write_with_cached_container_info(self): # pre-warm cache with container info, but verify that shard range cache # lookup is only attempted when the cached sharding state and status # are suitable, and full set of headers can be constructed from cache; # Note: backend response has state unsharded so no shard ranges cached + def do_test(info): - self._setup_shard_range_stubs() self.memcache.set('container/a/c', info) # expect the same outcomes as if there was no cached container info - self._do_test_GET_shard_ranges_no_cache_write( - {'X-Backend-Record-Type': 'shard', - 'X-Backend-Override-Shard-Name-Filter': 'true', - 'X-Backend-Sharding-State': 'unsharded'}) + resp_headers = {'X-Backend-Record-Type': 'shard', + 'X-Backend-Override-Shard-Name-Filter': 'true', + 'X-Backend-Sharding-State': 'unsharded'} + resp_headers.update(self.RESP_SHARD_FORMAT_HEADERS) + self._do_test_GET_namespaces_no_cache_write(resp_headers) # setup a default 'good' info info = headers_to_container_info(self.root_resp_hdrs) @@ -3427,85 +3813,116 @@ class TestGetPathNamespaceCaching(BaseTestContainerController): stale_info.pop('status_changed_at') do_test(stale_info) - def test_GET_shard_ranges_no_cache_write_for_non_sharded_states(self): - # verify that shard ranges are not written to cache when container + def test_GET_namespaces_no_cache_write_for_non_sharded_states(self): + # verify that namespaces are not written to cache when container # state returned by backend is not 'sharded'; we don't expect # 'X-Backend-Override-Shard-Name-Filter': 'true' to be returned unless # the sharding state is 'sharded' but include it in this test to check # that the state is checked by proxy controller - self._do_test_GET_shard_ranges_no_cache_write( - {'X-Backend-Record-Type': 'shard', - 'X-Backend-Override-Shard-Name-Filter': 'true', - 'X-Backend-Sharding-State': 'unsharded'}) - self._do_test_GET_shard_ranges_no_cache_write( - {'X-Backend-Record-Type': 'shard', - 'X-Backend-Override-Shard-Name-Filter': 'true', - 'X-Backend-Sharding-State': 'sharding'}) - self._do_test_GET_shard_ranges_no_cache_write( - {'X-Backend-Record-Type': 'shard', - 'X-Backend-Override-Shard-Name-Filter': 'true', - 'X-Backend-Sharding-State': 'collapsed'}) - self._do_test_GET_shard_ranges_no_cache_write( - {'X-Backend-Record-Type': 'shard', - 'X-Backend-Override-Shard-Name-Filter': 'true', - 'X-Backend-Sharding-State': 'unexpected'}) + resp_hdrs = {'X-Backend-Record-Type': 'shard', + 'X-Backend-Override-Shard-Name-Filter': 'true', + 'X-Backend-Sharding-State': 'unsharded'} + resp_hdrs.update(self.RESP_SHARD_FORMAT_HEADERS) + resp_hdrs['X-Backend-Sharding-State'] = 'sharding' + self._do_test_GET_namespaces_no_cache_write(resp_hdrs) + resp_hdrs['X-Backend-Sharding-State'] = 'collapsed' + self._do_test_GET_namespaces_no_cache_write(resp_hdrs) + resp_hdrs['X-Backend-Sharding-State'] = 'unexpected' + self._do_test_GET_namespaces_no_cache_write(resp_hdrs) - def test_GET_shard_ranges_no_cache_write_for_incomplete_listing(self): - # verify that shard ranges are not written to cache when container + def test_GET_namespaces_no_cache_write_for_incomplete_listing(self): + # verify that namespaces are not written to cache when container # response does not acknowledge x-backend-override-shard-name-filter # e.g. container server not upgraded - self._do_test_GET_shard_ranges_no_cache_write( - {'X-Backend-Record-Type': 'shard', - 'X-Backend-Sharding-State': 'sharded'}) - self._do_test_GET_shard_ranges_no_cache_write( - {'X-Backend-Record-Type': 'shard', - 'X-Backend-Override-Shard-Name-Filter': 'false', - 'X-Backend-Sharding-State': 'sharded'}) - self._do_test_GET_shard_ranges_no_cache_write( - {'X-Backend-Record-Type': 'shard', - 'X-Backend-Override-Shard-Name-Filter': 'rogue', - 'X-Backend-Sharding-State': 'sharded'}) + resp_hdrs = {'X-Backend-Record-Type': 'shard', + 'X-Backend-Sharding-State': 'sharded'} + resp_hdrs.update(self.RESP_SHARD_FORMAT_HEADERS) + self._do_test_GET_namespaces_no_cache_write(resp_hdrs) + resp_hdrs['X-Backend-Override-Shard-Name-Filter'] = 'false' + self._do_test_GET_namespaces_no_cache_write(resp_hdrs) + resp_hdrs['X-Backend-Override-Shard-Name-Filter'] = 'rogue' + self._do_test_GET_namespaces_no_cache_write(resp_hdrs) - def test_GET_shard_ranges_no_cache_write_for_object_listing(self): - # verify that shard ranges are not written to cache when container + def _do_test_GET_namespaces_no_cache_write_not_namespaces(self, resp_hdrs): + # verify that there's no cache write for namespaces when backend + # response doesn't return namespaces + self.logger.clear() + self.memcache.clear_calls() + req = self._build_request( + {'X-Backend-Record-Type': ''}, # no record type defaults to auto + {'states': 'listing'}, {}) + + backend_req, resp = self._capture_backend_request( + req, 200, self.bogus_listing_body, resp_hdrs) + self._check_backend_req( + req, backend_req, + extra_hdrs={'X-Backend-Record-Type': 'auto', + 'X-Backend-Record-Shard-Format': 'namespace', + 'X-Backend-Include-Deleted': 'false', + 'X-Backend-Override-Shard-Name-Filter': 'sharded'}) + exp_resp_hdrs = {'X-Backend-Recheck-Container-Existence': '60', + 'X-Backend-Override-Shard-Name-Filter': 'true'} + for k in ('X-Backend-Record-Shard-Format', 'X-Backend-Sharding-State'): + if k in resp_hdrs: + exp_resp_hdrs[k] = resp_hdrs[k] + self._check_response(resp, self.bogus_listing, exp_resp_hdrs) + self.assertEqual([], self.get_from_shards_lists) + # container metadata is looked up in memcache for sharding state + # container metadata is set in memcache + self.assertEqual( + [mock.call.get('container/a/c'), + mock.call.set('container/a/c', mock.ANY, time=60)], + self.memcache.calls) + self.assertEqual(resp.headers.get('X-Backend-Sharding-State'), + self.memcache.calls[1][1][1]['sharding_state']) + self.memcache.delete_all() + + def test_GET_namespaces_no_cache_write_for_object_listing(self): + # verify that namespaces are not written to cache when container # response does not return shard ranges - self._do_test_GET_shard_ranges_no_cache_write( + self._do_test_GET_namespaces_no_cache_write_not_namespaces( {'X-Backend-Record-Type': 'object', 'X-Backend-Override-Shard-Name-Filter': 'true', 'X-Backend-Sharding-State': 'sharded'}) - self._do_test_GET_shard_ranges_no_cache_write( + self._do_test_GET_namespaces_no_cache_write_not_namespaces( {'X-Backend-Record-Type': 'other', 'X-Backend-Override-Shard-Name-Filter': 'true', 'X-Backend-Sharding-State': 'sharded'}) - self._do_test_GET_shard_ranges_no_cache_write( + self._do_test_GET_namespaces_no_cache_write_not_namespaces( {'X-Backend-Record-Type': 'true', 'X-Backend-Override-Shard-Name-Filter': 'true', 'X-Backend-Sharding-State': 'sharded'}) - self._do_test_GET_shard_ranges_no_cache_write( + self._do_test_GET_namespaces_no_cache_write_not_namespaces( {'X-Backend-Override-Shard-Name-Filter': 'true', 'X-Backend-Sharding-State': 'sharded'}) - def _do_test_GET_shard_ranges_bad_response_body(self, resp_body): + def _do_test_GET_namespaces_bad_response_body(self, resp_body): # verify that resp body is not cached if shard range parsing fails; # check the original unparseable response body is returned - self._setup_shard_range_stubs() + self.bogus_listing_body = json.dumps(resp_body).encode('ascii') self.memcache.clear_calls() req = self._build_request( - {'X-Backend-Record-Type': 'shard'}, + {'X-Backend-Record-Type': ''}, {'states': 'listing'}, {}) resp_hdrs = {'X-Backend-Record-Type': 'shard', 'X-Backend-Override-Shard-Name-Filter': 'true', 'X-Backend-Sharding-State': 'sharded'} + resp_hdrs.update(self.RESP_SHARD_FORMAT_HEADERS) + backend_req, resp = self._capture_backend_request( - req, 200, json.dumps(resp_body).encode('ascii'), - resp_hdrs) + req, 200, self.bogus_listing_body, resp_hdrs) self._check_backend_req( req, backend_req, - extra_hdrs={'X-Backend-Record-Type': 'shard', + extra_hdrs={'X-Backend-Record-Type': 'auto', + 'X-Backend-Record-Shard-Format': 'namespace', + 'X-Backend-Include-Deleted': 'false', 'X-Backend-Override-Shard-Name-Filter': 'sharded'}) - expected_hdrs = {'X-Backend-Recheck-Container-Existence': '60'} - expected_hdrs.update(resp_hdrs) - self._check_response(resp, resp_body, expected_hdrs) + exp_resp_hdrs = {'X-Backend-Recheck-Container-Existence': '60', + 'X-Backend-Override-Shard-Name-Filter': 'true'} + if 'X-Backend-Sharding-State' in resp_hdrs: + exp_resp_hdrs['X-Backend-Sharding-State'] = \ + resp_hdrs['X-Backend-Sharding-State'] + self._check_response(resp, resp_body, exp_resp_hdrs) # container metadata is looked up in memcache for sharding state # container metadata is set in memcache self.assertEqual( @@ -3519,55 +3936,65 @@ class TestGetPathNamespaceCaching(BaseTestContainerController): self.logger.statsd_client.get_increment_counts()) self.memcache.delete_all() - def test_GET_shard_ranges_bad_response_body(self): - self._do_test_GET_shard_ranges_bad_response_body( + def test_GET_namespaces_bad_response_body(self): + self._do_test_GET_namespaces_bad_response_body( {'bad': 'data', 'not': ' a list'}) error_lines = self.logger.get_lines_for_level('error') self.assertEqual(1, len(error_lines), error_lines) self.assertIn('Problem with listing response', error_lines[0]) self.logger.clear() - self._do_test_GET_shard_ranges_bad_response_body( - [{'not': ' a shard range'}]) + self._do_test_GET_namespaces_bad_response_body( + [{'not': 'a namespace'}]) error_lines = self.logger.get_lines_for_level('error') self.assertEqual(1, len(error_lines), error_lines) self.assertIn('Failed to get namespaces', error_lines[0]) self.logger.clear() - self._do_test_GET_shard_ranges_bad_response_body( - 'not json') + self._do_test_GET_namespaces_bad_response_body('not a list') error_lines = self.logger.get_lines_for_level('error') self.assertEqual(1, len(error_lines), error_lines) self.assertIn('Problem with listing response', error_lines[0]) - def _do_test_GET_shards_no_cache(self, sharding_state, req_params, - req_hdrs=None): - # verify that a shard GET request does not lookup in cache or attempt - # to cache shard ranges fetched from backend + def _do_test_GET_namespaces_cache_unused(self, sharding_state, req_params, + req_hdrs=None): + # verify cases when a GET request does not lookup in cache or attempt + # to cache namespaces fetched from backend self.memcache.delete_all() self.memcache.clear_calls() req_params.update(dict(marker='egg', end_marker='jam')) - hdrs = {'X-Backend-Record-Type': 'shard'} + hdrs = {'X-Backend-Record-Type': ''} if req_hdrs: hdrs.update(req_hdrs) + req = self._build_request(hdrs, req_params, {}) - resp_shards = self.sr_dicts[:2] + resp_shards = self._stub_namespaces[:2] + + resp_headers = {'X-Backend-Record-Type': 'shard', + 'X-Backend-Sharding-State': sharding_state} + resp_headers.update(self.RESP_SHARD_FORMAT_HEADERS) backend_req, resp = self._capture_backend_request( req, 200, json.dumps(resp_shards).encode('ascii'), - {'X-Backend-Record-Type': 'shard', - 'X-Backend-Sharding-State': sharding_state}) - self._check_backend_req( - req, backend_req, extra_hdrs=hdrs, extra_params=req_params) - expected_shards = self.sr_dicts[:2] - self._check_response(resp, expected_shards, { - 'X-Backend-Recheck-Container-Existence': '60', - 'X-Backend-Record-Type': 'shard', - 'X-Backend-Sharding-State': sharding_state}) + resp_headers) - def _do_test_GET_shards_no_cache_listing(self, sharding_state): + exp_backend_req_hdrs = dict(hdrs) + exp_backend_req_hdrs.update({ + 'X-Backend-Record-Type': 'auto', + 'X-Backend-Record-Shard-Format': 'namespace', + 'X-Backend-Include-Deleted': 'false', + }) + self._check_backend_req( + req, backend_req, extra_hdrs=exp_backend_req_hdrs, + extra_params=req_params) + self._check_response(resp, self.bogus_listing, { + 'X-Backend-Recheck-Container-Existence': '60', + 'X-Backend-Sharding-State': sharding_state}) + self.assertEqual([self.namespaces[:2]], self.get_from_shards_lists) + + def _do_test_GET_namespaces_cache_unused_listing(self, sharding_state): # container metadata from backend response is set in memcache - self._do_test_GET_shards_no_cache(sharding_state, - {'states': 'listing'}) + self._do_test_GET_namespaces_cache_unused(sharding_state, + {'states': 'listing'}) self.assertEqual( [mock.call.get('container/a/c'), mock.call.set('container/a/c', mock.ANY, time=60)], @@ -3575,103 +4002,45 @@ class TestGetPathNamespaceCaching(BaseTestContainerController): self.assertEqual(sharding_state, self.memcache.calls[1][1][1]['sharding_state']) - def test_GET_shard_ranges_no_cache_recheck_listing_shard_ranges(self): - # verify that a GET for shards does not lookup or store in cache when + def test_GET_namespaces_cache_unused_recheck_listing_shard_ranges(self): + # verify that a GET does not lookup or store namespaces in cache when # cache expiry time is set to zero self.app.recheck_listing_shard_ranges = 0 - self._do_test_GET_shards_no_cache_listing('unsharded') - self._do_test_GET_shards_no_cache_listing('sharding') - self._do_test_GET_shards_no_cache_listing('sharded') - self._do_test_GET_shards_no_cache_listing('collapsed') - self._do_test_GET_shards_no_cache_listing('unexpected') + self._do_test_GET_namespaces_cache_unused_listing('unsharded') + self._do_test_GET_namespaces_cache_unused_listing('sharding') + self._do_test_GET_namespaces_cache_unused_listing('sharded') + self._do_test_GET_namespaces_cache_unused_listing('collapsed') + self._do_test_GET_namespaces_cache_unused_listing('unexpected') - def _do_test_GET_shards_no_cache_updating(self, sharding_state): - # container metadata from backend response is set in memcache - self._do_test_GET_shards_no_cache(sharding_state, - {'states': 'updating'}) - self.assertEqual( - [mock.call.set('container/a/c', mock.ANY, time=60)], - self.memcache.calls) - self.assertEqual(sharding_state, - self.memcache.calls[0][1][1]['sharding_state']) - - def test_GET_shard_ranges_no_cache_when_requesting_updating_shards(self): - # verify that a GET for shards in updating states does not lookup or - # store in cache - self._do_test_GET_shards_no_cache_updating('unsharded') - self._do_test_GET_shards_no_cache_updating('sharding') - self._do_test_GET_shards_no_cache_updating('sharded') - self._do_test_GET_shards_no_cache_updating('collapsed') - self._do_test_GET_shards_no_cache_updating('unexpected') - - def test_GET_shard_ranges_no_cache_when_include_deleted_shards(self): - # verify that a GET for shards in listing states does not lookup or - # store in cache if x-backend-include-deleted is true - self._do_test_GET_shards_no_cache( - 'unsharded', {'states': 'listing'}, - {'X-Backend-Include-Deleted': 'true'}) - self._do_test_GET_shards_no_cache( - 'sharding', {'states': 'listing'}, - {'X-Backend-Include-Deleted': 'true'}) - self._do_test_GET_shards_no_cache( - 'sharded', {'states': 'listing'}, - {'X-Backend-Include-Deleted': 'true'}) - self._do_test_GET_shards_no_cache( - 'collapsed', {'states': 'listing'}, - {'X-Backend-Include-Deleted': 'true'}) - self._do_test_GET_shards_no_cache( - 'unexpected', {'states': 'listing'}, - {'X-Backend-Include-Deleted': 'true'}) - - def test_GET_objects_makes_no_cache_lookup(self): - # verify that an object GET request does not lookup container metadata - # in cache - self.memcache.delete_all() - self.memcache.clear_calls() - req_hdrs = {'X-Backend-Record-Type': 'object'} - # we would not expect states=listing to be used with an object request - # but include it here to verify that it is ignored - req = self._build_request(req_hdrs, {'states': 'listing'}, {}) - resp_body = json.dumps(['object listing']).encode('ascii') - backend_req, resp = self._capture_backend_request( - req, 200, resp_body, - {'X-Backend-Record-Type': 'object', - 'X-Backend-Sharding-State': 'sharded'}) - self._check_backend_req( - req, backend_req, - extra_hdrs=req_hdrs) - self._check_response(resp, ['object listing'], { - 'X-Backend-Recheck-Container-Existence': '60', - 'X-Backend-Record-Type': 'object', - 'X-Backend-Sharding-State': 'sharded'}) - # container metadata from backend response is set in memcache - self.assertEqual( - [mock.call.set('container/a/c', mock.ANY, time=60)], - self.memcache.calls) - self.assertEqual('sharded', - self.memcache.calls[0][1][1]['sharding_state']) - - def test_GET_shard_ranges_no_memcache_available(self): - self.memcache.clear_calls() - hdrs = {'X-Backend-Record-Type': 'shard'} + def test_GET_namespaces_no_memcache_available(self): + req_hdrs = {'X-Backend-Record-Type': ''} params = {'states': 'listing'} - req = self._build_request(hdrs, params, {}) + req = self._build_request(req_hdrs, params, {}) req.environ['swift.cache'] = None + + resp_hdrs = {'X-Backend-Record-Type': 'shard', + 'X-Backend-Sharding-State': 'sharded'} + resp_hdrs.update(self.RESP_SHARD_FORMAT_HEADERS) backend_req, resp = self._capture_backend_request( - req, 200, self._stub_shards_dump, - {'X-Backend-Record-Type': 'shard', - 'X-Backend-Sharding-State': 'sharded'}) + req, 200, self._stub_namespaces_dump, resp_hdrs) + + exp_backend_req_hdrs = dict(req_hdrs) + exp_backend_req_hdrs.update( + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Record-Shard-Format': 'namespace', + 'X-Backend-Include-Deleted': 'false', + }) self._check_backend_req( - req, backend_req, extra_params=params, extra_hdrs=hdrs) - expected_shards = self.sr_dicts - self._check_response(resp, expected_shards, { + req, backend_req, extra_params=params, + extra_hdrs=exp_backend_req_hdrs) + self._check_response(resp, self.bogus_listing, { 'X-Backend-Recheck-Container-Existence': '60', - 'X-Backend-Record-Type': 'shard', 'X-Backend-Sharding-State': 'sharded'}) + self.assertEqual([self.namespaces], self.get_from_shards_lists) self.assertEqual([], self.memcache.calls) # sanity check def test_cache_clearing(self): - # verify that both metadata and shard ranges are purged form memcache + # verify that both metadata and shard ranges are purged from memcache # on PUT, POST and DELETE def do_test(method, resp_status, num_resp): self.assertGreater(num_resp, 0) # sanity check @@ -3697,6 +4066,157 @@ class TestGetPathNamespaceCaching(BaseTestContainerController): do_test('PUT', 202, self.CONTAINER_REPLICAS) +class TestGetPathNamespaceCachingLegacy(TestGetPathNamespaceCaching): + # old container servers did not return this header + RESP_SHARD_FORMAT_HEADERS = {} + + def setUp(self): + super(TestGetPathNamespaceCachingLegacy, self).setUp() + + def _setup_namespace_stubs(self): + # old container servers always returned full format ShardRange dicts + self._stub_namespaces = [ + dict(ShardRange(timestamp=Timestamp.now(), **ns)) + for ns in self.ns_dicts] + self._stub_namespaces_dump = json.dumps(self._stub_namespaces).encode( + 'ascii') + + +class TestGetExplicitRecordType(BaseTestContainerControllerGetPath): + RESP_SHARD_FORMAT_HEADERS = {'X-Backend-Record-Shard-Format': 'full'} + + def setUp(self): + super(TestGetExplicitRecordType, self).setUp() + self._setup_shard_range_stubs() + + def _setup_shard_range_stubs(self): + self._stub_shards = [dict(ShardRange(timestamp=Timestamp.now(), **ns)) + for ns in self.ns_dicts] + self._stub_shards_dump = json.dumps(self.ns_dicts).encode('ascii') + + def _do_test_GET_shard_ranges_no_cache(self, sharding_state, req_params, + req_hdrs=None): + # verify that an explicit shard GET request does not lookup in cache or + # attempt to cache shard ranges fetched from backend + self.memcache.delete_all() + self.memcache.clear_calls() + req_params.update(dict(marker='egg', end_marker='jam')) + hdrs = {'X-Backend-Record-Type': 'shard'} + if req_hdrs: + hdrs.update(req_hdrs) + + req = self._build_request(hdrs, req_params, {}) + resp_shards = self._stub_shards[:2] + + resp_headers = {'X-Backend-Record-Type': 'shard', + 'X-Backend-Sharding-State': sharding_state} + resp_headers.update(self.RESP_SHARD_FORMAT_HEADERS) + backend_req, resp = self._capture_backend_request( + req, 200, json.dumps(resp_shards).encode('ascii'), + resp_headers) + + exp_backend_req_hdrs = dict(hdrs) + exp_backend_req_hdrs.update({ + 'X-Backend-Record-Type': 'shard', + }) + self._check_backend_req( + req, backend_req, extra_hdrs=exp_backend_req_hdrs, + extra_params=req_params) + expected_shards = self._stub_shards[:2] + exp_resp_hdrs = dict(resp_headers) + exp_resp_hdrs.update(self.RESP_SHARD_FORMAT_HEADERS) + exp_resp_hdrs['X-Backend-Recheck-Container-Existence'] = '60' + self._check_response(resp, expected_shards, exp_resp_hdrs) + + def _do_test_GET_shard_ranges_no_cache_updating(self, sharding_state): + # container metadata from backend response is set in memcache + self._do_test_GET_shard_ranges_no_cache(sharding_state, + {'states': 'updating'}) + self.assertEqual( + [mock.call.set('container/a/c', mock.ANY, time=60)], + self.memcache.calls) + self.assertEqual(sharding_state, + self.memcache.calls[0][1][1]['sharding_state']) + + def test_GET_shard_ranges_no_cache_when_requesting_updating_shards(self): + # verify that a GET for shard record type in updating states does not + # lookup or store in cache + self._do_test_GET_shard_ranges_no_cache_updating('unsharded') + self._do_test_GET_shard_ranges_no_cache_updating('sharding') + self._do_test_GET_shard_ranges_no_cache_updating('sharded') + self._do_test_GET_shard_ranges_no_cache_updating('collapsed') + self._do_test_GET_shard_ranges_no_cache_updating('unexpected') + + def _do_test_GET_shard_ranges_no_cache_listing(self, sharding_state): + # container metadata from backend response is set in memcache + self._do_test_GET_shard_ranges_no_cache(sharding_state, + {'states': 'listing'}) + self.assertEqual( + [mock.call.set('container/a/c', mock.ANY, time=60)], + self.memcache.calls) + self.assertEqual(sharding_state, + self.memcache.calls[0][1][1]['sharding_state']) + + def test_GET_shard_ranges_no_cache_when_requesting_listing_shards(self): + # verify that a GET for shard record type in listing states does not + # lookup or store in cache + self._do_test_GET_shard_ranges_no_cache_listing('unsharded') + self._do_test_GET_shard_ranges_no_cache_listing('sharding') + self._do_test_GET_shard_ranges_no_cache_listing('sharded') + self._do_test_GET_shard_ranges_no_cache_listing('collapsed') + self._do_test_GET_shard_ranges_no_cache_listing('unexpected') + + def test_GET_shard_ranges_no_cache_when_include_deleted_shards(self): + # verify that a GET for shards in listing states does not lookup or + # store in cache if x-backend-include-deleted is true + self._do_test_GET_shard_ranges_no_cache( + 'unsharded', {'states': 'listing'}, + {'X-Backend-Include-Deleted': 'true'}) + self._do_test_GET_shard_ranges_no_cache( + 'sharding', {'states': 'listing'}, + {'X-Backend-Include-Deleted': 'true'}) + self._do_test_GET_shard_ranges_no_cache( + 'sharded', {'states': 'listing'}, + {'X-Backend-Include-Deleted': 'true'}) + self._do_test_GET_shard_ranges_no_cache( + 'collapsed', {'states': 'listing'}, + {'X-Backend-Include-Deleted': 'true'}) + self._do_test_GET_shard_ranges_no_cache( + 'unexpected', {'states': 'listing'}, + {'X-Backend-Include-Deleted': 'true'}) + + def test_GET_record_type_object_makes_no_cache_lookup(self): + # verify that an GET request explicitly asking for record-type 'object' + # does not lookup container metadata in cache + req_hdrs = {'X-Backend-Record-Type': 'object'} + # we would not expect states=listing to be used with an object request + # but include it here to verify that it is ignored + req = self._build_request(req_hdrs, {'states': 'listing'}, {}) + resp_body = json.dumps(['object listing']).encode('ascii') + backend_req, resp = self._capture_backend_request( + req, 200, resp_body, + {'X-Backend-Record-Type': 'object', + 'X-Backend-Sharding-State': 'sharded'}) + self._check_backend_req( + req, backend_req, + extra_hdrs=req_hdrs) + self._check_response(resp, ['object listing'], { + 'X-Backend-Recheck-Container-Existence': '60', + 'X-Backend-Record-Type': 'object', + 'X-Backend-Sharding-State': 'sharded'}) + # container metadata from backend response is set in memcache + self.assertEqual( + [mock.call.set('container/a/c', mock.ANY, time=60)], + self.memcache.calls) + self.assertEqual('sharded', + self.memcache.calls[0][1][1]['sharding_state']) + + +class TestGetExplicitRecordTypeLegacy(TestGetExplicitRecordType): + # old container servers did not return this header + RESP_SHARD_FORMAT_HEADERS = {} + + @patch_policies( [StoragePolicy(0, 'zero', True, object_ring=FakeRing(replicas=4))]) class TestContainerController4Replicas(TestContainerController):