proxy: only use listing shards cache for 'auto' listings
The proxy should NOT read or write to memcache when handling a container GET that explicitly requests 'shard' or 'object' record type. A request for 'shard' record type may specify 'namespace' format, but this request is unrelated to container listings or object updates and passes directly to the backend. This patch also removes unnecessary JSON serialisation and de-serialisation of namespaces within the proxy GET path when a sharded object listing is being built. The final response body will contain a list of objects so there is no need to write intermediate response bodies with a list of namespaces. Requests that explicitly specify record type of 'shard' will of course still have the response body with serialised shard dicts that is returned from the backend. Change-Id: Id79c156432350c11c52a4004d69b85e9eb904ca6
This commit is contained in:
Alistair Coles
parent
03b66c94f4
commit
252f0d36b7
@ -896,8 +896,10 @@ def get_namespaces_from_cache(req, cache_key, skip_chance):
|
||||
:param req: a :class:`swift.common.swob.Request` object.
|
||||
:param cache_key: the cache key for both infocache and memcache.
|
||||
:param skip_chance: the probability of skipping the memcache look-up.
|
||||
:return: a tuple of
|
||||
(:class:`swift.common.utils.NamespaceBoundList`, cache state)
|
||||
:return: a tuple of (value, cache state). Value is an instance of
|
||||
:class:`swift.common.utils.NamespaceBoundList` if a non-empty list is
|
||||
found in memcache. Otherwise value is ``None``, for example if memcache
|
||||
look-up was skipped, or no value was found, or an empty list was found.
|
||||
"""
|
||||
# try get namespaces from infocache first
|
||||
infocache = req.environ.setdefault('swift.infocache', {})
|
||||
|
||||
@ -108,15 +108,14 @@ class ContainerController(Controller):
|
||||
req.swift_entity_path, concurrency)
|
||||
return resp
|
||||
|
||||
def _make_namespaces_response_body(self, req, ns_bound_list):
|
||||
def _filter_complete_listing(self, req, namespaces):
|
||||
"""
|
||||
Filter namespaces according to request constraints and return a
|
||||
serialised list of namespaces.
|
||||
Filter complete list of namespaces to return only those specified by
|
||||
the request constraints.
|
||||
|
||||
:param req: the request object.
|
||||
:param ns_bound_list: an instance of
|
||||
:class:`~swift.common.utils.NamespaceBoundList`.
|
||||
:return: a serialised list of namespaces.
|
||||
:param req: a :class:`~swift.common.swob.Request`.
|
||||
:param namespaces: a list of :class:`~swift.common.utils.Namespace`.
|
||||
:return: a list of :class:`~swift.common.utils.Namespace`.
|
||||
"""
|
||||
marker = get_param(req, 'marker', '')
|
||||
end_marker = get_param(req, 'end_marker')
|
||||
@ -124,141 +123,150 @@ class ContainerController(Controller):
|
||||
reverse = config_true_value(get_param(req, 'reverse'))
|
||||
if reverse:
|
||||
marker, end_marker = end_marker, marker
|
||||
namespaces = ns_bound_list.get_namespaces()
|
||||
namespaces = filter_namespaces(
|
||||
namespaces, includes, marker, end_marker)
|
||||
if reverse:
|
||||
namespaces.reverse()
|
||||
return json.dumps([dict(ns) for ns in namespaces]).encode('ascii')
|
||||
return namespaces
|
||||
|
||||
def _get_shard_ranges_from_cache(self, req, headers):
|
||||
def _get_listing_namespaces_from_cache(self, req, headers):
|
||||
"""
|
||||
Try to fetch shard namespace data from cache and, if successful, return
|
||||
a response. Also return the cache state.
|
||||
|
||||
The response body will be a list of dicts each of which describes
|
||||
a Namespace (i.e. includes the keys ``lower``, ``upper`` and ``name``).
|
||||
a list of Namespaces. Also return the cache state.
|
||||
|
||||
:param req: an instance of ``swob.Request``.
|
||||
:param headers: Headers to be sent with request.
|
||||
:return: a tuple comprising (an instance of ``swob.Response``or
|
||||
``None`` if no namespaces were found in cache, the cache state).
|
||||
:return: a tuple comprising (a list instance of ``Namespace`` objects
|
||||
or ``None`` if no namespaces were found in cache, the cache state).
|
||||
"""
|
||||
cache_key = get_cache_key(self.account_name, self.container_name,
|
||||
shard='listing')
|
||||
skip_chance = self.app.container_listing_shard_ranges_skip_cache
|
||||
ns_bound_list, cache_state = get_namespaces_from_cache(
|
||||
req, cache_key, skip_chance)
|
||||
if ns_bound_list:
|
||||
# shard ranges can be returned from cache
|
||||
resp_body = self._make_namespaces_response_body(req, ns_bound_list)
|
||||
self.logger.debug('Found %d shards in cache for %s',
|
||||
len(ns_bound_list.bounds), req.path_qs)
|
||||
headers.update({'x-backend-record-type': 'shard',
|
||||
'x-backend-cached-results': 'true'})
|
||||
# mimic GetOrHeadHandler.get_working_response...
|
||||
# note: server sets charset with content_type but proxy
|
||||
# GETorHEAD_base does not, so don't set it here either
|
||||
resp = Response(request=req, body=resp_body)
|
||||
update_headers(resp, headers)
|
||||
resp.last_modified = Timestamp(headers['x-put-timestamp']).ceil()
|
||||
resp.environ['swift_x_timestamp'] = headers.get('x-timestamp')
|
||||
resp.accept_ranges = 'bytes'
|
||||
resp.content_type = 'application/json'
|
||||
else:
|
||||
resp = None
|
||||
if not ns_bound_list:
|
||||
return None, None, cache_state
|
||||
|
||||
return resp, cache_state
|
||||
# Namespaces found in cache so there is no need to go to backend,
|
||||
# but we need to build response headers: mimic
|
||||
# GetOrHeadHandler.get_working_response...
|
||||
# note: server sets charset with content_type but proxy
|
||||
# GETorHEAD_base does not, so don't set it here either
|
||||
namespaces = ns_bound_list.get_namespaces()
|
||||
self.logger.debug('Found %d shards in cache for %s',
|
||||
len(namespaces), req.path_qs)
|
||||
headers.update({'x-backend-record-type': 'shard',
|
||||
'x-backend-record-shard-format': 'namespace',
|
||||
'x-backend-cached-results': 'true'})
|
||||
resp = Response(request=req)
|
||||
update_headers(resp, headers)
|
||||
resp.last_modified = Timestamp(headers['x-put-timestamp']).ceil()
|
||||
resp.environ['swift_x_timestamp'] = headers.get('x-timestamp')
|
||||
resp.accept_ranges = 'bytes'
|
||||
resp.content_type = 'application/json'
|
||||
namespaces = self._filter_complete_listing(req, namespaces)
|
||||
return resp, namespaces, cache_state
|
||||
|
||||
def _store_shard_ranges_in_cache(self, req, resp):
|
||||
def _set_listing_namespaces_in_cache(self, req, namespaces):
|
||||
"""
|
||||
Parse shard ranges returned from backend, store them in both infocache
|
||||
and memcache.
|
||||
Store a list of namespaces in both infocache and memcache.
|
||||
|
||||
Note: the returned list of namespaces may not be identical to the given
|
||||
list. Any gaps in the given namespaces will be 'lost' as a result of
|
||||
compacting the list of namespaces to a NamespaceBoundList for caching.
|
||||
That is ok. When the cached NamespaceBoundList is transformed back to
|
||||
Namespaces to perform a listing, the Namespace before each gap will
|
||||
have expanded to include the gap, which means that the backend GET to
|
||||
that shard will have an end_marker beyond that shard's upper bound, and
|
||||
equal to the next available shard's lower. At worst, some misplaced
|
||||
objects, in the gap above the shard's upper, may be included in the
|
||||
shard's response.
|
||||
|
||||
:param req: the request object.
|
||||
:param resp: the response object for the shard range listing.
|
||||
:return: an instance of
|
||||
:class:`~swift.common.utils.NamespaceBoundList`.
|
||||
:param namespaces: a list of :class:`~swift.common.utils.Namespace`
|
||||
objects.
|
||||
:return: a list of :class:`~swift.common.utils.Namespace` objects.
|
||||
"""
|
||||
# Note: Any gaps in the response's shard ranges will be 'lost' as a
|
||||
# result of compacting the list of shard ranges to a
|
||||
# NamespaceBoundList. That is ok. When the cached NamespaceBoundList is
|
||||
# transformed back to shard range Namespaces to perform a listing, the
|
||||
# Namespace before each gap will have expanded to include the gap,
|
||||
# which means that the backend GET to that shard will have an
|
||||
# end_marker beyond that shard's upper bound, and equal to the next
|
||||
# available shard's lower. At worst, some misplaced objects, in the gap
|
||||
# above the shard's upper, may be included in the shard's response.
|
||||
data = self._parse_listing_response(req, resp)
|
||||
backend_shard_ranges = self._parse_namespaces(req, data, resp)
|
||||
if backend_shard_ranges is None:
|
||||
return None
|
||||
cache_key = get_cache_key(self.account_name, self.container_name,
|
||||
shard='listing')
|
||||
ns_bound_list = NamespaceBoundList.parse(namespaces)
|
||||
# cache in infocache even if no namespaces returned; this
|
||||
# is unexpected but use that result for this request
|
||||
set_cache_state = set_namespaces_in_cache(
|
||||
req, cache_key, ns_bound_list,
|
||||
self.app.recheck_listing_shard_ranges)
|
||||
if set_cache_state == 'set':
|
||||
self.logger.info(
|
||||
'Caching listing namespaces for %s (%d namespaces)',
|
||||
cache_key, len(ns_bound_list.bounds))
|
||||
# return the de-gapped namespaces
|
||||
return ns_bound_list.get_namespaces()
|
||||
|
||||
ns_bound_list = NamespaceBoundList.parse(backend_shard_ranges)
|
||||
if resp.headers.get('x-backend-sharding-state') == 'sharded':
|
||||
# cache in infocache even if no shard ranges returned; this
|
||||
# is unexpected but use that result for this request
|
||||
cache_key = get_cache_key(
|
||||
self.account_name, self.container_name, shard='listing')
|
||||
set_cache_state = set_namespaces_in_cache(
|
||||
req, cache_key, ns_bound_list,
|
||||
self.app.recheck_listing_shard_ranges)
|
||||
if set_cache_state == 'set':
|
||||
self.logger.info(
|
||||
'Caching listing namespaces for %s (%d namespaces)',
|
||||
cache_key, len(ns_bound_list.bounds))
|
||||
return ns_bound_list
|
||||
|
||||
def _get_shard_ranges_from_backend(self, req):
|
||||
def _get_listing_namespaces_from_backend(self, req, cache_enabled):
|
||||
"""
|
||||
Make a backend request for shard ranges and return a response.
|
||||
|
||||
The response body will be a list of dicts each of which describes
|
||||
a Namespace (i.e. includes the keys ``lower``, ``upper`` and ``name``).
|
||||
If the response headers indicate that the response body contains a
|
||||
complete list of shard ranges for a sharded container then the response
|
||||
body will be transformed to a ``NamespaceBoundsList`` and cached.
|
||||
Fetch shard namespace data from the backend and, if successful, return
|
||||
a list of Namespaces.
|
||||
|
||||
:param req: an instance of ``swob.Request``.
|
||||
:return: an instance of ``swob.Response``.
|
||||
:param cache_enabled: a boolean which should be True if memcache is
|
||||
available to cache the returned data, False otherwise.
|
||||
:return: a list instance of ``Namespace`` objects or ``None`` if no
|
||||
namespace data was returned from the backend.
|
||||
"""
|
||||
# Note: We instruct the backend server to ignore name constraints in
|
||||
# request params if returning shard ranges so that the response can
|
||||
# potentially be cached, but we only cache it if the container state is
|
||||
# 'sharded'. We don't attempt to cache shard ranges for a 'sharding'
|
||||
# container as they may include the container itself as a 'gap filler'
|
||||
# for shard ranges that have not yet cleaved; listings from 'gap
|
||||
# filler' shard ranges are likely to become stale as the container
|
||||
# continues to cleave objects to its shards and caching them is
|
||||
# therefore more likely to result in stale or incomplete listings on
|
||||
# subsequent container GETs.
|
||||
req.headers['x-backend-override-shard-name-filter'] = 'sharded'
|
||||
# Instruct the backend server to 'automatically' return namespaces
|
||||
# of shards in a 'listing' state if the container is sharded, and
|
||||
# that the more compact 'namespace' format is sufficient. Older
|
||||
# container servers may still respond with the 'full' shard range
|
||||
# format.
|
||||
req.headers['X-Backend-Record-Type'] = 'auto'
|
||||
req.headers['X-Backend-Record-Shard-Format'] = 'namespace'
|
||||
# 'x-backend-include-deleted' is not expected in 'auto' requests to
|
||||
# the proxy (it's not supported for objects and is used by the
|
||||
# sharder when explicitly fetching 'shard' record type), but we
|
||||
# explicitly set it to false here just in case. A newer container
|
||||
# server would ignore it when returning namespaces, but an older
|
||||
# container server would include unwanted deleted shard range.
|
||||
req.headers['X-Backend-Include-Deleted'] = 'false'
|
||||
params = req.params
|
||||
params['states'] = 'listing'
|
||||
req.params = params
|
||||
if cache_enabled:
|
||||
# Instruct the backend server to ignore name constraints in
|
||||
# request params if returning namespaces so that the response
|
||||
# can potentially be cached, but only if the container state is
|
||||
# 'sharded'. We don't attempt to cache namespaces for a
|
||||
# 'sharding' container as they may include the container itself
|
||||
# as a 'gap filler' for shards that have not yet cleaved;
|
||||
# listings from 'gap filler' namespaces are likely to become
|
||||
# stale as the container continues to cleave objects to its
|
||||
# shards and caching them is therefore more likely to result in
|
||||
# stale or incomplete listings on subsequent container GETs.
|
||||
req.headers['x-backend-override-shard-name-filter'] = 'sharded'
|
||||
resp = self._GETorHEAD_from_backend(req)
|
||||
|
||||
sharding_state = resp.headers.get(
|
||||
'x-backend-sharding-state', '').lower()
|
||||
resp_record_type = resp.headers.get(
|
||||
'x-backend-record-type', '').lower()
|
||||
sharding_state = resp.headers.get(
|
||||
'x-backend-sharding-state', '').lower()
|
||||
complete_listing = config_true_value(resp.headers.pop(
|
||||
'x-backend-override-shard-name-filter', False))
|
||||
# given that we sent 'x-backend-override-shard-name-filter=sharded' we
|
||||
# should only receive back 'x-backend-override-shard-name-filter=true'
|
||||
# if the sharding state is 'sharded', but check them both anyway...
|
||||
if (resp_record_type == 'shard' and
|
||||
sharding_state == 'sharded' and
|
||||
complete_listing):
|
||||
# note: old container servers return a list of shard ranges, newer
|
||||
# ones return a list of namespaces. If we ever need to know we can
|
||||
# look for a 'x-backend-record-shard-format' header from newer
|
||||
# container servers.
|
||||
ns_bound_list = self._store_shard_ranges_in_cache(req, resp)
|
||||
if ns_bound_list:
|
||||
resp.body = self._make_namespaces_response_body(
|
||||
req, ns_bound_list)
|
||||
return resp
|
||||
if resp_record_type == 'shard':
|
||||
data = self._parse_listing_response(req, resp)
|
||||
namespaces = self._parse_namespaces(req, data, resp)
|
||||
# given that we sent
|
||||
# 'x-backend-override-shard-name-filter=sharded' we should only
|
||||
# receive back 'x-backend-override-shard-name-filter=true' if
|
||||
# the sharding state is 'sharded', but check them both
|
||||
# anyway...
|
||||
if (namespaces and
|
||||
sharding_state == 'sharded' and
|
||||
complete_listing):
|
||||
namespaces = self._set_listing_namespaces_in_cache(
|
||||
req, namespaces)
|
||||
namespaces = self._filter_complete_listing(req, namespaces)
|
||||
else:
|
||||
namespaces = None
|
||||
return resp, namespaces
|
||||
|
||||
def _record_shard_listing_cache_metrics(
|
||||
self, cache_state, resp, resp_record_type, info):
|
||||
def _record_shard_listing_cache_metrics(self, cache_state, resp, info):
|
||||
"""
|
||||
Record a single cache operation by shard listing into its
|
||||
corresponding metrics.
|
||||
@ -267,21 +275,19 @@ class ContainerController(Controller):
|
||||
infocache_hit, memcache hit, miss, error, skip, force_skip
|
||||
and disabled.
|
||||
:param resp: the response from either backend or cache hit.
|
||||
:param resp_record_type: indicates the type of response record, e.g.
|
||||
'shard' for shard range listing, 'object' for object listing.
|
||||
:param info: the cached container info.
|
||||
"""
|
||||
should_record = False
|
||||
if is_success(resp.status_int):
|
||||
if resp_record_type == 'shard':
|
||||
# Here we either got shard ranges by hitting the cache, or we
|
||||
# got shard ranges from backend successfully for cache_state
|
||||
if resp.headers.get('X-Backend-Record-Type', '') == 'shard':
|
||||
# Here we either got namespaces by hitting the cache, or we
|
||||
# got namespaces from backend successfully for cache_state
|
||||
# other than cache hit. Note: it's possible that later we find
|
||||
# that shard ranges can't be parsed.
|
||||
# that namespaces can't be parsed.
|
||||
should_record = True
|
||||
elif (info and is_success(info['status'])
|
||||
and info.get('sharding_state') == 'sharded'):
|
||||
# The shard listing request failed when getting shard ranges from
|
||||
# The shard listing request failed when getting namespaces from
|
||||
# backend.
|
||||
# Note: In the absence of 'info' we cannot assume the container is
|
||||
# sharded, so we don't increment the metric if 'info' is None. Even
|
||||
@ -298,34 +304,55 @@ class ContainerController(Controller):
|
||||
self.logger, self.server_type.lower(), 'shard_listing',
|
||||
cache_state, resp)
|
||||
|
||||
def _GET_using_cache(self, req, info):
|
||||
# It may be possible to fulfil the request from cache: we only reach
|
||||
# here if request record_type is 'shard' or 'auto', so if the container
|
||||
# state is 'sharded' then look for cached shard ranges. However, if
|
||||
# X-Newest is true then we always fetch from the backend servers.
|
||||
headers = headers_from_container_info(info)
|
||||
if config_true_value(req.headers.get('x-newest', False)):
|
||||
cache_state = 'force_skip'
|
||||
self.logger.debug(
|
||||
'Skipping shard cache lookup (x-newest) for %s', req.path_qs)
|
||||
elif (headers and info and is_success(info['status']) and
|
||||
info.get('sharding_state') == 'sharded'):
|
||||
# container is sharded so we may have the shard ranges cached; only
|
||||
# use cached values if all required backend headers available.
|
||||
resp, cache_state = self._get_shard_ranges_from_cache(req, headers)
|
||||
if resp:
|
||||
return resp, cache_state
|
||||
def _GET_auto(self, req):
|
||||
# This is an object listing but the backend may be sharded.
|
||||
# Only lookup container info from cache and skip the backend HEAD,
|
||||
# since we are going to GET the backend container anyway.
|
||||
info = get_container_info(
|
||||
req.environ, self.app, swift_source=None, cache_only=True)
|
||||
memcache = cache_from_env(req.environ, True)
|
||||
cache_enabled = self.app.recheck_listing_shard_ranges > 0 and memcache
|
||||
resp = namespaces = None
|
||||
if cache_enabled:
|
||||
# if the container is sharded we may look for namespaces in cache
|
||||
headers = headers_from_container_info(info)
|
||||
if config_true_value(req.headers.get('x-newest', False)):
|
||||
cache_state = 'force_skip'
|
||||
self.logger.debug(
|
||||
'Skipping shard cache lookup (x-newest) for %s',
|
||||
req.path_qs)
|
||||
elif (headers and is_success(info['status']) and
|
||||
info.get('sharding_state') == 'sharded'):
|
||||
# container is sharded so we may have the namespaces cached,
|
||||
# but only use cached namespaces if all required response
|
||||
# headers are also available from cache.
|
||||
resp, namespaces, cache_state = \
|
||||
self._get_listing_namespaces_from_cache(req, headers)
|
||||
else:
|
||||
# container metadata didn't support a cache lookup, this could
|
||||
# be the case that container metadata was not in cache and we
|
||||
# don't know if the container was sharded, or the case that the
|
||||
# sharding state in metadata indicates the container was
|
||||
# unsharded.
|
||||
cache_state = 'bypass'
|
||||
else:
|
||||
# container metadata didn't support a cache lookup, this could be
|
||||
# the case that container metadata was not in cache and we don't
|
||||
# know if the container was sharded, or the case that the sharding
|
||||
# state in metadata indicates the container was unsharded.
|
||||
cache_state = 'bypass'
|
||||
# The request was not fulfilled from cache so send to backend server.
|
||||
return self._get_shard_ranges_from_backend(req), cache_state
|
||||
cache_state = 'disabled'
|
||||
|
||||
def GETorHEAD(self, req):
|
||||
"""Handler for HTTP GET/HEAD requests."""
|
||||
if not namespaces:
|
||||
resp, namespaces = self._get_listing_namespaces_from_backend(
|
||||
req, cache_enabled)
|
||||
self._record_shard_listing_cache_metrics(cache_state, resp, info)
|
||||
|
||||
if namespaces is not None:
|
||||
# we got namespaces, so the container must be sharded; now build
|
||||
# the listing from shards
|
||||
# NB: the filtered namespaces list may be empty but we still need
|
||||
# to build a response body with an empty list of objects
|
||||
resp = self._get_from_shards(req, resp, namespaces)
|
||||
|
||||
return resp
|
||||
|
||||
def _get_or_head_pre_check(self, req):
|
||||
ai = self.account_info(self.account_name, req)
|
||||
auto_account = self.account_name.startswith(
|
||||
self.app.auto_create_account_prefix)
|
||||
@ -339,78 +366,9 @@ class ContainerController(Controller):
|
||||
# Don't cache this. The lack of account will be cached, and that
|
||||
# is sufficient.
|
||||
return HTTPNotFound(request=req)
|
||||
return None
|
||||
|
||||
# The read-modify-write of params here is because the Request.params
|
||||
# getter dynamically generates a dict of params from the query string;
|
||||
# the setter must be called for new params to update the query string.
|
||||
params = req.params
|
||||
params['format'] = 'json'
|
||||
# x-backend-record-type may be sent via internal client e.g. from the
|
||||
# sharder, or by the proxy itself when making a recursive request, or
|
||||
# in probe tests. If the header is present then the only values that
|
||||
# the proxy respects are 'object' or 'shard'. However, the proxy may
|
||||
# use the value 'auto' when making requests to container server.
|
||||
orig_record_type = req.headers.get('X-Backend-Record-Type', '').lower()
|
||||
if orig_record_type in ('object', 'shard'):
|
||||
record_type = orig_record_type
|
||||
else:
|
||||
record_type = 'auto'
|
||||
req.headers['X-Backend-Record-Type'] = 'auto'
|
||||
req.headers['X-Backend-Record-Shard-Format'] = 'namespace'
|
||||
params['states'] = 'listing'
|
||||
req.params = params
|
||||
|
||||
if (req.method == 'GET'
|
||||
and get_param(req, 'states') == 'listing'
|
||||
and record_type != 'object'):
|
||||
may_get_listing_shards = True
|
||||
# Only lookup container info from cache and skip the backend HEAD,
|
||||
# since we are going to GET the backend container anyway.
|
||||
info = get_container_info(
|
||||
req.environ, self.app, swift_source=None, cache_only=True)
|
||||
else:
|
||||
info = None
|
||||
may_get_listing_shards = False
|
||||
|
||||
memcache = cache_from_env(req.environ, True)
|
||||
sr_cache_state = None
|
||||
if (may_get_listing_shards and
|
||||
self.app.recheck_listing_shard_ranges > 0
|
||||
and memcache
|
||||
and not config_true_value(
|
||||
req.headers.get('x-backend-include-deleted', False))):
|
||||
# This GET might be served from cache or might populate cache.
|
||||
# 'x-backend-include-deleted' is not usually expected in requests
|
||||
# to the proxy (it is used from sharder to container servers) but
|
||||
# it is included in the conditions just in case because we don't
|
||||
# cache deleted shard ranges.
|
||||
resp, sr_cache_state = self._GET_using_cache(req, info)
|
||||
else:
|
||||
resp = self._GETorHEAD_from_backend(req)
|
||||
if may_get_listing_shards and (
|
||||
not self.app.recheck_listing_shard_ranges or not memcache):
|
||||
sr_cache_state = 'disabled'
|
||||
|
||||
resp_record_type = resp.headers.get('X-Backend-Record-Type', '')
|
||||
if sr_cache_state:
|
||||
self._record_shard_listing_cache_metrics(
|
||||
sr_cache_state, resp, resp_record_type, info)
|
||||
|
||||
if all((req.method == "GET", record_type == 'auto',
|
||||
resp_record_type.lower() == 'shard')):
|
||||
data = self._parse_listing_response(req, resp)
|
||||
namespaces = self._parse_namespaces(req, data, resp)
|
||||
if namespaces is not None:
|
||||
# we got namespaces, so the container must be sharded; now
|
||||
# build the listing from shards
|
||||
# NB: the filtered namespaces list may be empty but we still
|
||||
# need to build a response body with an empty list of shards
|
||||
resp = self._get_from_shards(req, resp, namespaces)
|
||||
|
||||
if orig_record_type not in ('object', 'shard'):
|
||||
resp.headers.pop('X-Backend-Record-Type', None)
|
||||
resp.headers.pop('X-Backend-Record-Shard-Format', None)
|
||||
|
||||
def _get_or_head_post_check(self, req, resp):
|
||||
if not config_true_value(
|
||||
resp.headers.get('X-Backend-Cached-Results')):
|
||||
# Cache container metadata. We just made a request to a storage
|
||||
@ -419,6 +377,7 @@ class ContainerController(Controller):
|
||||
self.app.recheck_container_existence)
|
||||
set_info_cache(req.environ, self.account_name,
|
||||
self.container_name, resp)
|
||||
|
||||
if 'swift.authorize' in req.environ:
|
||||
req.acl = wsgi_to_str(resp.headers.get('x-container-read'))
|
||||
aresp = req.environ['swift.authorize'](req)
|
||||
@ -437,6 +396,51 @@ class ContainerController(Controller):
|
||||
'False'))
|
||||
return resp
|
||||
|
||||
@public
|
||||
@delay_denial
|
||||
@cors_validation
|
||||
def GET(self, req):
|
||||
"""Handler for HTTP GET requests."""
|
||||
# early checks for request validity
|
||||
validate_container_params(req)
|
||||
aresp = self._get_or_head_pre_check(req)
|
||||
if aresp:
|
||||
return aresp
|
||||
|
||||
# Always request json format from the backend. listing_formats
|
||||
# middleware will take care of what the client gets back.
|
||||
# The read-modify-write of params here is because the
|
||||
# Request.params getter dynamically generates a dict of params from
|
||||
# the query string; the setter must be called for new params to
|
||||
# update the query string.
|
||||
params = req.params
|
||||
params['format'] = 'json'
|
||||
req.params = params
|
||||
|
||||
# x-backend-record-type may be sent via internal client e.g. from
|
||||
# the sharder or in probe tests
|
||||
record_type = req.headers.get('X-Backend-Record-Type', '').lower()
|
||||
if record_type in ('object', 'shard'):
|
||||
# Go direct to the backend for HEADs, and GETs that *explicitly*
|
||||
# specify a record type. We won't be reading/writing namespaces in
|
||||
# cache nor building listings from shards. This path is used by
|
||||
# the sharder, manage_shard_ranges and other tools that fetch shard
|
||||
# ranges, and by the proxy itself when explicitly requesting
|
||||
# objects while recursively building a listing from shards.
|
||||
# Note: shard record type could be namespace or full format
|
||||
resp = self._GETorHEAD_from_backend(req)
|
||||
else:
|
||||
# Requests that do not explicitly specify a record type, or specify
|
||||
# 'auto', default to returning an object listing. The listing may
|
||||
# be built from shards and may involve reading/writing namespaces
|
||||
# in cache. This path is used for client requests and by the proxy
|
||||
# itself while recursively building a listing from shards.
|
||||
resp = self._GET_auto(req)
|
||||
resp.headers.pop('X-Backend-Record-Type', None)
|
||||
resp.headers.pop('X-Backend-Record-Shard-Format', None)
|
||||
|
||||
return self._get_or_head_post_check(req, resp)
|
||||
|
||||
def _get_from_shards(self, req, resp, namespaces):
|
||||
"""
|
||||
Construct an object listing using shards described by the list of
|
||||
@ -527,6 +531,8 @@ class ContainerController(Controller):
|
||||
shard_listing_history):
|
||||
# directed back to same container - force GET of objects
|
||||
headers['X-Backend-Record-Type'] = 'object'
|
||||
else:
|
||||
headers['X-Backend-Record-Type'] = 'auto'
|
||||
if config_true_value(req.headers.get('x-newest', False)):
|
||||
headers['X-Newest'] = 'true'
|
||||
|
||||
@ -611,21 +617,16 @@ class ContainerController(Controller):
|
||||
[o['bytes'] for o in objects])
|
||||
return resp
|
||||
|
||||
@public
|
||||
@delay_denial
|
||||
@cors_validation
|
||||
def GET(self, req):
|
||||
"""Handler for HTTP GET requests."""
|
||||
# early checks for request validity
|
||||
validate_container_params(req)
|
||||
return self.GETorHEAD(req)
|
||||
|
||||
@public
|
||||
@delay_denial
|
||||
@cors_validation
|
||||
def HEAD(self, req):
|
||||
"""Handler for HTTP HEAD requests."""
|
||||
return self.GETorHEAD(req)
|
||||
aresp = self._get_or_head_pre_check(req)
|
||||
if aresp:
|
||||
return aresp
|
||||
resp = self._GETorHEAD_from_backend(req)
|
||||
return self._get_or_head_post_check(req, resp)
|
||||
|
||||
@public
|
||||
@cors_validation
|
||||
|
||||
@ -2998,17 +2998,14 @@ class TestShardedAPI(BaseTestContainerSharding):
|
||||
params={'states': 'updating'})
|
||||
self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges)
|
||||
|
||||
# XXX the states=listing param provokes the proxy to cache the backend
|
||||
# values and then respond to the client with the cached *namespaces* !!
|
||||
# shard_ranges = self.get_container_shard_ranges(
|
||||
# params={'states': 'listing'})
|
||||
# self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges)
|
||||
shard_ranges = self.get_container_shard_ranges(
|
||||
params={'states': 'listing'})
|
||||
self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges)
|
||||
|
||||
# XXX ditto...
|
||||
# shard_ranges = self.get_container_shard_ranges(
|
||||
# headers={'X-Newest': 'true'},
|
||||
# params={'states': 'listing'})
|
||||
# self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges)
|
||||
shard_ranges = self.get_container_shard_ranges(
|
||||
headers={'X-Newest': 'true'},
|
||||
params={'states': 'listing'})
|
||||
self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges)
|
||||
|
||||
# this is what the sharder requests...
|
||||
shard_ranges = self.get_container_shard_ranges(
|
||||
|
||||
@ -7904,6 +7904,15 @@ class TestNamespaceBoundList(unittest.TestCase):
|
||||
self.end_ns = utils.Namespace('a/z-', 'z', '')
|
||||
self.lowerbounds = [start, atof, ftol, ltor, rtoz, end]
|
||||
|
||||
def test_eq(self):
|
||||
this = utils.NamespaceBoundList(self.lowerbounds)
|
||||
that = utils.NamespaceBoundList(self.lowerbounds)
|
||||
self.assertEqual(this, that)
|
||||
that = utils.NamespaceBoundList(self.lowerbounds[:1])
|
||||
self.assertNotEqual(this, that)
|
||||
self.assertNotEqual(this, None)
|
||||
self.assertNotEqual(this, self.lowerbounds)
|
||||
|
||||
def test_get_namespace(self):
|
||||
namespace_list = utils.NamespaceBoundList(self.lowerbounds)
|
||||
self.assertEqual(namespace_list.bounds, self.lowerbounds)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user