Merge "proxy: only use listing shards cache for 'auto' listings"
This commit is contained in:
Zuul
Gerrit Code Review
commit
486fb23447
@ -896,8 +896,10 @@ def get_namespaces_from_cache(req, cache_key, skip_chance):
|
||||
:param req: a :class:`swift.common.swob.Request` object.
|
||||
:param cache_key: the cache key for both infocache and memcache.
|
||||
:param skip_chance: the probability of skipping the memcache look-up.
|
||||
:return: a tuple of
|
||||
(:class:`swift.common.utils.NamespaceBoundList`, cache state)
|
||||
:return: a tuple of (value, cache state). Value is an instance of
|
||||
:class:`swift.common.utils.NamespaceBoundList` if a non-empty list is
|
||||
found in memcache. Otherwise value is ``None``, for example if memcache
|
||||
look-up was skipped, or no value was found, or an empty list was found.
|
||||
"""
|
||||
# try get namespaces from infocache first
|
||||
infocache = req.environ.setdefault('swift.infocache', {})
|
||||
|
||||
@ -108,15 +108,14 @@ class ContainerController(Controller):
|
||||
req.swift_entity_path, concurrency)
|
||||
return resp
|
||||
|
||||
def _make_namespaces_response_body(self, req, ns_bound_list):
|
||||
def _filter_complete_listing(self, req, namespaces):
|
||||
"""
|
||||
Filter namespaces according to request constraints and return a
|
||||
serialised list of namespaces.
|
||||
Filter complete list of namespaces to return only those specified by
|
||||
the request constraints.
|
||||
|
||||
:param req: the request object.
|
||||
:param ns_bound_list: an instance of
|
||||
:class:`~swift.common.utils.NamespaceBoundList`.
|
||||
:return: a serialised list of namespaces.
|
||||
:param req: a :class:`~swift.common.swob.Request`.
|
||||
:param namespaces: a list of :class:`~swift.common.utils.Namespace`.
|
||||
:return: a list of :class:`~swift.common.utils.Namespace`.
|
||||
"""
|
||||
marker = get_param(req, 'marker', '')
|
||||
end_marker = get_param(req, 'end_marker')
|
||||
@ -124,141 +123,150 @@ class ContainerController(Controller):
|
||||
reverse = config_true_value(get_param(req, 'reverse'))
|
||||
if reverse:
|
||||
marker, end_marker = end_marker, marker
|
||||
namespaces = ns_bound_list.get_namespaces()
|
||||
namespaces = filter_namespaces(
|
||||
namespaces, includes, marker, end_marker)
|
||||
if reverse:
|
||||
namespaces.reverse()
|
||||
return json.dumps([dict(ns) for ns in namespaces]).encode('ascii')
|
||||
return namespaces
|
||||
|
||||
def _get_shard_ranges_from_cache(self, req, headers):
|
||||
def _get_listing_namespaces_from_cache(self, req, headers):
|
||||
"""
|
||||
Try to fetch shard namespace data from cache and, if successful, return
|
||||
a response. Also return the cache state.
|
||||
|
||||
The response body will be a list of dicts each of which describes
|
||||
a Namespace (i.e. includes the keys ``lower``, ``upper`` and ``name``).
|
||||
a list of Namespaces. Also return the cache state.
|
||||
|
||||
:param req: an instance of ``swob.Request``.
|
||||
:param headers: Headers to be sent with request.
|
||||
:return: a tuple comprising (an instance of ``swob.Response``or
|
||||
``None`` if no namespaces were found in cache, the cache state).
|
||||
:return: a tuple comprising (a list instance of ``Namespace`` objects
|
||||
or ``None`` if no namespaces were found in cache, the cache state).
|
||||
"""
|
||||
cache_key = get_cache_key(self.account_name, self.container_name,
|
||||
shard='listing')
|
||||
skip_chance = self.app.container_listing_shard_ranges_skip_cache
|
||||
ns_bound_list, cache_state = get_namespaces_from_cache(
|
||||
req, cache_key, skip_chance)
|
||||
if ns_bound_list:
|
||||
# shard ranges can be returned from cache
|
||||
resp_body = self._make_namespaces_response_body(req, ns_bound_list)
|
||||
self.logger.debug('Found %d shards in cache for %s',
|
||||
len(ns_bound_list.bounds), req.path_qs)
|
||||
headers.update({'x-backend-record-type': 'shard',
|
||||
'x-backend-cached-results': 'true'})
|
||||
# mimic GetOrHeadHandler.get_working_response...
|
||||
# note: server sets charset with content_type but proxy
|
||||
# GETorHEAD_base does not, so don't set it here either
|
||||
resp = Response(request=req, body=resp_body)
|
||||
update_headers(resp, headers)
|
||||
resp.last_modified = Timestamp(headers['x-put-timestamp']).ceil()
|
||||
resp.environ['swift_x_timestamp'] = headers.get('x-timestamp')
|
||||
resp.accept_ranges = 'bytes'
|
||||
resp.content_type = 'application/json'
|
||||
else:
|
||||
resp = None
|
||||
if not ns_bound_list:
|
||||
return None, None, cache_state
|
||||
|
||||
return resp, cache_state
|
||||
# Namespaces found in cache so there is no need to go to backend,
|
||||
# but we need to build response headers: mimic
|
||||
# GetOrHeadHandler.get_working_response...
|
||||
# note: server sets charset with content_type but proxy
|
||||
# GETorHEAD_base does not, so don't set it here either
|
||||
namespaces = ns_bound_list.get_namespaces()
|
||||
self.logger.debug('Found %d shards in cache for %s',
|
||||
len(namespaces), req.path_qs)
|
||||
headers.update({'x-backend-record-type': 'shard',
|
||||
'x-backend-record-shard-format': 'namespace',
|
||||
'x-backend-cached-results': 'true'})
|
||||
resp = Response(request=req)
|
||||
update_headers(resp, headers)
|
||||
resp.last_modified = Timestamp(headers['x-put-timestamp']).ceil()
|
||||
resp.environ['swift_x_timestamp'] = headers.get('x-timestamp')
|
||||
resp.accept_ranges = 'bytes'
|
||||
resp.content_type = 'application/json'
|
||||
namespaces = self._filter_complete_listing(req, namespaces)
|
||||
return resp, namespaces, cache_state
|
||||
|
||||
def _store_shard_ranges_in_cache(self, req, resp):
|
||||
def _set_listing_namespaces_in_cache(self, req, namespaces):
|
||||
"""
|
||||
Parse shard ranges returned from backend, store them in both infocache
|
||||
and memcache.
|
||||
Store a list of namespaces in both infocache and memcache.
|
||||
|
||||
Note: the returned list of namespaces may not be identical to the given
|
||||
list. Any gaps in the given namespaces will be 'lost' as a result of
|
||||
compacting the list of namespaces to a NamespaceBoundList for caching.
|
||||
That is ok. When the cached NamespaceBoundList is transformed back to
|
||||
Namespaces to perform a listing, the Namespace before each gap will
|
||||
have expanded to include the gap, which means that the backend GET to
|
||||
that shard will have an end_marker beyond that shard's upper bound, and
|
||||
equal to the next available shard's lower. At worst, some misplaced
|
||||
objects, in the gap above the shard's upper, may be included in the
|
||||
shard's response.
|
||||
|
||||
:param req: the request object.
|
||||
:param resp: the response object for the shard range listing.
|
||||
:return: an instance of
|
||||
:class:`~swift.common.utils.NamespaceBoundList`.
|
||||
:param namespaces: a list of :class:`~swift.common.utils.Namespace`
|
||||
objects.
|
||||
:return: a list of :class:`~swift.common.utils.Namespace` objects.
|
||||
"""
|
||||
# Note: Any gaps in the response's shard ranges will be 'lost' as a
|
||||
# result of compacting the list of shard ranges to a
|
||||
# NamespaceBoundList. That is ok. When the cached NamespaceBoundList is
|
||||
# transformed back to shard range Namespaces to perform a listing, the
|
||||
# Namespace before each gap will have expanded to include the gap,
|
||||
# which means that the backend GET to that shard will have an
|
||||
# end_marker beyond that shard's upper bound, and equal to the next
|
||||
# available shard's lower. At worst, some misplaced objects, in the gap
|
||||
# above the shard's upper, may be included in the shard's response.
|
||||
data = self._parse_listing_response(req, resp)
|
||||
backend_shard_ranges = self._parse_namespaces(req, data, resp)
|
||||
if backend_shard_ranges is None:
|
||||
return None
|
||||
cache_key = get_cache_key(self.account_name, self.container_name,
|
||||
shard='listing')
|
||||
ns_bound_list = NamespaceBoundList.parse(namespaces)
|
||||
# cache in infocache even if no namespaces returned; this
|
||||
# is unexpected but use that result for this request
|
||||
set_cache_state = set_namespaces_in_cache(
|
||||
req, cache_key, ns_bound_list,
|
||||
self.app.recheck_listing_shard_ranges)
|
||||
if set_cache_state == 'set':
|
||||
self.logger.info(
|
||||
'Caching listing namespaces for %s (%d namespaces)',
|
||||
cache_key, len(ns_bound_list.bounds))
|
||||
# return the de-gapped namespaces
|
||||
return ns_bound_list.get_namespaces()
|
||||
|
||||
ns_bound_list = NamespaceBoundList.parse(backend_shard_ranges)
|
||||
if resp.headers.get('x-backend-sharding-state') == 'sharded':
|
||||
# cache in infocache even if no shard ranges returned; this
|
||||
# is unexpected but use that result for this request
|
||||
cache_key = get_cache_key(
|
||||
self.account_name, self.container_name, shard='listing')
|
||||
set_cache_state = set_namespaces_in_cache(
|
||||
req, cache_key, ns_bound_list,
|
||||
self.app.recheck_listing_shard_ranges)
|
||||
if set_cache_state == 'set':
|
||||
self.logger.info(
|
||||
'Caching listing namespaces for %s (%d namespaces)',
|
||||
cache_key, len(ns_bound_list.bounds))
|
||||
return ns_bound_list
|
||||
|
||||
def _get_shard_ranges_from_backend(self, req):
|
||||
def _get_listing_namespaces_from_backend(self, req, cache_enabled):
|
||||
"""
|
||||
Make a backend request for shard ranges and return a response.
|
||||
|
||||
The response body will be a list of dicts each of which describes
|
||||
a Namespace (i.e. includes the keys ``lower``, ``upper`` and ``name``).
|
||||
If the response headers indicate that the response body contains a
|
||||
complete list of shard ranges for a sharded container then the response
|
||||
body will be transformed to a ``NamespaceBoundsList`` and cached.
|
||||
Fetch shard namespace data from the backend and, if successful, return
|
||||
a list of Namespaces.
|
||||
|
||||
:param req: an instance of ``swob.Request``.
|
||||
:return: an instance of ``swob.Response``.
|
||||
:param cache_enabled: a boolean which should be True if memcache is
|
||||
available to cache the returned data, False otherwise.
|
||||
:return: a list instance of ``Namespace`` objects or ``None`` if no
|
||||
namespace data was returned from the backend.
|
||||
"""
|
||||
# Note: We instruct the backend server to ignore name constraints in
|
||||
# request params if returning shard ranges so that the response can
|
||||
# potentially be cached, but we only cache it if the container state is
|
||||
# 'sharded'. We don't attempt to cache shard ranges for a 'sharding'
|
||||
# container as they may include the container itself as a 'gap filler'
|
||||
# for shard ranges that have not yet cleaved; listings from 'gap
|
||||
# filler' shard ranges are likely to become stale as the container
|
||||
# continues to cleave objects to its shards and caching them is
|
||||
# therefore more likely to result in stale or incomplete listings on
|
||||
# subsequent container GETs.
|
||||
req.headers['x-backend-override-shard-name-filter'] = 'sharded'
|
||||
# Instruct the backend server to 'automatically' return namespaces
|
||||
# of shards in a 'listing' state if the container is sharded, and
|
||||
# that the more compact 'namespace' format is sufficient. Older
|
||||
# container servers may still respond with the 'full' shard range
|
||||
# format.
|
||||
req.headers['X-Backend-Record-Type'] = 'auto'
|
||||
req.headers['X-Backend-Record-Shard-Format'] = 'namespace'
|
||||
# 'x-backend-include-deleted' is not expected in 'auto' requests to
|
||||
# the proxy (it's not supported for objects and is used by the
|
||||
# sharder when explicitly fetching 'shard' record type), but we
|
||||
# explicitly set it to false here just in case. A newer container
|
||||
# server would ignore it when returning namespaces, but an older
|
||||
# container server would include unwanted deleted shard range.
|
||||
req.headers['X-Backend-Include-Deleted'] = 'false'
|
||||
params = req.params
|
||||
params['states'] = 'listing'
|
||||
req.params = params
|
||||
if cache_enabled:
|
||||
# Instruct the backend server to ignore name constraints in
|
||||
# request params if returning namespaces so that the response
|
||||
# can potentially be cached, but only if the container state is
|
||||
# 'sharded'. We don't attempt to cache namespaces for a
|
||||
# 'sharding' container as they may include the container itself
|
||||
# as a 'gap filler' for shards that have not yet cleaved;
|
||||
# listings from 'gap filler' namespaces are likely to become
|
||||
# stale as the container continues to cleave objects to its
|
||||
# shards and caching them is therefore more likely to result in
|
||||
# stale or incomplete listings on subsequent container GETs.
|
||||
req.headers['x-backend-override-shard-name-filter'] = 'sharded'
|
||||
resp = self._GETorHEAD_from_backend(req)
|
||||
|
||||
sharding_state = resp.headers.get(
|
||||
'x-backend-sharding-state', '').lower()
|
||||
resp_record_type = resp.headers.get(
|
||||
'x-backend-record-type', '').lower()
|
||||
sharding_state = resp.headers.get(
|
||||
'x-backend-sharding-state', '').lower()
|
||||
complete_listing = config_true_value(resp.headers.pop(
|
||||
'x-backend-override-shard-name-filter', False))
|
||||
# given that we sent 'x-backend-override-shard-name-filter=sharded' we
|
||||
# should only receive back 'x-backend-override-shard-name-filter=true'
|
||||
# if the sharding state is 'sharded', but check them both anyway...
|
||||
if (resp_record_type == 'shard' and
|
||||
sharding_state == 'sharded' and
|
||||
complete_listing):
|
||||
# note: old container servers return a list of shard ranges, newer
|
||||
# ones return a list of namespaces. If we ever need to know we can
|
||||
# look for a 'x-backend-record-shard-format' header from newer
|
||||
# container servers.
|
||||
ns_bound_list = self._store_shard_ranges_in_cache(req, resp)
|
||||
if ns_bound_list:
|
||||
resp.body = self._make_namespaces_response_body(
|
||||
req, ns_bound_list)
|
||||
return resp
|
||||
if resp_record_type == 'shard':
|
||||
data = self._parse_listing_response(req, resp)
|
||||
namespaces = self._parse_namespaces(req, data, resp)
|
||||
# given that we sent
|
||||
# 'x-backend-override-shard-name-filter=sharded' we should only
|
||||
# receive back 'x-backend-override-shard-name-filter=true' if
|
||||
# the sharding state is 'sharded', but check them both
|
||||
# anyway...
|
||||
if (namespaces and
|
||||
sharding_state == 'sharded' and
|
||||
complete_listing):
|
||||
namespaces = self._set_listing_namespaces_in_cache(
|
||||
req, namespaces)
|
||||
namespaces = self._filter_complete_listing(req, namespaces)
|
||||
else:
|
||||
namespaces = None
|
||||
return resp, namespaces
|
||||
|
||||
def _record_shard_listing_cache_metrics(
|
||||
self, cache_state, resp, resp_record_type, info):
|
||||
def _record_shard_listing_cache_metrics(self, cache_state, resp, info):
|
||||
"""
|
||||
Record a single cache operation by shard listing into its
|
||||
corresponding metrics.
|
||||
@ -267,21 +275,19 @@ class ContainerController(Controller):
|
||||
infocache_hit, memcache hit, miss, error, skip, force_skip
|
||||
and disabled.
|
||||
:param resp: the response from either backend or cache hit.
|
||||
:param resp_record_type: indicates the type of response record, e.g.
|
||||
'shard' for shard range listing, 'object' for object listing.
|
||||
:param info: the cached container info.
|
||||
"""
|
||||
should_record = False
|
||||
if is_success(resp.status_int):
|
||||
if resp_record_type == 'shard':
|
||||
# Here we either got shard ranges by hitting the cache, or we
|
||||
# got shard ranges from backend successfully for cache_state
|
||||
if resp.headers.get('X-Backend-Record-Type', '') == 'shard':
|
||||
# Here we either got namespaces by hitting the cache, or we
|
||||
# got namespaces from backend successfully for cache_state
|
||||
# other than cache hit. Note: it's possible that later we find
|
||||
# that shard ranges can't be parsed.
|
||||
# that namespaces can't be parsed.
|
||||
should_record = True
|
||||
elif (info and is_success(info['status'])
|
||||
and info.get('sharding_state') == 'sharded'):
|
||||
# The shard listing request failed when getting shard ranges from
|
||||
# The shard listing request failed when getting namespaces from
|
||||
# backend.
|
||||
# Note: In the absence of 'info' we cannot assume the container is
|
||||
# sharded, so we don't increment the metric if 'info' is None. Even
|
||||
@ -298,34 +304,55 @@ class ContainerController(Controller):
|
||||
self.logger, self.server_type.lower(), 'shard_listing',
|
||||
cache_state, resp)
|
||||
|
||||
def _GET_using_cache(self, req, info):
|
||||
# It may be possible to fulfil the request from cache: we only reach
|
||||
# here if request record_type is 'shard' or 'auto', so if the container
|
||||
# state is 'sharded' then look for cached shard ranges. However, if
|
||||
# X-Newest is true then we always fetch from the backend servers.
|
||||
headers = headers_from_container_info(info)
|
||||
if config_true_value(req.headers.get('x-newest', False)):
|
||||
cache_state = 'force_skip'
|
||||
self.logger.debug(
|
||||
'Skipping shard cache lookup (x-newest) for %s', req.path_qs)
|
||||
elif (headers and info and is_success(info['status']) and
|
||||
info.get('sharding_state') == 'sharded'):
|
||||
# container is sharded so we may have the shard ranges cached; only
|
||||
# use cached values if all required backend headers available.
|
||||
resp, cache_state = self._get_shard_ranges_from_cache(req, headers)
|
||||
if resp:
|
||||
return resp, cache_state
|
||||
def _GET_auto(self, req):
|
||||
# This is an object listing but the backend may be sharded.
|
||||
# Only lookup container info from cache and skip the backend HEAD,
|
||||
# since we are going to GET the backend container anyway.
|
||||
info = get_container_info(
|
||||
req.environ, self.app, swift_source=None, cache_only=True)
|
||||
memcache = cache_from_env(req.environ, True)
|
||||
cache_enabled = self.app.recheck_listing_shard_ranges > 0 and memcache
|
||||
resp = namespaces = None
|
||||
if cache_enabled:
|
||||
# if the container is sharded we may look for namespaces in cache
|
||||
headers = headers_from_container_info(info)
|
||||
if config_true_value(req.headers.get('x-newest', False)):
|
||||
cache_state = 'force_skip'
|
||||
self.logger.debug(
|
||||
'Skipping shard cache lookup (x-newest) for %s',
|
||||
req.path_qs)
|
||||
elif (headers and is_success(info['status']) and
|
||||
info.get('sharding_state') == 'sharded'):
|
||||
# container is sharded so we may have the namespaces cached,
|
||||
# but only use cached namespaces if all required response
|
||||
# headers are also available from cache.
|
||||
resp, namespaces, cache_state = \
|
||||
self._get_listing_namespaces_from_cache(req, headers)
|
||||
else:
|
||||
# container metadata didn't support a cache lookup, this could
|
||||
# be the case that container metadata was not in cache and we
|
||||
# don't know if the container was sharded, or the case that the
|
||||
# sharding state in metadata indicates the container was
|
||||
# unsharded.
|
||||
cache_state = 'bypass'
|
||||
else:
|
||||
# container metadata didn't support a cache lookup, this could be
|
||||
# the case that container metadata was not in cache and we don't
|
||||
# know if the container was sharded, or the case that the sharding
|
||||
# state in metadata indicates the container was unsharded.
|
||||
cache_state = 'bypass'
|
||||
# The request was not fulfilled from cache so send to backend server.
|
||||
return self._get_shard_ranges_from_backend(req), cache_state
|
||||
cache_state = 'disabled'
|
||||
|
||||
def GETorHEAD(self, req):
|
||||
"""Handler for HTTP GET/HEAD requests."""
|
||||
if not namespaces:
|
||||
resp, namespaces = self._get_listing_namespaces_from_backend(
|
||||
req, cache_enabled)
|
||||
self._record_shard_listing_cache_metrics(cache_state, resp, info)
|
||||
|
||||
if namespaces is not None:
|
||||
# we got namespaces, so the container must be sharded; now build
|
||||
# the listing from shards
|
||||
# NB: the filtered namespaces list may be empty but we still need
|
||||
# to build a response body with an empty list of objects
|
||||
resp = self._get_from_shards(req, resp, namespaces)
|
||||
|
||||
return resp
|
||||
|
||||
def _get_or_head_pre_check(self, req):
|
||||
ai = self.account_info(self.account_name, req)
|
||||
auto_account = self.account_name.startswith(
|
||||
self.app.auto_create_account_prefix)
|
||||
@ -339,78 +366,9 @@ class ContainerController(Controller):
|
||||
# Don't cache this. The lack of account will be cached, and that
|
||||
# is sufficient.
|
||||
return HTTPNotFound(request=req)
|
||||
return None
|
||||
|
||||
# The read-modify-write of params here is because the Request.params
|
||||
# getter dynamically generates a dict of params from the query string;
|
||||
# the setter must be called for new params to update the query string.
|
||||
params = req.params
|
||||
params['format'] = 'json'
|
||||
# x-backend-record-type may be sent via internal client e.g. from the
|
||||
# sharder, or by the proxy itself when making a recursive request, or
|
||||
# in probe tests. If the header is present then the only values that
|
||||
# the proxy respects are 'object' or 'shard'. However, the proxy may
|
||||
# use the value 'auto' when making requests to container server.
|
||||
orig_record_type = req.headers.get('X-Backend-Record-Type', '').lower()
|
||||
if orig_record_type in ('object', 'shard'):
|
||||
record_type = orig_record_type
|
||||
else:
|
||||
record_type = 'auto'
|
||||
req.headers['X-Backend-Record-Type'] = 'auto'
|
||||
req.headers['X-Backend-Record-Shard-Format'] = 'namespace'
|
||||
params['states'] = 'listing'
|
||||
req.params = params
|
||||
|
||||
if (req.method == 'GET'
|
||||
and get_param(req, 'states') == 'listing'
|
||||
and record_type != 'object'):
|
||||
may_get_listing_shards = True
|
||||
# Only lookup container info from cache and skip the backend HEAD,
|
||||
# since we are going to GET the backend container anyway.
|
||||
info = get_container_info(
|
||||
req.environ, self.app, swift_source=None, cache_only=True)
|
||||
else:
|
||||
info = None
|
||||
may_get_listing_shards = False
|
||||
|
||||
memcache = cache_from_env(req.environ, True)
|
||||
sr_cache_state = None
|
||||
if (may_get_listing_shards and
|
||||
self.app.recheck_listing_shard_ranges > 0
|
||||
and memcache
|
||||
and not config_true_value(
|
||||
req.headers.get('x-backend-include-deleted', False))):
|
||||
# This GET might be served from cache or might populate cache.
|
||||
# 'x-backend-include-deleted' is not usually expected in requests
|
||||
# to the proxy (it is used from sharder to container servers) but
|
||||
# it is included in the conditions just in case because we don't
|
||||
# cache deleted shard ranges.
|
||||
resp, sr_cache_state = self._GET_using_cache(req, info)
|
||||
else:
|
||||
resp = self._GETorHEAD_from_backend(req)
|
||||
if may_get_listing_shards and (
|
||||
not self.app.recheck_listing_shard_ranges or not memcache):
|
||||
sr_cache_state = 'disabled'
|
||||
|
||||
resp_record_type = resp.headers.get('X-Backend-Record-Type', '')
|
||||
if sr_cache_state:
|
||||
self._record_shard_listing_cache_metrics(
|
||||
sr_cache_state, resp, resp_record_type, info)
|
||||
|
||||
if all((req.method == "GET", record_type == 'auto',
|
||||
resp_record_type.lower() == 'shard')):
|
||||
data = self._parse_listing_response(req, resp)
|
||||
namespaces = self._parse_namespaces(req, data, resp)
|
||||
if namespaces is not None:
|
||||
# we got namespaces, so the container must be sharded; now
|
||||
# build the listing from shards
|
||||
# NB: the filtered namespaces list may be empty but we still
|
||||
# need to build a response body with an empty list of shards
|
||||
resp = self._get_from_shards(req, resp, namespaces)
|
||||
|
||||
if orig_record_type not in ('object', 'shard'):
|
||||
resp.headers.pop('X-Backend-Record-Type', None)
|
||||
resp.headers.pop('X-Backend-Record-Shard-Format', None)
|
||||
|
||||
def _get_or_head_post_check(self, req, resp):
|
||||
if not config_true_value(
|
||||
resp.headers.get('X-Backend-Cached-Results')):
|
||||
# Cache container metadata. We just made a request to a storage
|
||||
@ -419,6 +377,7 @@ class ContainerController(Controller):
|
||||
self.app.recheck_container_existence)
|
||||
set_info_cache(req.environ, self.account_name,
|
||||
self.container_name, resp)
|
||||
|
||||
if 'swift.authorize' in req.environ:
|
||||
req.acl = wsgi_to_str(resp.headers.get('x-container-read'))
|
||||
aresp = req.environ['swift.authorize'](req)
|
||||
@ -437,6 +396,51 @@ class ContainerController(Controller):
|
||||
'False'))
|
||||
return resp
|
||||
|
||||
@public
|
||||
@delay_denial
|
||||
@cors_validation
|
||||
def GET(self, req):
|
||||
"""Handler for HTTP GET requests."""
|
||||
# early checks for request validity
|
||||
validate_container_params(req)
|
||||
aresp = self._get_or_head_pre_check(req)
|
||||
if aresp:
|
||||
return aresp
|
||||
|
||||
# Always request json format from the backend. listing_formats
|
||||
# middleware will take care of what the client gets back.
|
||||
# The read-modify-write of params here is because the
|
||||
# Request.params getter dynamically generates a dict of params from
|
||||
# the query string; the setter must be called for new params to
|
||||
# update the query string.
|
||||
params = req.params
|
||||
params['format'] = 'json'
|
||||
req.params = params
|
||||
|
||||
# x-backend-record-type may be sent via internal client e.g. from
|
||||
# the sharder or in probe tests
|
||||
record_type = req.headers.get('X-Backend-Record-Type', '').lower()
|
||||
if record_type in ('object', 'shard'):
|
||||
# Go direct to the backend for HEADs, and GETs that *explicitly*
|
||||
# specify a record type. We won't be reading/writing namespaces in
|
||||
# cache nor building listings from shards. This path is used by
|
||||
# the sharder, manage_shard_ranges and other tools that fetch shard
|
||||
# ranges, and by the proxy itself when explicitly requesting
|
||||
# objects while recursively building a listing from shards.
|
||||
# Note: shard record type could be namespace or full format
|
||||
resp = self._GETorHEAD_from_backend(req)
|
||||
else:
|
||||
# Requests that do not explicitly specify a record type, or specify
|
||||
# 'auto', default to returning an object listing. The listing may
|
||||
# be built from shards and may involve reading/writing namespaces
|
||||
# in cache. This path is used for client requests and by the proxy
|
||||
# itself while recursively building a listing from shards.
|
||||
resp = self._GET_auto(req)
|
||||
resp.headers.pop('X-Backend-Record-Type', None)
|
||||
resp.headers.pop('X-Backend-Record-Shard-Format', None)
|
||||
|
||||
return self._get_or_head_post_check(req, resp)
|
||||
|
||||
def _get_from_shards(self, req, resp, namespaces):
|
||||
"""
|
||||
Construct an object listing using shards described by the list of
|
||||
@ -527,6 +531,8 @@ class ContainerController(Controller):
|
||||
shard_listing_history):
|
||||
# directed back to same container - force GET of objects
|
||||
headers['X-Backend-Record-Type'] = 'object'
|
||||
else:
|
||||
headers['X-Backend-Record-Type'] = 'auto'
|
||||
if config_true_value(req.headers.get('x-newest', False)):
|
||||
headers['X-Newest'] = 'true'
|
||||
|
||||
@ -611,21 +617,16 @@ class ContainerController(Controller):
|
||||
[o['bytes'] for o in objects])
|
||||
return resp
|
||||
|
||||
@public
|
||||
@delay_denial
|
||||
@cors_validation
|
||||
def GET(self, req):
|
||||
"""Handler for HTTP GET requests."""
|
||||
# early checks for request validity
|
||||
validate_container_params(req)
|
||||
return self.GETorHEAD(req)
|
||||
|
||||
@public
|
||||
@delay_denial
|
||||
@cors_validation
|
||||
def HEAD(self, req):
|
||||
"""Handler for HTTP HEAD requests."""
|
||||
return self.GETorHEAD(req)
|
||||
aresp = self._get_or_head_pre_check(req)
|
||||
if aresp:
|
||||
return aresp
|
||||
resp = self._GETorHEAD_from_backend(req)
|
||||
return self._get_or_head_post_check(req, resp)
|
||||
|
||||
@public
|
||||
@cors_validation
|
||||
|
||||
@ -2998,17 +2998,14 @@ class TestShardedAPI(BaseTestContainerSharding):
|
||||
params={'states': 'updating'})
|
||||
self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges)
|
||||
|
||||
# XXX the states=listing param provokes the proxy to cache the backend
|
||||
# values and then respond to the client with the cached *namespaces* !!
|
||||
# shard_ranges = self.get_container_shard_ranges(
|
||||
# params={'states': 'listing'})
|
||||
# self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges)
|
||||
shard_ranges = self.get_container_shard_ranges(
|
||||
params={'states': 'listing'})
|
||||
self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges)
|
||||
|
||||
# XXX ditto...
|
||||
# shard_ranges = self.get_container_shard_ranges(
|
||||
# headers={'X-Newest': 'true'},
|
||||
# params={'states': 'listing'})
|
||||
# self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges)
|
||||
shard_ranges = self.get_container_shard_ranges(
|
||||
headers={'X-Newest': 'true'},
|
||||
params={'states': 'listing'})
|
||||
self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges)
|
||||
|
||||
# this is what the sharder requests...
|
||||
shard_ranges = self.get_container_shard_ranges(
|
||||
|
||||
@ -7904,6 +7904,15 @@ class TestNamespaceBoundList(unittest.TestCase):
|
||||
self.end_ns = utils.Namespace('a/z-', 'z', '')
|
||||
self.lowerbounds = [start, atof, ftol, ltor, rtoz, end]
|
||||
|
||||
def test_eq(self):
|
||||
this = utils.NamespaceBoundList(self.lowerbounds)
|
||||
that = utils.NamespaceBoundList(self.lowerbounds)
|
||||
self.assertEqual(this, that)
|
||||
that = utils.NamespaceBoundList(self.lowerbounds[:1])
|
||||
self.assertNotEqual(this, that)
|
||||
self.assertNotEqual(this, None)
|
||||
self.assertNotEqual(this, self.lowerbounds)
|
||||
|
||||
def test_get_namespace(self):
|
||||
namespace_list = utils.NamespaceBoundList(self.lowerbounds)
|
||||
self.assertEqual(namespace_list.bounds, self.lowerbounds)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user