openstack/swift
Code Issues Proposed changes

proxy: only use listing shards cache for 'auto' listings

Browse Source 
The proxy should NOT read or write to memcache when handling a
container GET that explicitly requests 'shard' or 'object' record
type. A request for 'shard' record type may specify 'namespace'
format, but this request is unrelated to container listings or object
updates and passes directly to the backend.

This patch also removes unnecessary JSON serialisation and
de-serialisation of namespaces within the proxy GET path when a
sharded object listing is being built. The final response body will
contain a list of objects so there is no need to write intermediate
response bodies with a list of namespaces.

Requests that explicitly specify record type of 'shard' will of
course still have the response body with serialised shard dicts that
is returned from the backend.

Change-Id: Id79c156432350c11c52a4004d69b85e9eb904ca6
This commit is contained in:
Alistair Coles 2023-11-17 15:03:59 +00:00
parent 03b66c94f4
commit 252f0d36b7
5 changed files with 1229 additions and 700 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
swift/proxy/controllers/base.py
View File

@ -896,8 +896,10 @@ def get_namespaces_from_cache(req, cache_key, skip_chance):
:param req: a :class:`swift.common.swob.Request` object. :param req: a :class:`swift.common.swob.Request` object.
:param cache_key: the cache key for both infocache and memcache. :param cache_key: the cache key for both infocache and memcache.
:param skip_chance: the probability of skipping the memcache look-up. :param skip_chance: the probability of skipping the memcache look-up.
:return: a tuple of :return: a tuple of (value, cache state). Value is an instance of
(:class:`swift.common.utils.NamespaceBoundList`, cache state) :class:`swift.common.utils.NamespaceBoundList` if a non-empty list is
found in memcache. Otherwise value is ``None``, for example if memcache
look-up was skipped, or no value was found, or an empty list was found.
""" """
# try get namespaces from infocache first # try get namespaces from infocache first
infocache = req.environ.setdefault('swift.infocache', {}) infocache = req.environ.setdefault('swift.infocache', {})

449
swift/proxy/controllers/container.py
View File

@ -108,15 +108,14 @@ class ContainerController(Controller):
req.swift_entity_path, concurrency) req.swift_entity_path, concurrency)
return resp return resp
def _make_namespaces_response_body(self, req, ns_bound_list): def _filter_complete_listing(self, req, namespaces):
""" """
Filter namespaces according to request constraints and return a Filter complete list of namespaces to return only those specified by
serialised list of namespaces. the request constraints.
:param req: the request object. :param req: a :class:`~swift.common.swob.Request`.
:param ns_bound_list: an instance of :param namespaces: a list of :class:`~swift.common.utils.Namespace`.
:class:`~swift.common.utils.NamespaceBoundList`. :return: a list of :class:`~swift.common.utils.Namespace`.
:return: a serialised list of namespaces.
""" """
marker = get_param(req, 'marker', '') marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker') end_marker = get_param(req, 'end_marker')
@ -124,141 +123,150 @@ class ContainerController(Controller):
reverse = config_true_value(get_param(req, 'reverse')) reverse = config_true_value(get_param(req, 'reverse'))
if reverse: if reverse:
marker, end_marker = end_marker, marker marker, end_marker = end_marker, marker
namespaces = ns_bound_list.get_namespaces()
namespaces = filter_namespaces( namespaces = filter_namespaces(
namespaces, includes, marker, end_marker) namespaces, includes, marker, end_marker)
if reverse: if reverse:
namespaces.reverse() namespaces.reverse()
return json.dumps([dict(ns) for ns in namespaces]).encode('ascii') return namespaces
def _get_shard_ranges_from_cache(self, req, headers): def _get_listing_namespaces_from_cache(self, req, headers):
""" """
Try to fetch shard namespace data from cache and, if successful, return Try to fetch shard namespace data from cache and, if successful, return
a response. Also return the cache state. a list of Namespaces. Also return the cache state.
The response body will be a list of dicts each of which describes
a Namespace (i.e. includes the keys ``lower``, ``upper`` and ``name``).
:param req: an instance of ``swob.Request``. :param req: an instance of ``swob.Request``.
:param headers: Headers to be sent with request. :return: a tuple comprising (a list instance of ``Namespace`` objects
:return: a tuple comprising (an instance of ``swob.Response``or or ``None`` if no namespaces were found in cache, the cache state).
``None`` if no namespaces were found in cache, the cache state).
""" """
cache_key = get_cache_key(self.account_name, self.container_name, cache_key = get_cache_key(self.account_name, self.container_name,
shard='listing') shard='listing')
skip_chance = self.app.container_listing_shard_ranges_skip_cache skip_chance = self.app.container_listing_shard_ranges_skip_cache
ns_bound_list, cache_state = get_namespaces_from_cache( ns_bound_list, cache_state = get_namespaces_from_cache(
req, cache_key, skip_chance) req, cache_key, skip_chance)
if ns_bound_list: if not ns_bound_list:
# shard ranges can be returned from cache return None, None, cache_state
resp_body = self._make_namespaces_response_body(req, ns_bound_list)
self.logger.debug('Found %d shards in cache for %s',
len(ns_bound_list.bounds), req.path_qs)
headers.update({'x-backend-record-type': 'shard',
'x-backend-cached-results': 'true'})
# mimic GetOrHeadHandler.get_working_response...
# note: server sets charset with content_type but proxy
# GETorHEAD_base does not, so don't set it here either
resp = Response(request=req, body=resp_body)
update_headers(resp, headers)
resp.last_modified = Timestamp(headers['x-put-timestamp']).ceil()
resp.environ['swift_x_timestamp'] = headers.get('x-timestamp')
resp.accept_ranges = 'bytes'
resp.content_type = 'application/json'
else:
resp = None
return resp, cache_state # Namespaces found in cache so there is no need to go to backend,
# but we need to build response headers: mimic
# GetOrHeadHandler.get_working_response...
# note: server sets charset with content_type but proxy
# GETorHEAD_base does not, so don't set it here either
namespaces = ns_bound_list.get_namespaces()
self.logger.debug('Found %d shards in cache for %s',
len(namespaces), req.path_qs)
headers.update({'x-backend-record-type': 'shard',
'x-backend-record-shard-format': 'namespace',
'x-backend-cached-results': 'true'})
resp = Response(request=req)
update_headers(resp, headers)
resp.last_modified = Timestamp(headers['x-put-timestamp']).ceil()
resp.environ['swift_x_timestamp'] = headers.get('x-timestamp')
resp.accept_ranges = 'bytes'
resp.content_type = 'application/json'
namespaces = self._filter_complete_listing(req, namespaces)
return resp, namespaces, cache_state
def _store_shard_ranges_in_cache(self, req, resp): def _set_listing_namespaces_in_cache(self, req, namespaces):
""" """
Parse shard ranges returned from backend, store them in both infocache Store a list of namespaces in both infocache and memcache.
and memcache.
Note: the returned list of namespaces may not be identical to the given
list. Any gaps in the given namespaces will be 'lost' as a result of
compacting the list of namespaces to a NamespaceBoundList for caching.
That is ok. When the cached NamespaceBoundList is transformed back to
Namespaces to perform a listing, the Namespace before each gap will
have expanded to include the gap, which means that the backend GET to
that shard will have an end_marker beyond that shard's upper bound, and
equal to the next available shard's lower. At worst, some misplaced
objects, in the gap above the shard's upper, may be included in the
shard's response.
:param req: the request object. :param req: the request object.
:param resp: the response object for the shard range listing. :param namespaces: a list of :class:`~swift.common.utils.Namespace`
:return: an instance of objects.
:class:`~swift.common.utils.NamespaceBoundList`. :return: a list of :class:`~swift.common.utils.Namespace` objects.
""" """
# Note: Any gaps in the response's shard ranges will be 'lost' as a cache_key = get_cache_key(self.account_name, self.container_name,
# result of compacting the list of shard ranges to a shard='listing')
# NamespaceBoundList. That is ok. When the cached NamespaceBoundList is ns_bound_list = NamespaceBoundList.parse(namespaces)
# transformed back to shard range Namespaces to perform a listing, the # cache in infocache even if no namespaces returned; this
# Namespace before each gap will have expanded to include the gap, # is unexpected but use that result for this request
# which means that the backend GET to that shard will have an set_cache_state = set_namespaces_in_cache(
# end_marker beyond that shard's upper bound, and equal to the next req, cache_key, ns_bound_list,
# available shard's lower. At worst, some misplaced objects, in the gap self.app.recheck_listing_shard_ranges)
# above the shard's upper, may be included in the shard's response. if set_cache_state == 'set':
data = self._parse_listing_response(req, resp) self.logger.info(
backend_shard_ranges = self._parse_namespaces(req, data, resp) 'Caching listing namespaces for %s (%d namespaces)',
if backend_shard_ranges is None: cache_key, len(ns_bound_list.bounds))
return None # return the de-gapped namespaces
return ns_bound_list.get_namespaces()
ns_bound_list = NamespaceBoundList.parse(backend_shard_ranges) def _get_listing_namespaces_from_backend(self, req, cache_enabled):
if resp.headers.get('x-backend-sharding-state') == 'sharded':
# cache in infocache even if no shard ranges returned; this
# is unexpected but use that result for this request
cache_key = get_cache_key(
self.account_name, self.container_name, shard='listing')
set_cache_state = set_namespaces_in_cache(
req, cache_key, ns_bound_list,
self.app.recheck_listing_shard_ranges)
if set_cache_state == 'set':
self.logger.info(
'Caching listing namespaces for %s (%d namespaces)',
cache_key, len(ns_bound_list.bounds))
return ns_bound_list
def _get_shard_ranges_from_backend(self, req):
""" """
Make a backend request for shard ranges and return a response. Fetch shard namespace data from the backend and, if successful, return
a list of Namespaces.
The response body will be a list of dicts each of which describes
a Namespace (i.e. includes the keys ``lower``, ``upper`` and ``name``).
If the response headers indicate that the response body contains a
complete list of shard ranges for a sharded container then the response
body will be transformed to a ``NamespaceBoundsList`` and cached.
:param req: an instance of ``swob.Request``. :param req: an instance of ``swob.Request``.
:return: an instance of ``swob.Response``. :param cache_enabled: a boolean which should be True if memcache is
available to cache the returned data, False otherwise.
:return: a list instance of ``Namespace`` objects or ``None`` if no
namespace data was returned from the backend.
""" """
# Note: We instruct the backend server to ignore name constraints in # Instruct the backend server to 'automatically' return namespaces
# request params if returning shard ranges so that the response can # of shards in a 'listing' state if the container is sharded, and
# potentially be cached, but we only cache it if the container state is # that the more compact 'namespace' format is sufficient. Older
# 'sharded'. We don't attempt to cache shard ranges for a 'sharding' # container servers may still respond with the 'full' shard range
# container as they may include the container itself as a 'gap filler' # format.
# for shard ranges that have not yet cleaved; listings from 'gap req.headers['X-Backend-Record-Type'] = 'auto'
# filler' shard ranges are likely to become stale as the container req.headers['X-Backend-Record-Shard-Format'] = 'namespace'
# continues to cleave objects to its shards and caching them is # 'x-backend-include-deleted' is not expected in 'auto' requests to
# therefore more likely to result in stale or incomplete listings on # the proxy (it's not supported for objects and is used by the
# subsequent container GETs. # sharder when explicitly fetching 'shard' record type), but we
req.headers['x-backend-override-shard-name-filter'] = 'sharded' # explicitly set it to false here just in case. A newer container
# server would ignore it when returning namespaces, but an older
# container server would include unwanted deleted shard range.
req.headers['X-Backend-Include-Deleted'] = 'false'
params = req.params
params['states'] = 'listing'
req.params = params
if cache_enabled:
# Instruct the backend server to ignore name constraints in
# request params if returning namespaces so that the response
# can potentially be cached, but only if the container state is
# 'sharded'. We don't attempt to cache namespaces for a
# 'sharding' container as they may include the container itself
# as a 'gap filler' for shards that have not yet cleaved;
# listings from 'gap filler' namespaces are likely to become
# stale as the container continues to cleave objects to its
# shards and caching them is therefore more likely to result in
# stale or incomplete listings on subsequent container GETs.
req.headers['x-backend-override-shard-name-filter'] = 'sharded'
resp = self._GETorHEAD_from_backend(req) resp = self._GETorHEAD_from_backend(req)
sharding_state = resp.headers.get(
'x-backend-sharding-state', '').lower()
resp_record_type = resp.headers.get( resp_record_type = resp.headers.get(
'x-backend-record-type', '').lower() 'x-backend-record-type', '').lower()
sharding_state = resp.headers.get(
'x-backend-sharding-state', '').lower()
complete_listing = config_true_value(resp.headers.pop( complete_listing = config_true_value(resp.headers.pop(
'x-backend-override-shard-name-filter', False)) 'x-backend-override-shard-name-filter', False))
# given that we sent 'x-backend-override-shard-name-filter=sharded' we if resp_record_type == 'shard':
# should only receive back 'x-backend-override-shard-name-filter=true' data = self._parse_listing_response(req, resp)
# if the sharding state is 'sharded', but check them both anyway... namespaces = self._parse_namespaces(req, data, resp)
if (resp_record_type == 'shard' and # given that we sent
sharding_state == 'sharded' and # 'x-backend-override-shard-name-filter=sharded' we should only
complete_listing): # receive back 'x-backend-override-shard-name-filter=true' if
# note: old container servers return a list of shard ranges, newer # the sharding state is 'sharded', but check them both
# ones return a list of namespaces. If we ever need to know we can # anyway...
# look for a 'x-backend-record-shard-format' header from newer if (namespaces and
# container servers. sharding_state == 'sharded' and
ns_bound_list = self._store_shard_ranges_in_cache(req, resp) complete_listing):
if ns_bound_list: namespaces = self._set_listing_namespaces_in_cache(
resp.body = self._make_namespaces_response_body( req, namespaces)
req, ns_bound_list) namespaces = self._filter_complete_listing(req, namespaces)
return resp else:
namespaces = None
return resp, namespaces
def _record_shard_listing_cache_metrics( def _record_shard_listing_cache_metrics(self, cache_state, resp, info):
self, cache_state, resp, resp_record_type, info):
""" """
Record a single cache operation by shard listing into its Record a single cache operation by shard listing into its
corresponding metrics. corresponding metrics.
@ -267,21 +275,19 @@ class ContainerController(Controller):
infocache_hit, memcache hit, miss, error, skip, force_skip infocache_hit, memcache hit, miss, error, skip, force_skip
and disabled. and disabled.
:param resp: the response from either backend or cache hit. :param resp: the response from either backend or cache hit.
:param resp_record_type: indicates the type of response record, e.g.
'shard' for shard range listing, 'object' for object listing.
:param info: the cached container info. :param info: the cached container info.
""" """
should_record = False should_record = False
if is_success(resp.status_int): if is_success(resp.status_int):
if resp_record_type == 'shard': if resp.headers.get('X-Backend-Record-Type', '') == 'shard':
# Here we either got shard ranges by hitting the cache, or we # Here we either got namespaces by hitting the cache, or we
# got shard ranges from backend successfully for cache_state # got namespaces from backend successfully for cache_state
# other than cache hit. Note: it's possible that later we find # other than cache hit. Note: it's possible that later we find
# that shard ranges can't be parsed. # that namespaces can't be parsed.
should_record = True should_record = True
elif (info and is_success(info['status']) elif (info and is_success(info['status'])
and info.get('sharding_state') == 'sharded'): and info.get('sharding_state') == 'sharded'):
# The shard listing request failed when getting shard ranges from # The shard listing request failed when getting namespaces from
# backend. # backend.
# Note: In the absence of 'info' we cannot assume the container is # Note: In the absence of 'info' we cannot assume the container is
# sharded, so we don't increment the metric if 'info' is None. Even # sharded, so we don't increment the metric if 'info' is None. Even
@ -298,34 +304,55 @@ class ContainerController(Controller):
self.logger, self.server_type.lower(), 'shard_listing', self.logger, self.server_type.lower(), 'shard_listing',
cache_state, resp) cache_state, resp)
def _GET_using_cache(self, req, info): def _GET_auto(self, req):
# It may be possible to fulfil the request from cache: we only reach # This is an object listing but the backend may be sharded.
# here if request record_type is 'shard' or 'auto', so if the container # Only lookup container info from cache and skip the backend HEAD,
# state is 'sharded' then look for cached shard ranges. However, if # since we are going to GET the backend container anyway.
# X-Newest is true then we always fetch from the backend servers. info = get_container_info(
headers = headers_from_container_info(info) req.environ, self.app, swift_source=None, cache_only=True)
if config_true_value(req.headers.get('x-newest', False)): memcache = cache_from_env(req.environ, True)
cache_state = 'force_skip' cache_enabled = self.app.recheck_listing_shard_ranges > 0 and memcache
self.logger.debug( resp = namespaces = None
'Skipping shard cache lookup (x-newest) for %s', req.path_qs) if cache_enabled:
elif (headers and info and is_success(info['status']) and # if the container is sharded we may look for namespaces in cache
info.get('sharding_state') == 'sharded'): headers = headers_from_container_info(info)
# container is sharded so we may have the shard ranges cached; only if config_true_value(req.headers.get('x-newest', False)):
# use cached values if all required backend headers available. cache_state = 'force_skip'
resp, cache_state = self._get_shard_ranges_from_cache(req, headers) self.logger.debug(
if resp: 'Skipping shard cache lookup (x-newest) for %s',
return resp, cache_state req.path_qs)
elif (headers and is_success(info['status']) and
info.get('sharding_state') == 'sharded'):
# container is sharded so we may have the namespaces cached,
# but only use cached namespaces if all required response
# headers are also available from cache.
resp, namespaces, cache_state = \
self._get_listing_namespaces_from_cache(req, headers)
else:
# container metadata didn't support a cache lookup, this could
# be the case that container metadata was not in cache and we
# don't know if the container was sharded, or the case that the
# sharding state in metadata indicates the container was
# unsharded.
cache_state = 'bypass'
else: else:
# container metadata didn't support a cache lookup, this could be cache_state = 'disabled'
# the case that container metadata was not in cache and we don't
# know if the container was sharded, or the case that the sharding
# state in metadata indicates the container was unsharded.
cache_state = 'bypass'
# The request was not fulfilled from cache so send to backend server.
return self._get_shard_ranges_from_backend(req), cache_state
def GETorHEAD(self, req): if not namespaces:
"""Handler for HTTP GET/HEAD requests.""" resp, namespaces = self._get_listing_namespaces_from_backend(
req, cache_enabled)
self._record_shard_listing_cache_metrics(cache_state, resp, info)
if namespaces is not None:
# we got namespaces, so the container must be sharded; now build
# the listing from shards
# NB: the filtered namespaces list may be empty but we still need
# to build a response body with an empty list of objects
resp = self._get_from_shards(req, resp, namespaces)
return resp
def _get_or_head_pre_check(self, req):
ai = self.account_info(self.account_name, req) ai = self.account_info(self.account_name, req)
auto_account = self.account_name.startswith( auto_account = self.account_name.startswith(
self.app.auto_create_account_prefix) self.app.auto_create_account_prefix)
@ -339,78 +366,9 @@ class ContainerController(Controller):
# Don't cache this. The lack of account will be cached, and that # Don't cache this. The lack of account will be cached, and that
# is sufficient. # is sufficient.
return HTTPNotFound(request=req) return HTTPNotFound(request=req)
return None
# The read-modify-write of params here is because the Request.params def _get_or_head_post_check(self, req, resp):
# getter dynamically generates a dict of params from the query string;
# the setter must be called for new params to update the query string.
params = req.params
params['format'] = 'json'
# x-backend-record-type may be sent via internal client e.g. from the
# sharder, or by the proxy itself when making a recursive request, or
# in probe tests. If the header is present then the only values that
# the proxy respects are 'object' or 'shard'. However, the proxy may
# use the value 'auto' when making requests to container server.
orig_record_type = req.headers.get('X-Backend-Record-Type', '').lower()
if orig_record_type in ('object', 'shard'):
record_type = orig_record_type
else:
record_type = 'auto'
req.headers['X-Backend-Record-Type'] = 'auto'
req.headers['X-Backend-Record-Shard-Format'] = 'namespace'
params['states'] = 'listing'
req.params = params
if (req.method == 'GET'
and get_param(req, 'states') == 'listing'
and record_type != 'object'):
may_get_listing_shards = True
# Only lookup container info from cache and skip the backend HEAD,
# since we are going to GET the backend container anyway.
info = get_container_info(
req.environ, self.app, swift_source=None, cache_only=True)
else:
info = None
may_get_listing_shards = False
memcache = cache_from_env(req.environ, True)
sr_cache_state = None
if (may_get_listing_shards and
self.app.recheck_listing_shard_ranges > 0
and memcache
and not config_true_value(
req.headers.get('x-backend-include-deleted', False))):
# This GET might be served from cache or might populate cache.
# 'x-backend-include-deleted' is not usually expected in requests
# to the proxy (it is used from sharder to container servers) but
# it is included in the conditions just in case because we don't
# cache deleted shard ranges.
resp, sr_cache_state = self._GET_using_cache(req, info)
else:
resp = self._GETorHEAD_from_backend(req)
if may_get_listing_shards and (
not self.app.recheck_listing_shard_ranges or not memcache):
sr_cache_state = 'disabled'
resp_record_type = resp.headers.get('X-Backend-Record-Type', '')
if sr_cache_state:
self._record_shard_listing_cache_metrics(
sr_cache_state, resp, resp_record_type, info)
if all((req.method == "GET", record_type == 'auto',
resp_record_type.lower() == 'shard')):
data = self._parse_listing_response(req, resp)
namespaces = self._parse_namespaces(req, data, resp)
if namespaces is not None:
# we got namespaces, so the container must be sharded; now
# build the listing from shards
# NB: the filtered namespaces list may be empty but we still
# need to build a response body with an empty list of shards
resp = self._get_from_shards(req, resp, namespaces)
if orig_record_type not in ('object', 'shard'):
resp.headers.pop('X-Backend-Record-Type', None)
resp.headers.pop('X-Backend-Record-Shard-Format', None)
if not config_true_value( if not config_true_value(
resp.headers.get('X-Backend-Cached-Results')): resp.headers.get('X-Backend-Cached-Results')):
# Cache container metadata. We just made a request to a storage # Cache container metadata. We just made a request to a storage
@ -419,6 +377,7 @@ class ContainerController(Controller):
self.app.recheck_container_existence) self.app.recheck_container_existence)
set_info_cache(req.environ, self.account_name, set_info_cache(req.environ, self.account_name,
self.container_name, resp) self.container_name, resp)
if 'swift.authorize' in req.environ: if 'swift.authorize' in req.environ:
req.acl = wsgi_to_str(resp.headers.get('x-container-read')) req.acl = wsgi_to_str(resp.headers.get('x-container-read'))
aresp = req.environ['swift.authorize'](req) aresp = req.environ['swift.authorize'](req)
@ -437,6 +396,51 @@ class ContainerController(Controller):
'False')) 'False'))
return resp return resp
@public
@delay_denial
@cors_validation
def GET(self, req):
"""Handler for HTTP GET requests."""
# early checks for request validity
validate_container_params(req)
aresp = self._get_or_head_pre_check(req)
if aresp:
return aresp
# Always request json format from the backend. listing_formats
# middleware will take care of what the client gets back.
# The read-modify-write of params here is because the
# Request.params getter dynamically generates a dict of params from
# the query string; the setter must be called for new params to
# update the query string.
params = req.params
params['format'] = 'json'
req.params = params
# x-backend-record-type may be sent via internal client e.g. from
# the sharder or in probe tests
record_type = req.headers.get('X-Backend-Record-Type', '').lower()
if record_type in ('object', 'shard'):
# Go direct to the backend for HEADs, and GETs that *explicitly*
# specify a record type. We won't be reading/writing namespaces in
# cache nor building listings from shards. This path is used by
# the sharder, manage_shard_ranges and other tools that fetch shard
# ranges, and by the proxy itself when explicitly requesting
# objects while recursively building a listing from shards.
# Note: shard record type could be namespace or full format
resp = self._GETorHEAD_from_backend(req)
else:
# Requests that do not explicitly specify a record type, or specify
# 'auto', default to returning an object listing. The listing may
# be built from shards and may involve reading/writing namespaces
# in cache. This path is used for client requests and by the proxy
# itself while recursively building a listing from shards.
resp = self._GET_auto(req)
resp.headers.pop('X-Backend-Record-Type', None)
resp.headers.pop('X-Backend-Record-Shard-Format', None)
return self._get_or_head_post_check(req, resp)
def _get_from_shards(self, req, resp, namespaces): def _get_from_shards(self, req, resp, namespaces):
""" """
Construct an object listing using shards described by the list of Construct an object listing using shards described by the list of
@ -527,6 +531,8 @@ class ContainerController(Controller):
shard_listing_history): shard_listing_history):
# directed back to same container - force GET of objects # directed back to same container - force GET of objects
headers['X-Backend-Record-Type'] = 'object' headers['X-Backend-Record-Type'] = 'object'
else:
headers['X-Backend-Record-Type'] = 'auto'
if config_true_value(req.headers.get('x-newest', False)): if config_true_value(req.headers.get('x-newest', False)):
headers['X-Newest'] = 'true' headers['X-Newest'] = 'true'
@ -611,21 +617,16 @@ class ContainerController(Controller):
[o['bytes'] for o in objects]) [o['bytes'] for o in objects])
return resp return resp
@public
@delay_denial
@cors_validation
def GET(self, req):
"""Handler for HTTP GET requests."""
# early checks for request validity
validate_container_params(req)
return self.GETorHEAD(req)
@public @public
@delay_denial @delay_denial
@cors_validation @cors_validation
def HEAD(self, req): def HEAD(self, req):
"""Handler for HTTP HEAD requests.""" """Handler for HTTP HEAD requests."""
return self.GETorHEAD(req) aresp = self._get_or_head_pre_check(req)
if aresp:
return aresp
resp = self._GETorHEAD_from_backend(req)
return self._get_or_head_post_check(req, resp)
@public @public
@cors_validation @cors_validation

17
test/probe/test_sharder.py
View File

@ -2998,17 +2998,14 @@ class TestShardedAPI(BaseTestContainerSharding):
params={'states': 'updating'}) params={'states': 'updating'})
self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges) self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges)
# XXX the states=listing param provokes the proxy to cache the backend shard_ranges = self.get_container_shard_ranges(
# values and then respond to the client with the cached *namespaces* !! params={'states': 'listing'})
# shard_ranges = self.get_container_shard_ranges( self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges)
# params={'states': 'listing'})
# self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges)
# XXX ditto... shard_ranges = self.get_container_shard_ranges(
# shard_ranges = self.get_container_shard_ranges( headers={'X-Newest': 'true'},
# headers={'X-Newest': 'true'}, params={'states': 'listing'})
# params={'states': 'listing'}) self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges)
# self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges)
# this is what the sharder requests... # this is what the sharder requests...
shard_ranges = self.get_container_shard_ranges( shard_ranges = self.get_container_shard_ranges(

9
test/unit/common/test_utils.py
View File

@ -7904,6 +7904,15 @@ class TestNamespaceBoundList(unittest.TestCase):
self.end_ns = utils.Namespace('a/z-', 'z', '') self.end_ns = utils.Namespace('a/z-', 'z', '')
self.lowerbounds = [start, atof, ftol, ltor, rtoz, end] self.lowerbounds = [start, atof, ftol, ltor, rtoz, end]
def test_eq(self):
this = utils.NamespaceBoundList(self.lowerbounds)
that = utils.NamespaceBoundList(self.lowerbounds)
self.assertEqual(this, that)
that = utils.NamespaceBoundList(self.lowerbounds[:1])
self.assertNotEqual(this, that)
self.assertNotEqual(this, None)
self.assertNotEqual(this, self.lowerbounds)
def test_get_namespace(self): def test_get_namespace(self):
namespace_list = utils.NamespaceBoundList(self.lowerbounds) namespace_list = utils.NamespaceBoundList(self.lowerbounds)
self.assertEqual(namespace_list.bounds, self.lowerbounds) self.assertEqual(namespace_list.bounds, self.lowerbounds)

1448
test/unit/proxy/controllers/test_container.py
View File

File diff suppressed because it is too large Load Diff