openstack/swift
Code Issues Proposed changes

Merge "slo: part-number=N query parameter support"

Browse Source
This commit is contained in:
Zuul 2024-03-13 00:13:45 +00:00 committed by Gerrit Code Review
commit 60db1f847c
7 changed files with 1206 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

193
swift/common/middleware/slo.py
View File

@ -270,11 +270,27 @@ A GET request with the query parameters::
will return the contents of the original manifest as it was sent by the client. will return the contents of the original manifest as it was sent by the client.
The main purpose for both calls is solely debugging. The main purpose for both calls is solely debugging.
When the manifest object is uploaded you are more or less guaranteed that A GET request to a manifest object with the query parameter::
every segment in the manifest exists and matched the specifications.
However, there is nothing that prevents the user from breaking the ?part-number=<n>
SLO download by deleting/replacing a segment referenced in the manifest. It is
left to the user to use caution in handling the segments. will return the contents of the ``nth`` segment. Segments are indexed from 1,
so ``n`` must be an integer between 1 and the total number of segments in the
manifest. The response status will be ``206 Partial Content`` and its headers
will include: an ``X-Parts-Count`` header equal to the total number of
segments; a ``Content-Length`` header equal to the length of the specified
segment; a ``Content-Range`` header describing the byte range of the specified
part within the SLO. A HEAD request with a ``part-number`` parameter will also
return a response with status ``206 Partial Content`` and the same headers.
.. note::
When the manifest object is uploaded you are more or less guaranteed that
every segment in the manifest exists and matched the specifications.
However, there is nothing that prevents the user from breaking the SLO
download by deleting/replacing a segment referenced in the manifest. It is
left to the user to use caution in handling the segments.
----------------------- -----------------------
Deleting a Large Object Deleting a Large Object
@ -353,7 +369,7 @@ from swift.common.registry import register_swift_info
from swift.common.request_helpers import SegmentedIterable, \ from swift.common.request_helpers import SegmentedIterable, \
get_sys_meta_prefix, update_etag_is_at_header, resolve_etag_is_at_header, \ get_sys_meta_prefix, update_etag_is_at_header, resolve_etag_is_at_header, \
get_container_update_override_key, update_ignore_range_header, \ get_container_update_override_key, update_ignore_range_header, \
get_param get_param, get_valid_part_num
from swift.common.constraints import check_utf8, AUTO_CREATE_ACCOUNT_PREFIX from swift.common.constraints import check_utf8, AUTO_CREATE_ACCOUNT_PREFIX
from swift.common.http import HTTP_NOT_FOUND, HTTP_UNAUTHORIZED from swift.common.http import HTTP_NOT_FOUND, HTTP_UNAUTHORIZED
from swift.common.wsgi import WSGIContext, make_subrequest, make_env, \ from swift.common.wsgi import WSGIContext, make_subrequest, make_env, \
@ -564,6 +580,60 @@ def _annotate_segments(segments, logger=None):
seg_dict['segment_length'] = segment_length seg_dict['segment_length'] = segment_length
def calculate_byterange_for_part_num(req, segments, part_num):
"""
Helper function to calculate the byterange for a part_num response.
N.B. as a side-effect of calculating the single tuple representing the
byterange required for a part_num response this function will also mutate
the request's Range header so that swob knows to return 206.
:param req: the request object
:param segments: the list of seg_dicts
:param part_num: the part number of the object to return
:returns: a tuple representing the byterange
"""
start = 0
for seg in segments[:part_num - 1]:
start += seg['segment_length']
last = start + segments[part_num - 1]['segment_length']
# We need to mutate the request's Range header so that swob knows to
# handle these partial content requests correctly.
req.range = "bytes=%d-%d" % (start, last - 1)
return start, last - 1
def calculate_byteranges(req, segments, resp_attrs, part_num):
"""
Calculate the byteranges based on the request, segments, and part number.
N.B. as a side-effect of calculating the single tuple representing the
byterange required for a part_num response this function will also mutate
the request's Range header so that swob knows to return 206.
:param req: the request object
:param segments: the list of seg_dicts
:param resp_attrs: the slo response attributes
:param part_num: the part number of the object to return
:returns: a list of tuples representing byteranges
"""
if req.range:
byteranges = [
# For some reason, swob.Range.ranges_for_length adds 1 to the
# last byte's position.
(start, end - 1) for start, end
in req.range.ranges_for_length(resp_attrs.slo_size)]
elif part_num:
byteranges = [
calculate_byterange_for_part_num(req, segments, part_num)]
else:
byteranges = [(0, resp_attrs.slo_size - 1)]
return byteranges
class RespAttrs(object): class RespAttrs(object):
""" """
Encapsulate properties of a GET or HEAD response that are pertinent to Encapsulate properties of a GET or HEAD response that are pertinent to
@ -684,6 +754,9 @@ class SloGetContext(WSGIContext):
method='GET', method='GET',
headers={'x-auth-token': req.headers.get('x-auth-token')}, headers={'x-auth-token': req.headers.get('x-auth-token')},
agent='%(orig)s SLO MultipartGET', swift_source='SLO') agent='%(orig)s SLO MultipartGET', swift_source='SLO')
params_copy = dict(req.params)
params_copy.pop('part-number', None)
sub_req.params = params_copy
sub_resp = sub_req.get_response(self.slo.app) sub_resp = sub_req.get_response(self.slo.app)
if not sub_resp.is_success: if not sub_resp.is_success:
@ -847,8 +920,7 @@ class SloGetContext(WSGIContext):
# we can avoid re-fetching the object. # we can avoid re-fetching the object.
return first_byte == 0 and last_byte == length - 1 return first_byte == 0 and last_byte == length - 1
def _is_manifest_and_need_to_refetch(self, req, resp_attrs, def _need_to_refetch_manifest(self, req, resp_attrs, is_part_num_request):
is_manifest_get):
""" """
Check if the segments will be needed to service the request and update Check if the segments will be needed to service the request and update
the segment_listing_needed attribute. the segment_listing_needed attribute.
@ -856,19 +928,11 @@ class SloGetContext(WSGIContext):
:return: boolean indicating if we need to refetch, only if the segments :return: boolean indicating if we need to refetch, only if the segments
ARE needed we MAY need to refetch them! ARE needed we MAY need to refetch them!
""" """
if not resp_attrs.is_slo:
# Not a static large object manifest, maybe an error, regardless
# no refetch needed
return False
if is_manifest_get:
# Any manifest json object response will do
return False
if req.method == 'HEAD': if req.method == 'HEAD':
# There may be some cases in the future where a HEAD resp on even a # There may be some cases in the future where a HEAD resp on even a
# modern manifest should refetch, e.g. lp bug #2029174 # modern manifest should refetch, e.g. lp bug #2029174
self.segment_listing_needed = resp_attrs.is_legacy self.segment_listing_needed = (resp_attrs.is_legacy or
is_part_num_request)
# it will always be the case that a HEAD must re-fetch iff # it will always be the case that a HEAD must re-fetch iff
# segment_listing_needed # segment_listing_needed
return self.segment_listing_needed return self.segment_listing_needed
@ -965,22 +1029,56 @@ class SloGetContext(WSGIContext):
replace_headers) replace_headers)
def _return_slo_response(self, req, start_response, resp_iter, resp_attrs): def _return_slo_response(self, req, start_response, resp_iter, resp_attrs):
headers = {
'Etag': '"%s"' % resp_attrs.slo_etag,
'X-Manifest-Etag': resp_attrs.json_md5,
# swob will fix this for a GET with Range
'Content-Length': str(resp_attrs.slo_size),
# ignore bogus content-range, make swob figure it out
'Content-Range': None,
}
if self.segment_listing_needed: if self.segment_listing_needed:
# consume existing resp_iter; we'll create a new one # consume existing resp_iter; we'll create a new one
segments = self._parse_segments(resp_iter) segments = self._parse_segments(resp_iter)
resp_attrs.update_from_segments(segments) resp_attrs.update_from_segments(segments)
headers['Etag'] = '"%s"' % resp_attrs.slo_etag
headers['Content-Length'] = str(resp_attrs.slo_size)
part_num = get_valid_part_num(req)
if part_num:
headers['X-Parts-Count'] = len(segments)
if part_num and part_num > len(segments):
if req.method == 'HEAD': if req.method == 'HEAD':
resp_iter = [] resp_iter = []
headers['Content-Length'] = '0'
else: else:
resp_iter = self._build_resp_iter(req, segments, resp_attrs) body = b'The requested part number is not satisfiable'
headers = { resp_iter = [body]
'Etag': '"%s"' % resp_attrs.slo_etag, headers['Content-Length'] = len(body)
'X-Manifest-Etag': resp_attrs.json_md5, headers['Content-Range'] = 'bytes */%d' % resp_attrs.slo_size
# This isn't correct for range requests, but swob will fix it? self._response_status = '416 Requested Range Not Satisfiable'
'Content-Length': str(resp_attrs.slo_size), elif part_num and req.method == 'HEAD':
# ignore bogus content-range, make swob figure it out resp_iter = []
'Content-Range': None headers['Content-Length'] = \
} segments[part_num - 1].get('segment_length')
start, end = calculate_byterange_for_part_num(
req, segments, part_num)
headers['Content-Range'] = \
'bytes {}-{}/{}'.format(start, end,
resp_attrs.slo_size)
# The RFC specifies 206 in the context of Range requests, and
# Range headers MUST be ignored for HEADs [1], so a HEAD will
# not normally return a 206. However, a part-number HEAD
# returns Content-Length equal to the part size, rather than
# the whole object size, so in this case we do return 206.
# [1] https://www.rfc-editor.org/rfc/rfc9110#name-range
self._response_status = '206 Partial Content'
elif req.method == 'HEAD':
resp_iter = []
else:
byteranges = calculate_byteranges(
req, segments, resp_attrs, part_num)
resp_iter = self._build_resp_iter(req, segments, byteranges)
return self._return_response(req, start_response, resp_iter, return self._return_response(req, start_response, resp_iter,
replace_headers=headers) replace_headers=headers)
@ -1046,19 +1144,30 @@ class SloGetContext(WSGIContext):
update_ignore_range_header(req, 'X-Static-Large-Object') update_ignore_range_header(req, 'X-Static-Large-Object')
# process original request # process original request
orig_path_info = req.path_info
resp_iter = self._app_call(req.environ) resp_iter = self._app_call(req.environ)
resp_attrs = RespAttrs.from_headers(self._response_headers) resp_attrs = RespAttrs.from_headers(self._response_headers)
# the next two calls hide a couple side-effects, sorry: if resp_attrs.is_slo and not is_manifest_get:
try:
# only validate part-number if the request is to an SLO
part_num = get_valid_part_num(req)
except HTTPException:
friendly_close(resp_iter)
raise
# the next two calls hide a couple side effects, sorry:
# #
# 1) regardless of the return value the "need_to_refetch" check *may* # 1) regardless of the return value the "need_to_refetch" check
# also set self.segment_listing_needed = True (it's commented to # *may* also set self.segment_listing_needed = True (it's
# help you wrap your head around that one, good luck) # commented to help you wrap your head around that one,
# 2) if we refetch, we overwrite the current resp_iter and resp_attrs # good luck)
# variables, partly because we *might* get back a NOT # 2) if we refetch, we overwrite the current resp_iter and
# resp_attrs variables, partly because we *might* get back a NOT
# resp_attrs.is_slo response (even if we had one to start), but # resp_attrs.is_slo response (even if we had one to start), but
# hopefully they're just the manifest resp we needed to refetch! # hopefully they're just the manifest resp we needed to refetch!
if self._is_manifest_and_need_to_refetch(req, resp_attrs, if self._need_to_refetch_manifest(req, resp_attrs, part_num):
is_manifest_get): # reset path in case it was modified during original request
# (e.g. object versioning might re-write the path)
req.path_info = orig_path_info
resp_attrs, resp_iter = self._refetch_manifest( resp_attrs, resp_iter = self._refetch_manifest(
req, resp_iter, resp_attrs) req, resp_iter, resp_attrs)
@ -1115,24 +1224,16 @@ class SloGetContext(WSGIContext):
raise HTTPServerError(msg) raise HTTPServerError(msg)
return segments return segments
def _build_resp_iter(self, req, segments, resp_attrs): def _build_resp_iter(self, req, segments, byteranges):
""" """
Build a response iterable for a GET request. Build a response iterable for a GET request.
:param req: the request object :param req: the request object
:param resp_attrs: the slo attributes :param segments: the list of seg_dicts
:param byteranges: a list of tuples representing byteranges
:returns: a segmented iterable :returns: a segmented iterable
""" """
if req.range:
byteranges = [
# For some reason, swob.Range.ranges_for_length adds 1 to the
# last byte's position.
(start, end - 1) for start, end
in req.range.ranges_for_length(resp_attrs.slo_size)]
else:
byteranges = [(0, resp_attrs.slo_size - 1)]
ver, account, _junk = req.split_path(3, 3, rest_with_last=True) ver, account, _junk = req.split_path(3, 3, rest_with_last=True)
account = wsgi_to_str(account) account = wsgi_to_str(account)
plain_listing_iter = self._segment_listing_iterator( plain_listing_iter = self._segment_listing_iterator(

33
swift/common/request_helpers.py
View File

@ -95,6 +95,39 @@ def get_param(req, name, default=None):
return value return value
def get_valid_part_num(req):
"""
Any non-range GET or HEAD request for a SLO object may include a
part-number parameter in query string. If the passed in request
includes a part-number parameter it will be parsed into a valid integer
and returned. If the passed in request does not include a part-number
param we will return None. If the part-number parameter is invalid for
the given request we will raise the appropriate HTTP exception
:param req: the request object
:returns: validated part-number value or None
:raises HTTPBadRequest: if request or part-number param is not valid
"""
part_number_param = get_param(req, 'part-number')
if part_number_param is None:
return None
try:
part_number = int(part_number_param)
if part_number <= 0:
raise ValueError
except ValueError:
raise HTTPBadRequest('Part number must be an integer greater '
'than 0')
if req.range:
raise HTTPBadRequest(req=req,
body='Range requests are not supported '
'with part number queries')
return part_number
def validate_params(req, names): def validate_params(req, names):
""" """
Get list of parameters from an HTTP request, validating the encoding of Get list of parameters from an HTTP request, validating the encoding of

4
test/functional/swift_test_client.py
View File

@ -938,7 +938,7 @@ class File(Base):
return True return True
def info(self, hdrs=None, parms=None, cfg=None): def info(self, hdrs=None, parms=None, cfg=None, exp_status=200):
if hdrs is None: if hdrs is None:
hdrs = {} hdrs = {}
if parms is None: if parms is None:
@ -946,7 +946,7 @@ class File(Base):
if cfg is None: if cfg is None:
cfg = {} cfg = {}
if self.conn.make_request('HEAD', self.path, hdrs=hdrs, if self.conn.make_request('HEAD', self.path, hdrs=hdrs,
parms=parms, cfg=cfg) != 200: parms=parms, cfg=cfg) != exp_status:
raise ResponseError(self.conn.response, 'HEAD', raise ResponseError(self.conn.response, 'HEAD',
self.conn.make_path(self.path)) self.conn.make_path(self.path))

131
test/functional/test_object_versioning.py
View File

@ -2311,11 +2311,12 @@ class TestSloWithVersioning(TestObjectVersioningBase):
'/', 1)[-1] '/', 1)[-1]
return self._account_name return self._account_name
def _create_manifest(self, seg_name): def _create_manifest(self, seg_names):
# create a manifest in the versioning container # create a manifest in the versioning container
file_item = self.container.file("my-slo-manifest") file_item = self.container.file("my-slo-manifest")
manifest = [self.seg_info[seg_name] for seg_name in seg_names]
resp = file_item.write( resp = file_item.write(
json.dumps([self.seg_info[seg_name]]).encode('ascii'), json.dumps(manifest).encode('ascii'),
parms={'multipart-manifest': 'put'}, parms={'multipart-manifest': 'put'},
return_resp=True) return_resp=True)
version_id = resp.getheader('x-object-version-id') version_id = resp.getheader('x-object-version-id')
@ -2340,9 +2341,10 @@ class TestSloWithVersioning(TestObjectVersioningBase):
self.assertEqual(1, len(manifest)) self.assertEqual(1, len(manifest))
key_map = {'etag': 'hash', 'size_bytes': 'bytes', 'path': 'name'} key_map = {'etag': 'hash', 'size_bytes': 'bytes', 'path': 'name'}
for k_client, k_slo in key_map.items(): for k_client, k_slo in key_map.items():
self.assertEqual(self.seg_info[seg_name][k_client], self.assertEqual(self.seg_info[seg_name][k_client],
manifest[0][k_slo]) Utils.encode_if_py2(manifest[0][k_slo]))
def _assert_is_object(self, file_item, seg_data, version_id=None): def _assert_is_object(self, file_item, seg_data, version_id=None):
if version_id: if version_id:
@ -2357,13 +2359,13 @@ class TestSloWithVersioning(TestObjectVersioningBase):
self._tear_down_files(self.container) self._tear_down_files(self.container)
def test_slo_manifest_version(self): def test_slo_manifest_version(self):
file_item, v1_version_id = self._create_manifest('a') file_item, v1_version_id = self._create_manifest(['a'])
# sanity check: read the manifest, then the large object # sanity check: read the manifest, then the large object
self._assert_is_manifest(file_item, 'a') self._assert_is_manifest(file_item, 'a')
self._assert_is_object(file_item, b'a') self._assert_is_object(file_item, b'a')
# upload new manifest # upload new manifest
file_item, v2_version_id = self._create_manifest('b') file_item, v2_version_id = self._create_manifest(['b'])
# sanity check: read the manifest, then the large object # sanity check: read the manifest, then the large object
self._assert_is_manifest(file_item, 'b') self._assert_is_manifest(file_item, 'b')
self._assert_is_object(file_item, b'b') self._assert_is_object(file_item, b'b')
@ -2445,7 +2447,7 @@ class TestSloWithVersioning(TestObjectVersioningBase):
self.assertEqual(409, caught.exception.status) self.assertEqual(409, caught.exception.status)
def test_links_to_slo(self): def test_links_to_slo(self):
file_item, v1_version_id = self._create_manifest('a') file_item, v1_version_id = self._create_manifest(['a'])
slo_info = file_item.info() slo_info = file_item.info()
symlink_name = Utils.create_name() symlink_name = Utils.create_name()
@ -2463,6 +2465,123 @@ class TestSloWithVersioning(TestObjectVersioningBase):
symlink.write(b'', hdrs=sym_headers) symlink.write(b'', hdrs=sym_headers)
self.assertEqual(slo_info, symlink.info()) self.assertEqual(slo_info, symlink.info())
def test_slo_HEAD_part_number_with_version(self):
file_item, version_id = self._create_manifest(['a', 'b'])
file_item.info(parms={'part-number': '1',
'version-id': version_id},
exp_status=206)
sizes = [seg['size_bytes']
for seg in (self.seg_info['a'], self.seg_info['b'])]
total_size = sum(sizes)
resp = file_item.conn.response
self.assertEqual(version_id, resp.getheader('X-Object-Version-Id'))
self.assertEqual('2', resp.getheader('X-Parts-Count'))
self.assertEqual('bytes 0-%s/%s' % (sizes[0] - 1, total_size),
resp.getheader('Content-Range'))
file_item.info(parms={'part-number': '2',
'version-id': version_id},
exp_status=206)
resp = file_item.conn.response
self.assertEqual(version_id, resp.getheader('X-Object-Version-Id'))
self.assertEqual('2', resp.getheader('X-Parts-Count'))
self.assertEqual('bytes %s-%s/%s'
% (sizes[1], total_size - 1, total_size),
resp.getheader('Content-Range'))
file_item.info(parms={'part-number': '3',
'version-id': version_id},
exp_status=416)
resp = file_item.conn.response
self.assertEqual(version_id, resp.getheader('X-Object-Version-Id'))
self.assertEqual('2', resp.getheader('X-Parts-Count'))
self.assertEqual('bytes */%s' % total_size,
resp.getheader('Content-Range'))
def test_slo_GET_part_number_with_version(self):
file_item, version_id = self._create_manifest(['a', 'b'])
body = file_item.read(parms={'part-number': '1',
'version-id': version_id})
sizes = [seg['size_bytes']
for seg in (self.seg_info['a'], self.seg_info['b'])]
total_size = sum(sizes)
resp = file_item.conn.response
self.assertEqual(version_id, resp.getheader('X-Object-Version-Id'))
self.assertEqual('2', resp.getheader('X-Parts-Count'))
self.assertEqual('bytes 0-%s/%s' % (sizes[0] - 1, total_size),
resp.getheader('Content-Range'))
self.assertEqual(('a' * sizes[0]).encode('ascii'), body)
body = file_item.read(parms={'part-number': '2',
'version-id': version_id})
resp = file_item.conn.response
self.assertEqual(version_id, resp.getheader('X-Object-Version-Id'))
self.assertEqual('2', resp.getheader('X-Parts-Count'))
self.assertEqual('bytes %s-%s/%s'
% (sizes[1], total_size - 1, total_size),
resp.getheader('Content-Range'))
self.assertEqual(('b' * sizes[0]).encode('ascii'), body)
with self.assertRaises(ResponseError):
file_item.read(parms={'part-number': '3',
'version-id': version_id})
self.assertEqual(416, file_item.conn.response.status)
resp = file_item.conn.response
self.assertEqual(version_id, resp.getheader('X-Object-Version-Id'))
self.assertEqual('2', resp.getheader('X-Parts-Count'))
self.assertEqual('bytes */%s' % total_size,
resp.getheader('Content-Range'))
def test_slo_HEAD_part_number_multiple_versions(self):
file_item, version_id_1 = self._create_manifest(['a', 'b'])
file_item, version_id_2 = self._create_manifest(['a'])
# older version has 2 parts
file_item.info(parms={'part-number': '2',
'version-id': version_id_1},
exp_status=206)
sizes = [seg['size_bytes']
for seg in (self.seg_info['a'], self.seg_info['b'])]
total_size = sum(sizes)
resp = file_item.conn.response
self.assertEqual(version_id_1, resp.getheader('x-object-version-id'))
self.assertEqual('2', resp.getheader('X-Parts-Count'))
self.assertEqual('bytes %s-%s/%s'
% (sizes[1], total_size - 1, total_size),
resp.getheader('Content-Range'))
# newer version has only 1 part
file_item.info(parms={'part-number': '1',
'version-id': version_id_2},
exp_status=206)
resp = file_item.conn.response
self.assertEqual(version_id_2, resp.getheader('X-Object-Version-Id'))
self.assertEqual('1', resp.getheader('X-Parts-Count'))
self.assertEqual('bytes %s-%s/%s'
% (0, sizes[0] - 1, sizes[0]),
resp.getheader('Content-Range'))
file_item.info(parms={'part-number': '2',
'version-id': version_id_2},
exp_status=416)
resp = file_item.conn.response
self.assertEqual(version_id_2, resp.getheader('X-Object-Version-Id'))
self.assertEqual('1', resp.getheader('X-Parts-Count'))
self.assertEqual('bytes */%s' % sizes[0],
resp.getheader('Content-Range'))
# current version == newer version has only 1 part
file_item.info(parms={'part-number': '2'},
exp_status=416)
resp = file_item.conn.response
self.assertEqual(version_id_2, resp.getheader('X-Object-Version-Id'))
self.assertEqual('1', resp.getheader('X-Parts-Count'))
self.assertEqual('bytes */%s' % sizes[0],
resp.getheader('Content-Range'))
class TestSloWithVersioningUTF8(Base2, TestSloWithVersioning):
pass
class TestVersionsLocationWithVersioning(TestObjectVersioningBase): class TestVersionsLocationWithVersioning(TestObjectVersioningBase):

157
test/functional/test_slo.py
View File

@ -293,6 +293,163 @@ class TestSlo(Base):
(b'e', 1), (b'e', 1),
], group_file_contents(file_contents)) ], group_file_contents(file_contents))
def test_slo_multipart_delete_part_number_ignored(self):
# create a container just for this test because we're going to delete
# objects that we create
container = self.env.account.container(Utils.create_name())
self.assertTrue(container.create())
# create segments in same container
seg_info = self.env.create_segments(container)
file_item = container.file("manifest-abcde")
self.assertTrue(file_item.write(
json.dumps([seg_info['seg_a'], seg_info['seg_b'],
seg_info['seg_c'], seg_info['seg_d'],
seg_info['seg_e']]).encode('ascii'),
parms={'multipart-manifest': 'put'}))
# sanity check, we have SLO...
file_item.initialize(parms={'part-number': '5'})
self.assertEqual(
file_item.conn.response.getheader('X-Static-Large-Object'), 'True')
self.assertEqual(
file_item.conn.response.getheader('X-Parts-Count'), '5')
self.assertEqual(6, len(container.files()))
# part-number should be ignored
status = file_item.conn.make_request(
'DELETE', file_item.path,
parms={'multipart-manifest': 'delete',
'part-number': '2'})
self.assertEqual(200, status)
# everything is gone
self.assertFalse(container.files())
def test_get_head_part_number_invalid(self):
file_item = self.env.container.file('manifest-abcde')
file_item.initialize()
ok_resp = file_item.conn.response
self.assertEqual(ok_resp.getheader('X-Static-Large-Object'), 'True')
self.assertEqual(ok_resp.getheader('Etag'), file_item.etag)
self.assertEqual(ok_resp.getheader('Content-Length'),
str(file_item.size))
# part-number is 1-indexed
self.assertRaises(ResponseError, file_item.read,
parms={'part-number': '0'})
resp_body = file_item.conn.response.read()
self.assertEqual(400, file_item.conn.response.status, resp_body)
self.assertEqual(b'Part number must be an integer greater than 0',
resp_body)
self.assertRaises(ResponseError, file_item.initialize,
parms={'part-number': '0'})
resp_body = file_item.conn.response.read()
self.assertEqual(400, file_item.conn.response.status)
self.assertEqual(b'', resp_body)
def test_get_head_part_number_out_of_range(self):
file_item = self.env.container.file('manifest-abcde')
file_item.initialize()
ok_resp = file_item.conn.response
self.assertEqual(ok_resp.getheader('X-Static-Large-Object'), 'True')
self.assertEqual(ok_resp.getheader('Etag'), file_item.etag)
self.assertEqual(ok_resp.getheader('Content-Length'),
str(file_item.size))
manifest_etag = ok_resp.getheader('Manifest-Etag')
def check_headers(resp):
self.assertEqual(resp.getheader('X-Static-Large-Object'), 'True')
self.assertEqual(resp.getheader('Etag'), file_item.etag)
self.assertEqual(resp.getheader('Manifest-Etag'), manifest_etag)
self.assertEqual(resp.getheader('X-Parts-Count'), '5')
self.assertEqual(resp.getheader('Content-Range'),
'bytes */%s' % file_item.size)
self.assertRaises(ResponseError, file_item.read,
parms={'part-number': '10001'})
resp_body = file_item.conn.response.read()
self.assertEqual(416, file_item.conn.response.status, resp_body)
self.assertEqual(b'The requested part number is not satisfiable',
resp_body)
check_headers(file_item.conn.response)
self.assertEqual(file_item.conn.response.getheader('Content-Length'),
str(len(resp_body)))
self.assertRaises(ResponseError, file_item.info,
parms={'part-number': '10001'})
resp_body = file_item.conn.response.read()
self.assertEqual(416, file_item.conn.response.status)
self.assertEqual(b'', resp_body)
check_headers(file_item.conn.response)
self.assertEqual(file_item.conn.response.getheader('Content-Length'),
'0')
def test_get_part_number_simple_manifest(self):
file_item = self.env.container.file('manifest-abcde')
seg_info_list = [
self.env.seg_info["seg_%s" % letter]
for letter in ['a', 'b', 'c', 'd', 'e']
]
checksum = md5(usedforsecurity=False)
total_size = 0
for seg_info in seg_info_list:
checksum.update(seg_info['etag'].encode('ascii'))
total_size += seg_info['size_bytes']
slo_etag = checksum.hexdigest()
start = 0
manifest_etag = None
for i, seg_info in enumerate(seg_info_list, start=1):
part_contents = file_item.read(parms={'part-number': i})
self.assertEqual(len(part_contents), seg_info['size_bytes'])
headers = dict(
(h.lower(), v)
for h, v in file_item.conn.response.getheaders())
self.assertEqual(headers['content-length'],
str(seg_info['size_bytes']))
self.assertEqual(headers['etag'], '"%s"' % slo_etag)
if not manifest_etag:
manifest_etag = headers['x-manifest-etag']
else:
self.assertEqual(headers['x-manifest-etag'], manifest_etag)
end = start + seg_info['size_bytes'] - 1
self.assertEqual(headers['content-range'],
'bytes %d-%d/%d' % (start, end, total_size), i)
self.assertEqual(headers['x-parts-count'], '5')
start = end + 1
def test_head_part_number_simple_manifest(self):
file_item = self.env.container.file('manifest-abcde')
seg_info_list = [
self.env.seg_info["seg_%s" % letter]
for letter in ['a', 'b', 'c', 'd', 'e']
]
checksum = md5(usedforsecurity=False)
total_size = 0
for seg_info in seg_info_list:
checksum.update(seg_info['etag'].encode('ascii'))
total_size += seg_info['size_bytes']
slo_etag = checksum.hexdigest()
start = 0
manifest_etag = None
for i, seg_info in enumerate(seg_info_list, start=1):
part_info = file_item.info(parms={'part-number': i},
exp_status=206)
headers = dict(
(h.lower(), v)
for h, v in file_item.conn.response.getheaders())
self.assertEqual(headers['content-length'],
str(seg_info['size_bytes']))
self.assertEqual(headers['etag'], '"%s"' % slo_etag)
self.assertEqual(headers['etag'], part_info['etag'])
if not manifest_etag:
manifest_etag = headers['x-manifest-etag']
else:
self.assertEqual(headers['x-manifest-etag'], manifest_etag)
end = start + seg_info['size_bytes'] - 1
self.assertEqual(headers['content-range'],
'bytes %d-%d/%d' % (start, end, total_size), i)
self.assertEqual(headers['x-parts-count'], '5')
start = end + 1
def test_slo_container_listing(self): def test_slo_container_listing(self):
# the listing object size should equal the sum of the size of the # the listing object size should equal the sum of the size of the
# segments, not the size of the manifest body # segments, not the size of the manifest body

10
test/functional/tests.py
View File

@ -53,6 +53,12 @@ def tearDownModule():
class Utils(object): class Utils(object):
@classmethod
def encode_if_py2(cls, value):
if six.PY2 and isinstance(value, six.text_type):
return value.encode('utf8')
return value
@classmethod @classmethod
def create_ascii_name(cls, length=None): def create_ascii_name(cls, length=None):
return uuid.uuid4().hex return uuid.uuid4().hex
@ -71,9 +77,7 @@ class Utils(object):
u'\u5608\u3706\u1804\u0903\u03A9\u2603' u'\u5608\u3706\u1804\u0903\u03A9\u2603'
ustr = u''.join([random.choice(utf8_chars) ustr = u''.join([random.choice(utf8_chars)
for x in range(length)]) for x in range(length)])
if six.PY2: return cls.encode_if_py2(ustr)
return ustr.encode('utf-8')
return ustr
create_name = create_ascii_name create_name = create_ascii_name

731
test/unit/common/middleware/test_slo.py
View File

@ -19,6 +19,7 @@ from datetime import datetime
import json import json
import time import time
import unittest import unittest
import string
import mock import mock
from mock import patch from mock import patch
@ -1905,7 +1906,10 @@ class SloGETorHEADTestCase(SloTestCase):
They're nothing special, just small regular objects with names that They're nothing special, just small regular objects with names that
describe their content and size. describe their content and size.
""" """
for letter, size in zip(letters, range(5, 5 * len(letters) + 1, 5)): for i, letter in enumerate(string.ascii_lowercase):
if letter not in letters:
continue
size = (i + 1) * 5
body = letter * size body = letter * size
path = '/v1/AUTH_test/%s/%s_%s' % (container, letter, size) path = '/v1/AUTH_test/%s/%s_%s' % (container, letter, size)
self.app.register('GET', path, swob.HTTPOk, { self.app.register('GET', path, swob.HTTPOk, {
@ -1949,7 +1953,7 @@ class SloGETorHEADTestCase(SloTestCase):
# has *no* content-type, both empty or missing Content-Type header # has *no* content-type, both empty or missing Content-Type header
# on ?multipart-manifest=put result in a default # on ?multipart-manifest=put result in a default
# "application/octet-stream" value being stored in the manifest # "application/octet-stream" value being stored in the manifest
# metadata; sitll I wouldn't assert on this value in these tests or # metadata; still I wouldn't assert on this value in these tests,
# you may not be testing what you think you are - N.B. some tests # you may not be testing what you think you are - N.B. some tests
# will override this value with the "extra_headers" param. # will override this value with the "extra_headers" param.
'Content-Type': 'application/octet-stream', 'Content-Type': 'application/octet-stream',
@ -1964,6 +1968,34 @@ class SloGETorHEADTestCase(SloTestCase):
'GET', '/v1/AUTH_test/%s/%s' % (container, obj_key), 'GET', '/v1/AUTH_test/%s/%s' % (container, obj_key),
swob.HTTPOk, manifest_headers, manifest_json.encode('ascii')) swob.HTTPOk, manifest_headers, manifest_json.encode('ascii'))
def _setup_manifest_single_segment(self):
"""
This manifest's segments are all regular objects.
"""
_single_segment_manifest = [
{'name': '/gettest/b_50', 'hash': md5hex('b' * 50), 'bytes': '50',
'content_type': 'text/plain'},
]
self._setup_manifest(
'single-segment', _single_segment_manifest,
extra_headers={'X-Object-Meta-Nature': 'Regular'},
container='gettest')
def _setup_manifest_data(self):
_data_manifest = [
{
'data': base64.b64encode(b'123456').decode('ascii')
}, {
'name': '/gettest/a_5',
'hash': md5hex('a' * 5),
'content_type': 'text/plain',
'bytes': '5',
}, {
'data': base64.b64encode(b'ABCDEF').decode('ascii')
},
]
self._setup_manifest('data', _data_manifest)
def _setup_manifest_bc(self): def _setup_manifest_bc(self):
""" """
This manifest's segments are all regular objects. This manifest's segments are all regular objects.
@ -5656,6 +5688,701 @@ class TestSloConditionalGetNewManifest(TestSloConditionalGetOldManifest):
modern_manifest_headers = True modern_manifest_headers = True
class TestPartNumber(SloGETorHEADTestCase):
modern_manifest_headers = True
def setUp(self):
super(TestPartNumber, self).setUp()
self._setup_alphabet_objects('bcdj')
self._setup_manifest_bc()
self._setup_manifest_abcd()
self._setup_manifest_abcdefghijkl()
self._setup_manifest_bc_ranges()
self._setup_manifest_abcd_ranges()
self._setup_manifest_abcd_subranges()
self._setup_manifest_aabbccdd()
self._setup_manifest_single_segment()
# this b_50 object doesn't follow the alphabet convention
self.app.register(
'GET', '/v1/AUTH_test/gettest/b_50',
swob.HTTPPartialContent, {'Content-Length': '50',
'Etag': md5hex('b' * 50)},
'b' * 50)
self._setup_manifest_data()
def test_head_part_number(self):
req = Request.blank(
'/v1/AUTH_test/gettest/manifest-bc?part-number=1',
environ={'REQUEST_METHOD': 'HEAD'})
status, headers, body = self.call_slo(req)
expected_calls = [
('HEAD', '/v1/AUTH_test/gettest/manifest-bc?part-number=1'),
('GET', '/v1/AUTH_test/gettest/manifest-bc?part-number=1')
]
self.assertEqual(status, '206 Partial Content')
self.assertEqual(headers['Etag'], '"%s"' % self.manifest_bc_slo_etag)
self.assertEqual(headers['Content-Length'], '10')
self.assertEqual(headers['Content-Range'], 'bytes 0-9/25')
self.assertEqual(headers['X-Manifest-Etag'], self.manifest_bc_json_md5)
self.assertEqual(headers['X-Static-Large-Object'], 'true')
self.assertEqual(headers['X-Parts-Count'], '2')
self.assertEqual(headers['Content-Type'], 'application/octet-stream')
self.assertEqual(body, b'') # it's a HEAD request, after all
self.assertEqual(headers['X-Object-Meta-Plant'], 'Ficus')
self.assertEqual(self.app.calls, expected_calls)
def test_head_part_number_refetch_path(self):
# verify that any modification of the request path by a downstream
# middleware is ignored when refetching
req = Request.blank(
'/v1/AUTH_test/gettest/mani?part-number=1',
environ={'REQUEST_METHOD': 'HEAD'})
captured_calls = []
orig_call = FakeSwift.__call__
def pseudo_middleware(app, env, start_response):
captured_calls.append((env['REQUEST_METHOD'], env['PATH_INFO']))
# pretend another middleware modified the path
# note: for convenience, the path "modification" actually results
# in one of the pre-registered paths
env['PATH_INFO'] += 'fest-bc'
return orig_call(app, env, start_response)
with patch.object(FakeSwift, '__call__', pseudo_middleware):
status, headers, body = self.call_slo(req)
# pseudo-middleware gets the original path for the refetch
self.assertEqual([('HEAD', '/v1/AUTH_test/gettest/mani'),
('GET', '/v1/AUTH_test/gettest/mani')],
captured_calls)
self.assertEqual(status, '206 Partial Content')
expected_calls = [
# original path is modified...
('HEAD', '/v1/AUTH_test/gettest/manifest-bc?part-number=1'),
# refetch: the *original* path is modified...
('GET', '/v1/AUTH_test/gettest/manifest-bc?part-number=1')
]
self.assertEqual(self.app.calls, expected_calls)
def test_get_part_number(self):
# part number 1 is b_10
req = Request.blank(
'/v1/AUTH_test/gettest/manifest-bc?part-number=1')
status, headers, body = self.call_slo(req)
expected_calls = [
('GET', '/v1/AUTH_test/gettest/manifest-bc?part-number=1'),
('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get')
]
self.assertEqual(status, '206 Partial Content')
self.assertEqual(headers['Etag'], '"%s"' % self.manifest_bc_slo_etag)
self.assertEqual(headers['X-Manifest-Etag'], self.manifest_bc_json_md5)
self.assertEqual(headers['Content-Length'], '10')
self.assertEqual(headers['Content-Range'], 'bytes 0-9/25')
self.assertEqual(headers['X-Static-Large-Object'], 'true')
self.assertEqual(headers['X-Parts-Count'], '2')
self.assertEqual(headers['Content-Type'], 'application/octet-stream')
self.assertEqual(body, b'b' * 10)
self.assertEqual(headers['X-Object-Meta-Plant'], 'Ficus')
self.assertEqual(self.app.calls, expected_calls)
# part number 2 is c_15
self.app.clear_calls()
expected_calls = [
('GET', '/v1/AUTH_test/gettest/manifest-bc?part-number=2'),
('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get')
]
req = Request.blank(
'/v1/AUTH_test/gettest/manifest-bc?part-number=2')
status, headers, body = self.call_slo(req)
self.assertEqual(status, '206 Partial Content')
self.assertEqual(headers['Etag'], '"%s"' % self.manifest_bc_slo_etag)
self.assertEqual(headers['X-Manifest-Etag'], self.manifest_bc_json_md5)
self.assertEqual(headers['Content-Length'], '15')
self.assertEqual(headers['Content-Range'], 'bytes 10-24/25')
self.assertEqual(headers['X-Static-Large-Object'], 'true')
self.assertEqual(headers['X-Parts-Count'], '2')
self.assertEqual(headers['Content-Type'], 'application/octet-stream')
self.assertEqual(body, b'c' * 15)
self.assertEqual(headers['X-Object-Meta-Plant'], 'Ficus')
self.assertEqual(self.app.calls, expected_calls)
# we now test it with single segment slo
self.app.clear_calls()
req = Request.blank(
'/v1/AUTH_test/gettest/manifest-single-segment?part-number=1')
status, headers, body = self.call_slo(req)
self.assertEqual(status, '206 Partial Content')
self.assertEqual(headers['Etag'], '"%s"' %
self.manifest_single_segment_slo_etag)
self.assertEqual(headers['X-Manifest-Etag'],
self.manifest_single_segment_json_md5)
self.assertEqual(headers['Content-Length'], '50')
self.assertEqual(headers['Content-Range'], 'bytes 0-49/50')
self.assertEqual(headers['X-Static-Large-Object'], 'true')
self.assertEqual(headers['X-Object-Meta-Nature'], 'Regular')
self.assertEqual(headers['X-Parts-Count'], '1')
self.assertEqual(headers['Content-Type'], 'application/octet-stream')
expected_calls = [
('GET', '/v1/AUTH_test/gettest/manifest-single-segment?'
'part-number=1'),
('GET', '/v1/AUTH_test/gettest/b_50?multipart-manifest=get')
]
self.assertEqual(self.app.calls, expected_calls)
def test_get_part_number_sub_slo(self):
req = Request.blank(
'/v1/AUTH_test/gettest/manifest-abcd?part-number=3')
status, headers, body = self.call_slo(req)
expected_calls = [
('GET', '/v1/AUTH_test/gettest/manifest-abcd?part-number=3'),
('GET', '/v1/AUTH_test/gettest/d_20?multipart-manifest=get')
]
self.assertEqual(status, '206 Partial Content')
self.assertEqual(headers['Etag'], '"%s"' % self.manifest_abcd_slo_etag)
self.assertEqual(headers['X-Manifest-Etag'],
self.manifest_abcd_json_md5)
self.assertEqual(headers['Content-Length'], '20')
self.assertEqual(headers['Content-Range'], 'bytes 30-49/50')
self.assertEqual(headers['X-Static-Large-Object'], 'true')
self.assertEqual(headers['X-Parts-Count'], '3')
self.assertEqual(headers['Content-Type'], 'application/json')
self.assertEqual(body, b'd' * 20)
self.assertEqual(self.app.calls, expected_calls)
self.app.clear_calls()
req = Request.blank(
'/v1/AUTH_test/gettest/manifest-abcd?part-number=2')
status, headers, body = self.call_slo(req)
expected_calls = [
('GET', '/v1/AUTH_test/gettest/manifest-abcd?part-number=2'),
('GET', '/v1/AUTH_test/gettest/manifest-bc'),
('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get'),
('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get')
]
self.assertEqual(status, '206 Partial Content')
self.assertEqual(headers['Etag'], '"%s"' % self.manifest_abcd_slo_etag)
self.assertEqual(headers['X-Manifest-Etag'],
self.manifest_abcd_json_md5)
self.assertEqual(headers['Content-Length'], '25')
self.assertEqual(headers['Content-Range'], 'bytes 5-29/50')
self.assertEqual(headers['X-Static-Large-Object'], 'true')
self.assertEqual(headers['X-Parts-Count'], '3')
self.assertEqual(headers['Content-Type'], 'application/json')
self.assertEqual(body, b'b' * 10 + b'c' * 15)
self.assertEqual(self.app.calls, expected_calls)
def test_get_part_number_large_manifest(self):
req = Request.blank(
'/v1/AUTH_test/gettest/manifest-abcdefghijkl?part-number=10')
status, headers, body = self.call_slo(req)
expected_calls = [
('GET', '/v1/AUTH_test/gettest/manifest-abcdefghijkl?'
'part-number=10'),
('GET', '/v1/AUTH_test/gettest/j_50?multipart-manifest=get')
]
self.assertEqual(status, '206 Partial Content')
self.assertEqual(headers['Etag'], '"%s"' %
self.manifest_abcdefghijkl_slo_etag)
self.assertEqual(headers['X-Manifest-Etag'],
self.manifest_abcdefghijkl_json_md5)
self.assertEqual(headers['Content-Length'], '50')
self.assertEqual(headers['Content-Range'], 'bytes 225-274/390')
self.assertEqual(headers['X-Static-Large-Object'], 'true')
self.assertEqual(headers['X-Parts-Count'], '12')
self.assertEqual(headers['Content-Type'], 'application/octet-stream')
self.assertEqual(body, b'j' * 50)
self.assertEqual(self.app.calls, expected_calls)
def test_part_number_with_range_segments(self):
req = Request.blank('/v1/AUTH_test/gettest/manifest-bc-ranges',
params={'part-number': 1})
status, headers, body = self.call_slo(req)
self.assertEqual(status, '206 Partial Content')
self.assertEqual(headers['Etag'], '"%s"' %
self.manifest_bc_ranges_slo_etag)
self.assertEqual(headers['X-Manifest-Etag'],
self.manifest_bc_ranges_json_md5)
self.assertEqual(headers['Content-Length'], '4')
self.assertEqual(headers['Content-Range'],
'bytes 0-3/%s' % self.manifest_bc_ranges_slo_size)
self.assertEqual(headers['X-Static-Large-Object'], 'true')
self.assertEqual(headers['X-Parts-Count'], '4')
self.assertEqual(body, b'b' * 4)
expected_calls = [
('GET', '/v1/AUTH_test/gettest/manifest-bc-ranges?part-number=1'),
('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get')
]
self.assertEqual(self.app.calls, expected_calls)
# since the our requested part-number is range-segment we expect Range
# header on b_10 segment subrequest
self.assertEqual('bytes=4-7',
self.app.calls_with_headers[1].headers['Range'])
def test_part_number_sub_ranges_manifest(self):
req = Request.blank(
'/v1/AUTH_test/gettest/manifest-abcd-subranges?part-number=3')
status, headers, body = self.call_slo(req)
expected_calls = [
('GET', '/v1/AUTH_test/gettest/manifest-abcd-subranges?'
'part-number=3'),
('GET', '/v1/AUTH_test/gettest/manifest-abcd-ranges'),
('GET', '/v1/AUTH_test/gettest/manifest-bc-ranges'),
('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get'),
('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get')
]
self.assertEqual(status, '206 Partial Content')
self.assertEqual(headers['Etag'], '"%s"' %
self.manifest_abcd_subranges_slo_etag)
self.assertEqual(headers['X-Manifest-Etag'],
self.manifest_abcd_subranges_json_md5)
self.assertEqual(headers['Content-Length'], '5')
self.assertEqual(headers['Content-Range'], 'bytes 6-10/17')
self.assertEqual(headers['X-Static-Large-Object'], 'true')
self.assertEqual(headers['X-Parts-Count'], '5')
self.assertEqual(headers['Content-Type'], 'application/json')
self.assertEqual(body, b'c' * 2 + b'b' * 3)
self.assertEqual(self.app.calls, expected_calls)
def test_get_part_num_with_repeated_segments(self):
req = Request.blank(
'/v1/AUTH_test/gettest/manifest-aabbccdd?part-number=3',
environ={'REQUEST_METHOD': 'GET'})
status, headers, body = self.call_slo(req)
expected_calls = [
('GET', '/v1/AUTH_test/gettest/manifest-aabbccdd?part-number=3'),
('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get')
]
self.assertEqual(status, '206 Partial Content')
self.assertEqual(headers['Etag'], '"%s"' %
self.manifest_aabbccdd_slo_etag)
self.assertEqual(headers['X-Manifest-Etag'],
self.manifest_aabbccdd_json_md5)
self.assertEqual(headers['Content-Length'], '10')
self.assertEqual(headers['Content-Range'], 'bytes 10-19/100')
self.assertEqual(headers['X-Static-Large-Object'], 'true')
self.assertEqual(headers['X-Parts-Count'], '8')
self.assertEqual(headers['Content-Type'], 'application/octet-stream')
self.assertEqual(body, b'b' * 10)
self.assertEqual(self.app.calls, expected_calls)
def test_part_number_zero_invalid(self):
# part-number query param is 1-indexed, part-number=0 is no joy
req = Request.blank(
'/v1/AUTH_test/gettest/manifest-bc?part-number=0')
status, headers, body = self.call_slo(req)
self.assertEqual(status, '400 Bad Request')
self.assertNotIn('Content-Range', headers)
self.assertNotIn('Etag', headers)
self.assertNotIn('X-Static-Large-Object', headers)
self.assertNotIn('X-Parts-Count', headers)
self.assertEqual(body,
b'Part number must be an integer greater than 0')
expected_calls = [
('GET', '/v1/AUTH_test/gettest/manifest-bc?part-number=0')
]
self.assertEqual(expected_calls, self.app.calls)
self.app.clear_calls()
self.slo.max_manifest_segments = 3999
req = Request.blank('/v1/AUTH_test/gettest/manifest-bc',
params={'part-number': 0})
status, headers, body = self.call_slo(req)
self.assertEqual(status, '400 Bad Request')
self.assertNotIn('Content-Range', headers)
self.assertNotIn('Etag', headers)
self.assertNotIn('X-Static-Large-Object', headers)
self.assertNotIn('X-Parts-Count', headers)
self.assertEqual(body,
b'Part number must be an integer greater than 0')
self.assertEqual(expected_calls, self.app.calls)
def test_head_part_number_zero_invalid(self):
# you can HEAD part-number=0 either
req = Request.blank(
'/v1/AUTH_test/gettest/manifest-bc', method='HEAD',
params={'part-number': 0})
status, headers, body = self.call_slo(req)
self.assertEqual(status, '400 Bad Request')
self.assertNotIn('Content-Range', headers)
self.assertNotIn('Etag', headers)
self.assertNotIn('X-Static-Large-Object', headers)
self.assertNotIn('X-Parts-Count', headers)
self.assertEqual(body, b'') # HEAD response, makes sense
expected_calls = [
('HEAD', '/v1/AUTH_test/gettest/manifest-bc?part-number=0')
]
self.assertEqual(expected_calls, self.app.calls)
def test_part_number_zero_invalid_on_subrange(self):
# either manifest, doesn't matter, part-number=0 is always invalid
req = Request.blank(
'/v1/AUTH_test/gettest/manifest-abcd-subranges?part-number=0')
status, headers, body = self.call_slo(req)
self.assertEqual(status, '400 Bad Request')
self.assertNotIn('Content-Range', headers)
self.assertNotIn('Etag', headers)
self.assertNotIn('X-Static-Large-Object', headers)
self.assertNotIn('X-Parts-Count', headers)
self.assertEqual(body,
b'Part number must be an integer greater than 0')
expected_calls = [
('GET',
'/v1/AUTH_test/gettest/manifest-abcd-subranges?part-number=0')
]
self.assertEqual(expected_calls, self.app.calls)
def test_negative_part_number_invalid(self):
# negative numbers are never any good
req = Request.blank(
'/v1/AUTH_test/gettest/manifest-bc?part-number=-1')
status, headers, body = self.call_slo(req)
self.assertEqual(status, '400 Bad Request')
self.assertNotIn('Content-Range', headers)
self.assertNotIn('Etag', headers)
self.assertNotIn('X-Static-Large-Object', headers)
self.assertNotIn('X-Parts-Count', headers)
self.assertEqual(body,
b'Part number must be an integer greater than 0')
expected_calls = [
('GET', '/v1/AUTH_test/gettest/manifest-bc?part-number=-1')
]
self.assertEqual(expected_calls, self.app.calls)
def test_head_negative_part_number_invalid_on_subrange(self):
req = Request.blank(
'/v1/AUTH_test/gettest/manifest-abcd-subranges', method='HEAD',
params={'part-number': '-1'})
status, headers, body = self.call_slo(req)
self.assertEqual(status, '400 Bad Request')
self.assertNotIn('Content-Range', headers)
self.assertNotIn('Etag', headers)
self.assertNotIn('X-Static-Large-Object', headers)
self.assertNotIn('X-Parts-Count', headers)
self.assertEqual(body, b'')
expected_calls = [
('HEAD',
'/v1/AUTH_test/gettest/manifest-abcd-subranges?part-number=-1')
]
self.assertEqual(expected_calls, self.app.calls)
def test_head_non_integer_part_number_invalid(self):
# some kind of string is bad too
req = Request.blank(
'/v1/AUTH_test/gettest/manifest-bc', method='HEAD',
params={'part-number': 'foo'})
status, headers, body = self.call_slo(req)
self.assertEqual(status, '400 Bad Request')
self.assertEqual(body, b'')
expected_calls = [
('HEAD', '/v1/AUTH_test/gettest/manifest-bc?part-number=foo')
]
self.assertEqual(expected_calls, self.app.calls)
def test_get_non_integer_part_number_invalid(self):
req = Request.blank(
'/v1/AUTH_test/gettest/manifest-bc', params={'part-number': 'foo'})
status, headers, body = self.call_slo(req)
self.assertEqual(status, '400 Bad Request')
self.assertNotIn('Content-Range', headers)
self.assertNotIn('Etag', headers)
self.assertNotIn('X-Static-Large-Object', headers)
self.assertNotIn('X-Parts-Count', headers)
self.assertEqual(body, b'Part number must be an integer greater'
b' than 0')
expected_calls = [
('GET', '/v1/AUTH_test/gettest/manifest-bc?part-number=foo')
]
self.assertEqual(expected_calls, self.app.calls)
def test_get_out_of_range_part_number(self):
# you can't go past the actual number of parts either
req = Request.blank(
'/v1/AUTH_test/gettest/manifest-bc?part-number=4')
status, headers, body = self.call_slo(req)
self.assertEqual(status, '416 Requested Range Not Satisfiable')
self.assertEqual(headers['Content-Range'],
'bytes */%d' % self.manifest_bc_slo_size)
self.assertEqual(headers['Etag'], '"%s"' % self.manifest_bc_slo_etag)
self.assertEqual(headers['X-Manifest-Etag'], self.manifest_bc_json_md5)
self.assertEqual(headers['X-Static-Large-Object'], 'true')
self.assertEqual(headers['X-Parts-Count'], '2')
self.assertEqual(int(headers['Content-Length']), len(body))
self.assertEqual(body, b'The requested part number is not '
b'satisfiable')
self.assertEqual(headers['X-Object-Meta-Plant'], 'Ficus')
expected_app_calls = [
('GET', '/v1/AUTH_test/gettest/manifest-bc?part-number=4'),
]
self.assertEqual(self.app.calls, expected_app_calls)
self.app.clear_calls()
req = Request.blank(
'/v1/AUTH_test/gettest/manifest-single-segment?part-number=2')
status, headers, body = self.call_slo(req)
self.assertEqual(status, '416 Requested Range Not Satisfiable')
self.assertEqual(headers['Content-Range'],
'bytes */%d' % self.manifest_single_segment_slo_size)
self.assertEqual(int(headers['Content-Length']), len(body))
self.assertEqual(headers['Etag'],
'"%s"' % self.manifest_single_segment_slo_etag)
self.assertEqual(headers['X-Manifest-Etag'],
self.manifest_single_segment_json_md5)
self.assertEqual(headers['X-Static-Large-Object'], 'true')
self.assertEqual(headers['X-Parts-Count'], '1')
self.assertEqual(body, b'The requested part number is not '
b'satisfiable')
self.assertEqual(headers['X-Object-Meta-Nature'], 'Regular')
expected_app_calls = [
('GET', '/v1/AUTH_test/gettest/manifest-single-segment?'
'part-number=2'),
]
self.assertEqual(self.app.calls, expected_app_calls)
def test_head_out_of_range_part_number(self):
req = Request.blank(
'/v1/AUTH_test/gettest/manifest-bc?part-number=4')
req.method = 'HEAD'
status, headers, body = self.call_slo(req)
self.assertEqual(status, '416 Requested Range Not Satisfiable')
self.assertEqual(headers['Content-Range'],
'bytes */%d' % self.manifest_bc_slo_size)
self.assertEqual(headers['Etag'], '"%s"' % self.manifest_bc_slo_etag)
self.assertEqual(headers['X-Manifest-Etag'], self.manifest_bc_json_md5)
self.assertEqual(headers['X-Static-Large-Object'], 'true')
self.assertEqual(headers['X-Parts-Count'], '2')
self.assertEqual(int(headers['Content-Length']), len(body))
self.assertEqual(body, b'')
self.assertEqual(headers['X-Object-Meta-Plant'], 'Ficus')
expected_app_calls = [
('HEAD', '/v1/AUTH_test/gettest/manifest-bc?part-number=4'),
# segments needed
('GET', '/v1/AUTH_test/gettest/manifest-bc?part-number=4')
]
self.assertEqual(self.app.calls, expected_app_calls)
def test_part_number_exceeds_max_manifest_segments_is_ok(self):
# verify that an existing part can be fetched regardless of the current
# max_manifest_segments
req = Request.blank(
'/v1/AUTH_test/gettest/manifest-bc?part-number=2')
self.slo.max_manifest_segments = 1
status, headers, body = self.call_slo(req)
self.assertEqual(status, '206 Partial Content')
self.assertEqual(headers['Etag'], '"%s"' % self.manifest_bc_slo_etag)
self.assertEqual(headers['X-Manifest-Etag'], self.manifest_bc_json_md5)
self.assertEqual(headers['Content-Length'], '15')
self.assertEqual(headers['Content-Range'], 'bytes 10-24/25')
self.assertEqual(headers['X-Static-Large-Object'], 'true')
self.assertEqual(headers['X-Parts-Count'], '2')
self.assertEqual(headers['Content-Type'], 'application/octet-stream')
self.assertEqual(body, b'c' * 15)
expected_calls = [
('GET', '/v1/AUTH_test/gettest/manifest-bc?part-number=2'),
('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get')
]
self.assertEqual(self.app.calls, expected_calls)
def test_part_number_ignored_for_non_slo_object(self):
# verify that a part-number param is ignored for a non-slo object
def do_test(query_string):
self.app.clear_calls()
req = Request.blank(
'/v1/AUTH_test/gettest/c_15?%s' % query_string)
self.slo.max_manifest_segments = 1
status, headers, body = self.call_slo(req)
self.assertEqual(status, '200 OK')
self.assertEqual(headers['Etag'], '%s' % md5hex('c' * 15))
self.assertEqual(headers['Content-Length'], '15')
self.assertEqual(body, b'c' * 15)
self.assertEqual(1, self.app.call_count)
method, path = self.app.calls[0]
actual_req = Request.blank(path, method=method)
self.assertEqual(req.path, actual_req.path)
self.assertEqual(req.params, actual_req.params)
do_test('part-number=-1')
do_test('part-number=0')
do_test('part-number=1')
do_test('part-number=2')
do_test('part-number=foo')
do_test('part-number=foo&multipart-manifest=get')
def test_part_number_ignored_for_non_slo_object_with_range(self):
# verify that a part-number param is ignored for a non-slo object
def do_test(query_string):
self.app.clear_calls()
req = Request.blank(
'/v1/AUTH_test/gettest/c_15?%s' % query_string,
headers={'Range': 'bytes=1-2'})
self.slo.max_manifest_segments = 1
status, headers, body = self.call_slo(req)
self.assertEqual(status, '206 Partial Content')
self.assertEqual(headers['Etag'], '%s' % md5hex('c' * 15))
self.assertEqual(headers['Content-Length'], '2')
self.assertEqual(headers['Content-Range'], 'bytes 1-2/15')
self.assertEqual(body, b'c' * 2)
self.assertEqual(1, self.app.call_count)
method, path = self.app.calls[0]
actual_req = Request.blank(path, method=method)
self.assertEqual(req.path, actual_req.path)
self.assertEqual(req.params, actual_req.params)
do_test('part-number=-1')
do_test('part-number=0')
do_test('part-number=1')
do_test('part-number=2')
do_test('part-number=foo')
do_test('part-number=foo&multipart-manifest=get')
def test_part_number_ignored_for_manifest_get(self):
def do_test(query_string):
self.app.clear_calls()
req = Request.blank(
'/v1/AUTH_test/gettest/manifest-bc?%s' % query_string)
self.slo.max_manifest_segments = 1
status, headers, body = self.call_slo(req)
self.assertEqual(status, '200 OK')
self.assertEqual(headers['Etag'], self.manifest_bc_json_md5)
self.assertEqual(headers['Content-Length'],
str(self.manifest_bc_json_size))
self.assertEqual(headers['X-Static-Large-Object'], 'true')
self.assertEqual(headers['Content-Type'],
'application/json; charset=utf-8')
self.assertEqual(headers['X-Object-Meta-Plant'], 'Ficus')
self.assertEqual(1, self.app.call_count)
method, path = self.app.calls[0]
actual_req = Request.blank(path, method=method)
self.assertEqual(req.path, actual_req.path)
self.assertEqual(req.params, actual_req.params)
do_test('part-number=-1&multipart-manifest=get')
do_test('part-number=0&multipart-manifest=get')
do_test('part-number=1&multipart-manifest=get')
do_test('part-number=2&multipart-manifest=get')
do_test('part-number=foo&multipart-manifest=get')
def test_head_out_of_range_part_number_on_subrange(self):
# you can't go past the actual number of parts either
req = Request.blank(
'/v1/AUTH_test/gettest/manifest-abcd-subranges',
method='HEAD',
params={'part-number': 6})
expected_calls = [
('HEAD', '/v1/AUTH_test/gettest/manifest-abcd-subranges?'
'part-number=6'),
('GET', '/v1/AUTH_test/gettest/manifest-abcd-subranges?'
'part-number=6')]
status, headers, body = self.call_slo(req)
self.assertEqual(status, '416 Requested Range Not Satisfiable')
self.assertEqual(headers['Content-Range'],
'bytes */%d' % self.manifest_abcd_subranges_slo_size)
self.assertEqual(headers['Etag'],
'"%s"' % self.manifest_abcd_subranges_slo_etag)
self.assertEqual(headers['X-Manifest-Etag'],
self.manifest_abcd_subranges_json_md5)
self.assertEqual(headers['X-Static-Large-Object'], 'true')
self.assertEqual(headers['X-Parts-Count'], '5')
self.assertEqual(int(headers['Content-Length']), len(body))
self.assertEqual(body, b'')
self.assertEqual(self.app.calls, expected_calls)
def test_range_with_part_number_is_error(self):
req = Request.blank(
'/v1/AUTH_test/gettest/manifest-abcd-subranges?part-number=2',
headers={'Range': 'bytes=4-12'})
status, headers, body = self.call_slo(req)
self.assertEqual(status, '400 Bad Request')
self.assertNotIn('Content-Range', headers)
self.assertNotIn('Etag', headers)
self.assertNotIn('X-Static-Large-Object', headers)
self.assertNotIn('X-Parts-Count', headers)
self.assertEqual(body, b'Range requests are not supported with '
b'part number queries')
expected_calls = [
('GET',
'/v1/AUTH_test/gettest/manifest-abcd-subranges?part-number=2')
]
self.assertEqual(expected_calls, self.app.calls)
def test_head_part_number_subrange(self):
req = Request.blank(
'/v1/AUTH_test/gettest/manifest-abcd-subranges',
method='HEAD', params={'part-number': 2})
status, headers, body = self.call_slo(req)
# Range header can be ignored in a HEAD request
self.assertEqual(status, '206 Partial Content')
self.assertEqual(headers['Etag'],
'"%s"' % self.manifest_abcd_subranges_slo_etag)
self.assertEqual(headers['Content-Length'], '1')
self.assertEqual(headers['X-Static-Large-Object'], 'true')
self.assertEqual(headers['Content-Type'], 'application/json')
self.assertEqual(headers['X-Parts-Count'], '5')
self.assertEqual(body, b'') # it's a HEAD request, after all
expected_calls = [
('HEAD', '/v1/AUTH_test/gettest/manifest-abcd-subranges'
'?part-number=2'),
('GET', '/v1/AUTH_test/gettest/manifest-abcd-subranges'
'?part-number=2'),
]
self.assertEqual(self.app.calls, expected_calls)
def test_head_part_number_data_manifest(self):
req = Request.blank(
'/v1/AUTH_test/c/manifest-data',
method='HEAD', params={'part-number': 1})
status, headers, body = self.call_slo(req)
self.assertEqual(status, '206 Partial Content')
self.assertEqual(headers['Etag'],
'"%s"' % self.manifest_data_slo_etag)
self.assertEqual(headers['Content-Length'], '6')
self.assertEqual(headers['X-Static-Large-Object'], 'true')
self.assertEqual(headers['X-Parts-Count'], '3')
self.assertEqual(body, b'') # it's a HEAD request, after all
expected_calls = [
('HEAD', '/v1/AUTH_test/c/manifest-data?part-number=1'),
('GET', '/v1/AUTH_test/c/manifest-data?part-number=1'),
]
self.assertEqual(self.app.calls, expected_calls)
def test_get_part_number_data_manifest(self):
req = Request.blank(
'/v1/AUTH_test/c/manifest-data',
params={'part-number': 3})
status, headers, body = self.call_slo(req)
self.assertEqual(status, '206 Partial Content')
self.assertEqual(headers['Etag'],
'"%s"' % self.manifest_data_slo_etag)
self.assertEqual(headers['Content-Length'], '6')
self.assertEqual(headers['X-Static-Large-Object'], 'true')
self.assertEqual(headers['X-Parts-Count'], '3')
self.assertEqual(body, b'ABCDEF')
expected_calls = [
('GET', '/v1/AUTH_test/c/manifest-data?part-number=3'),
]
self.assertEqual(self.app.calls, expected_calls)
class TestPartNumberLegacyManifest(TestPartNumber):
modern_manifest_headers = False
class TestSloBulkDeleter(unittest.TestCase): class TestSloBulkDeleter(unittest.TestCase):
def test_reused_logger(self): def test_reused_logger(self):
slo_mware = slo.filter_factory({})('fake app') slo_mware = slo.filter_factory({})('fake app')