diff --git a/swift/common/middleware/s3api/acl_handlers.py b/swift/common/middleware/s3api/acl_handlers.py
index 699b387713..ebb7acdee1 100644
--- a/swift/common/middleware/s3api/acl_handlers.py
+++ b/swift/common/middleware/s3api/acl_handlers.py
@@ -133,6 +133,15 @@ class BaseAclHandler(object):
query = {}
else:
query = {'version-id': version_id}
+ if self.req.method == 'HEAD':
+ # This HEAD for ACL is going to also be the definitive response
+ # to the client so we need to include client params. We don't
+ # do this for other client request methods because they may
+ # have invalid combinations of params and headers for a swift
+ # HEAD request.
+ part_number = self.req.params.get('partNumber')
+ if part_number is not None:
+ query['part-number'] = part_number
resp = self.req.get_acl_response(app, 'HEAD',
container, obj,
headers, query=query)
diff --git a/swift/common/middleware/s3api/controllers/multi_upload.py b/swift/common/middleware/s3api/controllers/multi_upload.py
index 6bfa6eed51..735d2be7cd 100644
--- a/swift/common/middleware/s3api/controllers/multi_upload.py
+++ b/swift/common/middleware/s3api/controllers/multi_upload.py
@@ -196,15 +196,7 @@ class PartController(Controller):
raise InvalidArgument('ResourceType', 'partNumber',
'Unexpected query string parameter')
- try:
- part_number = int(get_param(req, 'partNumber'))
- if part_number < 1 or self.conf.max_upload_part_num < part_number:
- raise Exception()
- except Exception:
- err_msg = 'Part number must be an integer between 1 and %d,' \
- ' inclusive' % self.conf.max_upload_part_num
- raise InvalidArgument('partNumber', get_param(req, 'partNumber'),
- err_msg)
+ part_number = req.validate_part_number()
upload_id = get_param(req, 'uploadId')
_get_upload_info(req, self.app, upload_id)
diff --git a/swift/common/middleware/s3api/controllers/obj.py b/swift/common/middleware/s3api/controllers/obj.py
index b1f310e6e6..c116711593 100644
--- a/swift/common/middleware/s3api/controllers/obj.py
+++ b/swift/common/middleware/s3api/controllers/obj.py
@@ -90,8 +90,14 @@ class ObjectController(Controller):
if version_id not in ('null', None) and \
'object_versioning' not in get_swift_info():
raise S3NotImplemented()
+ part_number = req.validate_part_number(check_max=False)
+
+ query = {}
+ if version_id is not None:
+ query['version-id'] = version_id
+ if part_number is not None:
+ query['part-number'] = part_number
- query = {} if version_id is None else {'version-id': version_id}
if version_id not in ('null', None):
container_info = req.get_container_info(self.app)
if not container_info.get(
@@ -101,6 +107,19 @@ class ObjectController(Controller):
resp = req.get_response(self.app, query=query)
+ if not resp.is_slo:
+ # SLO ignores part_number for non-slo objects, but s3api only
+ # allows the query param for non-MPU if it's exactly 1.
+ part_number = req.validate_part_number(parts_count=1)
+ if part_number == 1:
+ # When the query param *is* exactly 1 the response status code
+ # and headers are updated.
+ resp.status = HTTP_PARTIAL_CONTENT
+ resp.headers['Content-Range'] = \
+ 'bytes 0-%d/%s' % (int(resp.headers['Content-Length']) - 1,
+ resp.headers['Content-Length'])
+ # else: part_number is None
+
if req.method == 'HEAD':
resp.app_iter = None
diff --git a/swift/common/middleware/s3api/s3request.py b/swift/common/middleware/s3api/s3request.py
index da5220046d..dcc0fb361b 100644
--- a/swift/common/middleware/s3api/s3request.py
+++ b/swift/common/middleware/s3api/s3request.py
@@ -56,7 +56,8 @@ from swift.common.middleware.s3api.s3response import AccessDenied, \
MissingContentLength, InvalidStorageClass, S3NotImplemented, InvalidURI, \
MalformedXML, InvalidRequest, RequestTimeout, InvalidBucketName, \
BadDigest, AuthorizationHeaderMalformed, SlowDown, \
- AuthorizationQueryParametersError, ServiceUnavailable, BrokenMPU
+ AuthorizationQueryParametersError, ServiceUnavailable, BrokenMPU, \
+ InvalidPartNumber, InvalidPartArgument
from swift.common.middleware.s3api.exception import NotS3Request
from swift.common.middleware.s3api.utils import utf8encode, \
S3Timestamp, mktime, MULTIUPLOAD_SUFFIX
@@ -558,6 +559,57 @@ class S3Request(swob.Request):
# by full URL when absolute path given. See swift.swob for more detail.
self.environ['swift.leave_relative_location'] = True
+ def validate_part_number(self, parts_count=None, check_max=True):
+ """
+ Get the partNumber param, if it exists, and check it is valid.
+
+ To be valid, a partNumber must satisfy two criteria. First, it must be
+ an integer between 1 and the maximum allowed parts, inclusive. The
+ maximum allowed parts is the maximum of the configured
+ ``max_upload_part_num`` and, if given, ``parts_count``. Second, the
+ partNumber must be less than or equal to the ``parts_count``, if it is
+ given.
+
+ :param parts_count: if given, this is the number of parts in an
+ existing object.
+ :raises InvalidPartArgument: if the partNumber param is invalid i.e.
+ less than 1 or greater than the maximum allowed parts.
+ :raises InvalidPartNumber: if the partNumber param is valid but greater
+ than ``num_parts``.
+ :return: an integer part number if the partNumber param exists,
+ otherwise ``None``.
+ """
+ part_number = self.params.get('partNumber')
+ if part_number is None:
+ return None
+
+ if self.range:
+ raise InvalidRequest('Cannot specify both Range header and '
+ 'partNumber query parameter')
+
+ try:
+ parts_count = int(parts_count)
+ except (TypeError, ValueError):
+ # an invalid/empty param is treated like parts_count=max_parts
+ parts_count = self.conf.max_upload_part_num
+ # max_parts may be raised to the number of existing parts
+ max_parts = max(self.conf.max_upload_part_num, parts_count)
+
+ try:
+ part_number = int(part_number)
+ if part_number < 1:
+ raise ValueError
+ except ValueError:
+ raise InvalidPartArgument(max_parts, part_number) # 400
+
+ if check_max:
+ if part_number > max_parts:
+ raise InvalidPartArgument(max_parts, part_number) # 400
+ if part_number > parts_count:
+ raise InvalidPartNumber() # 416
+
+ return part_number
+
def check_signature(self, secret):
secret = utf8encode(secret)
user_signature = self.signature
@@ -1044,7 +1096,10 @@ class S3Request(swob.Request):
if 'logging' in self.params:
return LoggingStatusController
if 'partNumber' in self.params:
- return PartController
+ if self.method == 'PUT':
+ return PartController
+ else:
+ return ObjectController
if 'uploadId' in self.params:
return UploadController
if 'uploads' in self.params:
@@ -1315,7 +1370,6 @@ class S3Request(swob.Request):
'GET': {
HTTP_NOT_FOUND: not_found_handler,
HTTP_PRECONDITION_FAILED: PreconditionFailed,
- HTTP_REQUESTED_RANGE_NOT_SATISFIABLE: InvalidRange,
},
'PUT': {
HTTP_NOT_FOUND: (NoSuchBucket, container),
@@ -1414,7 +1468,7 @@ class S3Request(swob.Request):
raise InvalidArgument('X-Delete-At',
self.headers['X-Delete-At'],
err_str)
- if 'X-Delete-After' in err_msg.decode('utf8'):
+ if 'X-Delete-After' in err_str:
raise InvalidArgument('X-Delete-After',
self.headers['X-Delete-After'],
err_str)
@@ -1425,6 +1479,10 @@ class S3Request(swob.Request):
**self.signature_does_not_match_kwargs())
if status == HTTP_FORBIDDEN:
raise AccessDenied(reason='forbidden')
+ if status == HTTP_REQUESTED_RANGE_NOT_SATISFIABLE:
+ self.validate_part_number(
+ parts_count=resp.headers.get('x-amz-mp-parts-count'))
+ raise InvalidRange()
if status == HTTP_SERVICE_UNAVAILABLE:
raise ServiceUnavailable()
if status in (HTTP_RATE_LIMITED, HTTP_TOO_MANY_REQUESTS):
diff --git a/swift/common/middleware/s3api/s3response.py b/swift/common/middleware/s3api/s3response.py
index bc1aff7c0a..bcd2240c33 100644
--- a/swift/common/middleware/s3api/s3response.py
+++ b/swift/common/middleware/s3api/s3response.py
@@ -72,6 +72,8 @@ def translate_swift_to_s3(key, val):
return key, val
elif _key == 'x-object-version-id':
return 'x-amz-version-id', val
+ elif _key == 'x-parts-count':
+ return 'x-amz-mp-parts-count', val
elif _key == 'x-copied-from-version-id':
return 'x-amz-copy-source-version-id', val
elif _key == 'x-backend-content-type' and \
@@ -449,6 +451,17 @@ class InvalidObjectState(ErrorResponse):
_msg = 'The operation is not valid for the current state of the object.'
+class InvalidPartArgument(InvalidArgument):
+ _code = 'InvalidArgument'
+
+ def __init__(self, max_parts, value):
+ err_msg = ('Part number must be an integer between '
+ '1 and %s, inclusive' % max_parts)
+ super(InvalidArgument, self).__init__(err_msg,
+ argument_name='partNumber',
+ argument_value=value)
+
+
class InvalidPart(ErrorResponse):
_status = '400 Bad Request'
_msg = 'One or more of the specified parts could not be found. The part ' \
@@ -478,6 +491,11 @@ class InvalidRange(ErrorResponse):
_msg = 'The requested range cannot be satisfied.'
+class InvalidPartNumber(ErrorResponse):
+ _status = '416 Requested Range Not Satisfiable'
+ _msg = 'The requested partnumber is not satisfiable'
+
+
class InvalidRequest(ErrorResponse):
_status = '400 Bad Request'
_msg = 'Invalid Request.'
diff --git a/swift/common/middleware/s3api/utils.py b/swift/common/middleware/s3api/utils.py
index 40ff9388f6..2d535cd008 100644
--- a/swift/common/middleware/s3api/utils.py
+++ b/swift/common/middleware/s3api/utils.py
@@ -172,6 +172,7 @@ class Config(dict):
'allow_no_owner': False,
'allowable_clock_skew': 900,
'ratelimit_as_client_error': False,
+ 'max_upload_part_num': 1000,
}
def __init__(self, base=None):
diff --git a/test/s3api/__init__.py b/test/s3api/__init__.py
index 45a2a84a12..ad5bf0fd6d 100644
--- a/test/s3api/__init__.py
+++ b/test/s3api/__init__.py
@@ -151,12 +151,10 @@ class BaseS3TestCase(unittest.TestCase):
# Default to v4 signatures (as aws-cli does), but subclasses can override
signature_version = 's3v4'
- @classmethod
- def get_s3_client(cls, user):
- return get_s3_client(user, cls.signature_version)
+ def get_s3_client(self, user):
+ return get_s3_client(user, self.signature_version)
- @classmethod
- def _remove_all_object_versions_from_bucket(cls, client, bucket_name):
+ def _remove_all_object_versions_from_bucket(self, client, bucket_name):
resp = client.list_object_versions(Bucket=bucket_name)
objs_to_delete = (resp.get('Versions', []) +
resp.get('DeleteMarkers', []))
@@ -182,11 +180,10 @@ class BaseS3TestCase(unittest.TestCase):
objs_to_delete = (resp.get('Versions', []) +
resp.get('DeleteMarkers', []))
- @classmethod
- def clear_bucket(cls, client, bucket_name):
+ def clear_bucket(self, client, bucket_name):
timeout = time.time() + 10
backoff = 0.1
- cls._remove_all_object_versions_from_bucket(client, bucket_name)
+ self._remove_all_object_versions_from_bucket(client, bucket_name)
try:
client.delete_bucket(Bucket=bucket_name)
except ClientError as e:
@@ -199,7 +196,7 @@ class BaseS3TestCase(unittest.TestCase):
Bucket=bucket_name,
VersioningConfiguration={'Status': 'Suspended'})
while True:
- cls._remove_all_object_versions_from_bucket(
+ self._remove_all_object_versions_from_bucket(
client, bucket_name)
# also try some version-unaware operations...
for key in client.list_objects(Bucket=bucket_name).get(
@@ -224,13 +221,12 @@ class BaseS3TestCase(unittest.TestCase):
def create_name(self, slug):
return '%s%s-%s' % (TEST_PREFIX, slug, uuid.uuid4().hex)
- @classmethod
- def clear_account(cls, client):
+ def clear_account(self, client):
for bucket in client.list_buckets()['Buckets']:
if not bucket['Name'].startswith(TEST_PREFIX):
# these tests run against real s3 accounts
continue
- cls.clear_bucket(client, bucket['Name'])
+ self.clear_bucket(client, bucket['Name'])
def tearDown(self):
client = self.get_s3_client(1)
diff --git a/test/s3api/test_mpu.py b/test/s3api/test_mpu.py
index 26e550354e..28e7f0313a 100644
--- a/test/s3api/test_mpu.py
+++ b/test/s3api/test_mpu.py
@@ -17,7 +17,7 @@ from test.s3api import BaseS3TestCase
from botocore.exceptions import ClientError
-class TestMultiPartUploads(BaseS3TestCase):
+class BaseMultiPartUploadTestCase(BaseS3TestCase):
maxDiff = None
@@ -26,10 +26,157 @@ class TestMultiPartUploads(BaseS3TestCase):
self.bucket_name = self.create_name('test-mpu')
resp = self.client.create_bucket(Bucket=self.bucket_name)
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
+ self.num_parts = 3
+ self.part_size = 5 * (2 ** 20) # 5 MB
def tearDown(self):
self.clear_bucket(self.client, self.bucket_name)
- super(TestMultiPartUploads, self).tearDown()
+ super(BaseMultiPartUploadTestCase, self).tearDown()
+
+ def _make_part_bodies(self):
+ return [
+ ('%d' % i) * self.part_size
+ for i in range(self.num_parts)
+ ]
+
+ def _iter_part_num_ranges(self):
+ for i in range(self.num_parts):
+ start = self.part_size * i
+ end = start + self.part_size
+ # part_num is 1 indexed
+ yield i + 1, start, end
+
+ def _upload_mpu(self, key_name):
+ create_mpu_resp = self.client.create_multipart_upload(
+ Bucket=self.bucket_name, Key=key_name)
+ self.assertEqual(200, create_mpu_resp[
+ 'ResponseMetadata']['HTTPStatusCode'])
+ upload_id = create_mpu_resp['UploadId']
+
+ part_bodies = self._make_part_bodies()
+ parts = []
+ for i, body in enumerate(part_bodies, 1):
+ part_resp = self.client.upload_part(
+ Body=body, Bucket=self.bucket_name, Key=key_name,
+ PartNumber=i, UploadId=upload_id)
+ self.assertEqual(200, part_resp[
+ 'ResponseMetadata']['HTTPStatusCode'])
+ parts.append({
+ 'ETag': part_resp['ETag'],
+ 'PartNumber': i,
+ })
+ # this helper doesn't bother calling list-parts, it's not required
+ # and we know what we uploaded
+ complete_mpu_resp = self.client.complete_multipart_upload(
+ Bucket=self.bucket_name, Key=key_name,
+ MultipartUpload={
+ 'Parts': parts,
+ },
+ UploadId=upload_id,
+ )
+ self.assertEqual(200, complete_mpu_resp[
+ 'ResponseMetadata']['HTTPStatusCode'])
+ return complete_mpu_resp
+
+ def upload_mpu_version(self, key_name):
+ complete_mpu_resp = self._upload_mpu(key_name)
+ # AWS returns the version_id *in* the MPU-complete response but s3api
+ # does NOT (see https://bugs.launchpad.net/swift/+bug/2043619), so we
+ # do an extra HEAD to get the version
+ head_object_resp = self.client.head_object(
+ Bucket=self.bucket_name, Key=key_name)
+
+ self.assertEqual(200, head_object_resp[
+ 'ResponseMetadata']['HTTPStatusCode'])
+
+ return complete_mpu_resp['ETag'], head_object_resp.get('VersionId')
+
+ def upload_mpu(self, key_name):
+ complete_mpu_resp = self._upload_mpu(key_name)
+ return complete_mpu_resp['ETag']
+
+ def _verify_part_num_response(self, method, key_name, mpu_etag,
+ version=None):
+ part_bodies = self._make_part_bodies()
+ total_size = self.num_parts * self.part_size
+
+ for part_num, start, end in self._iter_part_num_ranges():
+ extra_kwargs = {}
+ if version is not None:
+ extra_kwargs['VersionId'] = version
+ resp = method(Bucket=self.bucket_name, Key=key_name,
+ PartNumber=part_num, **extra_kwargs)
+ self.assertEqual(206, resp['ResponseMetadata'][
+ 'HTTPStatusCode'])
+ self.assertEqual(self.part_size, resp['ContentLength'])
+ if method == self.client.get_object:
+ resp_body = b''.join(resp['Body']).decode()
+ # our part_bodies are zero indexed
+ self.assertEqual(resp_body, part_bodies[part_num - 1])
+ expected_range = 'bytes %s-%s/%s' % (
+ start, end - 1, total_size)
+ self.assertEqual(expected_range, resp['ContentRange'])
+ # ETag and PartsCount are from the MPU
+ self.assertEqual(mpu_etag, resp['ETag'], mpu_etag)
+ self.assertEqual(self.num_parts, resp['PartsCount'])
+ self.assertEqual('bytes', resp['AcceptRanges'])
+ if version is None:
+ self.assertNotIn('VersionId', resp)
+ else:
+ self.assertEqual(version, resp['VersionId'])
+
+ def _verify_copy_parts(self, key_src, key_dest, upload_id):
+ parts = []
+ for part_num, start, end in self._iter_part_num_ranges():
+ copy_range = 'bytes=%d-%d' % (start, end - 1)
+ copy_resp = self.client.\
+ upload_part_copy(Bucket=self.bucket_name,
+ Key=key_dest, PartNumber=part_num,
+ CopySource={
+ 'Bucket': self.bucket_name,
+ 'Key': key_src,
+ }, CopySourceRange=copy_range,
+ UploadId=upload_id)
+ self.assertEqual(200, copy_resp[
+ 'ResponseMetadata']['HTTPStatusCode'])
+ self.assertTrue(copy_resp['CopyPartResult']['ETag'])
+ self.assertTrue(copy_resp['CopyPartResult']['LastModified'])
+ parts.append({
+ 'ETag': copy_resp['CopyPartResult']['ETag'],
+ 'PartNumber': part_num,
+ })
+
+ complete_mpu_resp = self.client.complete_multipart_upload(
+ Bucket=self.bucket_name, Key=key_dest,
+ MultipartUpload={
+ 'Parts': parts,
+ },
+ UploadId=upload_id,
+ )
+ self.assertEqual(200, complete_mpu_resp[
+ 'ResponseMetadata']['HTTPStatusCode'])
+
+ return complete_mpu_resp['ETag']
+
+
+class TestMultiPartUpload(BaseMultiPartUploadTestCase):
+
+ def setUp(self):
+ super(TestMultiPartUpload, self).setUp()
+
+ def _discover_max_part_num(self):
+ key_name = self.create_name('discover-max-part-num')
+ self.upload_mpu(key_name)
+ with self.assertRaises(ClientError) as cm:
+ self.client.get_object(Bucket=self.bucket_name,
+ Key=key_name, PartNumber=0)
+ err_resp = cm.exception.response
+ self.assertEqual(400, err_resp['ResponseMetadata']['HTTPStatusCode'])
+ self.assertEqual('InvalidArgument', err_resp['Error']['Code'])
+ err_msg = err_resp['Error']['Message']
+ preamble = 'Part number must be an integer between 1 and '
+ self.assertIn(preamble, err_msg)
+ return int(err_msg[len(preamble):].split(',')[0])
def test_basic_upload(self):
key_name = self.create_name('key')
@@ -68,6 +215,303 @@ class TestMultiPartUploads(BaseS3TestCase):
self.assertEqual(200, complete_mpu_resp[
'ResponseMetadata']['HTTPStatusCode'])
+ def _check_part_num_invalid_exc(self, exc, val, max_part_num,
+ is_head=False):
+ err_resp = exc.response
+ self.assertEqual(400, err_resp['ResponseMetadata']['HTTPStatusCode'])
+ if is_head:
+ err_code = '400'
+ err_msg = 'Bad Request'
+ else:
+ err_code = 'InvalidArgument'
+ err_msg = 'Part number must be an integer between ' \
+ '1 and %d, inclusive' % max_part_num
+ self.assertEqual(err_code, err_resp['Error']['Code'], err_resp)
+ self.assertEqual(err_msg, err_resp['Error']['Message'])
+ if is_head:
+ self.assertNotIn('ArgumentName', err_resp['Error'])
+ self.assertNotIn('ArgumentValue', err_resp['Error'])
+ else:
+ self.assertEqual('partNumber', err_resp['Error']['ArgumentName'])
+ self.assertEqual(str(val), err_resp['Error']['ArgumentValue'])
+
+ def _check_part_num_out_of_range_exc(self, exc, is_head=False):
+ err_resp = exc.response
+ self.assertEqual(416, err_resp['ResponseMetadata']['HTTPStatusCode'])
+ if is_head:
+ err_code = '416'
+ err_msg = 'Requested Range Not Satisfiable'
+ else:
+ err_code = 'InvalidPartNumber'
+ err_msg = 'The requested partnumber is not satisfiable'
+ self.assertEqual(err_code, err_resp['Error']['Code'], err_resp)
+ self.assertEqual(err_msg, err_resp['Error']['Message'], err_resp)
+
+ def test_get_object_partNumber_errors(self):
+ max_part_num = self._discover_max_part_num()
+ key_name = self.create_name('invalid-part-num-test')
+ mpu_etag = self.upload_mpu(key_name)
+
+ # partNumber argument is 1 indexed
+ with self.assertRaises(ClientError) as caught:
+ self.client.get_object(Bucket=self.bucket_name,
+ Key=key_name, PartNumber=0)
+ self._check_part_num_invalid_exc(caught.exception, 0, max_part_num)
+
+ # all other partNumber args are valid
+ self._verify_part_num_response(
+ self.client.get_object, key_name, mpu_etag)
+
+ with self.assertRaises(ClientError) as caught:
+ self.client.get_object(Bucket=self.bucket_name,
+ Key=key_name, PartNumber=self.num_parts + 1)
+ self._check_part_num_out_of_range_exc(caught.exception)
+
+ with self.assertRaises(ClientError) as caught:
+ self.client.get_object(Bucket=self.bucket_name,
+ Key=key_name, PartNumber=max_part_num)
+ self._check_part_num_out_of_range_exc(caught.exception)
+
+ # because of ParamValidationError we can't test 'foo'
+ val = -1
+ with self.assertRaises(ClientError) as caught:
+ self.client.get_object(Bucket=self.bucket_name,
+ Key=key_name, PartNumber=val)
+ self._check_part_num_invalid_exc(caught.exception, val, max_part_num)
+ val = max_part_num + 1
+ with self.assertRaises(ClientError) as caught:
+ self.client.get_object(Bucket=self.bucket_name,
+ Key=key_name, PartNumber=val)
+ self._check_part_num_invalid_exc(caught.exception, val, max_part_num)
+
+ def test_head_object_partNumber_errors(self):
+ max_part_num = self._discover_max_part_num()
+ key_name = self.create_name('invalid-part-num-head')
+ mpu_etag = self.upload_mpu(key_name)
+
+ # partNumber argument is 1 indexed
+ with self.assertRaises(ClientError) as caught:
+ self.client.head_object(Bucket=self.bucket_name,
+ Key=key_name, PartNumber=0)
+ self._check_part_num_invalid_exc(caught.exception, 0, max_part_num,
+ is_head=True)
+
+ # all other partNumber args are valid
+ self._verify_part_num_response(
+ self.client.head_object, key_name, mpu_etag)
+
+ with self.assertRaises(ClientError) as caught:
+ self.client.head_object(Bucket=self.bucket_name, Key=key_name,
+ PartNumber=self.num_parts + 1)
+ self._check_part_num_out_of_range_exc(caught.exception, is_head=True)
+
+ with self.assertRaises(ClientError) as caught:
+ self.client.head_object(Bucket=self.bucket_name, Key=key_name,
+ PartNumber=max_part_num)
+ self._check_part_num_out_of_range_exc(caught.exception, is_head=True)
+
+ # because of ParamValidationError we can't test 'foo'
+ val = -1
+ with self.assertRaises(ClientError) as caught:
+ self.client.head_object(Bucket=self.bucket_name, Key=key_name,
+ PartNumber=val)
+ self._check_part_num_invalid_exc(caught.exception, val, max_part_num,
+ is_head=True)
+ val = max_part_num + 1
+ with self.assertRaises(ClientError) as caught:
+ self.client.head_object(Bucket=self.bucket_name, Key=key_name,
+ PartNumber=val)
+ self._check_part_num_invalid_exc(caught.exception, val, max_part_num,
+ is_head=True)
+
+ def test_part_number_non_mpu(self):
+ max_part_num = self._discover_max_part_num()
+ key_name = self.create_name('part-num-non-mpu')
+ self.client.put_object(Bucket=self.bucket_name,
+ Key=key_name,
+ Body=b'non-mpu-object')
+ head_resp = self.client.head_object(Bucket=self.bucket_name,
+ Key=key_name)
+ # sanity check
+ self.assertEqual(200,
+ head_resp['ResponseMetadata']['HTTPStatusCode'])
+ self.assertEqual(head_resp['AcceptRanges'], 'bytes')
+ self.assertEqual(head_resp['ContentLength'], 14)
+
+ head_resp = self.client.head_object(Bucket=self.bucket_name,
+ Key=key_name,
+ PartNumber=1)
+ self.assertEqual(206,
+ head_resp['ResponseMetadata']['HTTPStatusCode'])
+ self.assertEqual(head_resp['ContentLength'], 14)
+
+ get_resp = self.client.get_object(Bucket=self.bucket_name,
+ Key=key_name,
+ PartNumber=1)
+ self.assertEqual(206,
+ get_resp['ResponseMetadata']['HTTPStatusCode'])
+ self.assertEqual(get_resp['ContentLength'], 14)
+ self.assertEqual(get_resp['ContentRange'], 'bytes 0-13/14')
+ self.assertEqual(b'non-mpu-object', b''.join(get_resp['Body']))
+
+ with self.assertRaises(ClientError) as caught:
+ self.client.get_object(Bucket=self.bucket_name,
+ Key=key_name,
+ PartNumber=4)
+ self._check_part_num_out_of_range_exc(caught.exception)
+
+ with self.assertRaises(ClientError) as caught:
+ self.client.head_object(Bucket=self.bucket_name,
+ Key=key_name,
+ PartNumber=4)
+ self._check_part_num_out_of_range_exc(caught.exception, is_head=True)
+
+ with self.assertRaises(ClientError) as caught:
+ self.client.get_object(Bucket=self.bucket_name,
+ Key=key_name,
+ PartNumber=0)
+ self._check_part_num_invalid_exc(caught.exception, 0, max_part_num)
+
+ with self.assertRaises(ClientError) as caught:
+ self.client.head_object(Bucket=self.bucket_name,
+ Key=key_name,
+ PartNumber=0)
+ self._check_part_num_invalid_exc(caught.exception, 0, max_part_num,
+ is_head=True)
+
+ invalid_part_num = 10001
+ with self.assertRaises(ClientError) as caught:
+ self.client.get_object(Bucket=self.bucket_name,
+ Key=key_name,
+ PartNumber=invalid_part_num)
+ self._check_part_num_invalid_exc(caught.exception, invalid_part_num,
+ max_part_num)
+
+ with self.assertRaises(ClientError) as caught:
+ self.client.head_object(Bucket=self.bucket_name,
+ Key=key_name,
+ PartNumber=invalid_part_num)
+ self._check_part_num_invalid_exc(caught.exception, invalid_part_num,
+ max_part_num, is_head=True)
+
+ def test_get_object_partNumber_and_range(self):
+ # partNumber not allowed with Range even for non-mpu object
+ key_name = self.create_name('part-num-mpu')
+ self._upload_mpu(key_name)
+ with self.assertRaises(ClientError) as caught:
+ self.client.get_object(Bucket=self.bucket_name,
+ Key=key_name,
+ PartNumber=1,
+ Range='bytes=1-2')
+ err_resp = caught.exception.response
+ self.assertEqual(400, err_resp['ResponseMetadata']['HTTPStatusCode'])
+ self.assertEqual('InvalidRequest', err_resp['Error']['Code'], err_resp)
+ self.assertEqual('Cannot specify both Range header and partNumber '
+ 'query parameter', err_resp['Error']['Message'])
+
+ key_name = self.create_name('part-num-non-mpu')
+ self.client.put_object(Bucket=self.bucket_name,
+ Key=key_name,
+ Body=b'non-mpu-object')
+ with self.assertRaises(ClientError) as caught:
+ self.client.get_object(Bucket=self.bucket_name,
+ Key=key_name,
+ PartNumber=1,
+ Range='bytes=1-2')
+ err_resp = caught.exception.response
+ self.assertEqual(400, err_resp['ResponseMetadata']['HTTPStatusCode'])
+ self.assertEqual('InvalidRequest', err_resp['Error']['Code'], err_resp)
+ self.assertEqual('Cannot specify both Range header and partNumber '
+ 'query parameter', err_resp['Error']['Message'])
+
+ # partNumber + Range error trumps bad partNumber
+ with self.assertRaises(ClientError) as caught:
+ self.client.get_object(Bucket=self.bucket_name,
+ Key=key_name,
+ PartNumber=0,
+ Range='bytes=1-2')
+ err_resp = caught.exception.response
+ self.assertEqual(400, err_resp['ResponseMetadata']['HTTPStatusCode'])
+ self.assertEqual('InvalidRequest', err_resp['Error']['Code'], err_resp)
+ self.assertEqual('Cannot specify both Range header and partNumber '
+ 'query parameter', err_resp['Error']['Message'])
+
+ def test_upload_part_copy(self):
+ self.num_parts = 4
+ key_src = self.create_name('part-copy-src')
+ key_dest = self.create_name('part-copy-dest')
+ mpu_etag_src = self.upload_mpu(key_src)
+ self._verify_part_num_response(
+ self.client.get_object, key_src, mpu_etag_src)
+ self._verify_part_num_response(
+ self.client.head_object, key_src, mpu_etag_src)
+
+ create_mpu_dest = self.client.create_multipart_upload(
+ Bucket=self.bucket_name, Key=key_dest)
+ self.assertEqual(200, create_mpu_dest[
+ 'ResponseMetadata']['HTTPStatusCode'])
+
+ upload_id = create_mpu_dest['UploadId']
+ mpu_etag_dst = self._verify_copy_parts(key_src, key_dest, upload_id)
+ self._verify_part_num_response(
+ self.client.get_object, key_dest, mpu_etag_dst)
+ self._verify_part_num_response(
+ self.client.head_object, key_dest, mpu_etag_dst)
+
+ def test_copy_mpu_from_parts(self):
+ key_src = self.create_name('copy-from-from-src')
+ mpu_etag_src = self.upload_mpu(key_src)
+
+ # client wanting to copy object would first HEAD
+ head_object_resp = self.client.head_object(
+ Bucket=self.bucket_name, Key=key_src)
+ # the client will know it's an mpu and how many parts
+ self.assertEqual(mpu_etag_src, head_object_resp['ETag'])
+ self.assertIn('-', mpu_etag_src)
+ num_parts = int(mpu_etag_src.strip('"').rsplit('-')[-1])
+
+ # create new mpu
+ key_dest = self.create_name('copy-from-from-dest')
+ create_mpu_dest = self.client.create_multipart_upload(
+ Bucket=self.bucket_name, Key=key_dest)
+ self.assertEqual(200, create_mpu_dest[
+ 'ResponseMetadata']['HTTPStatusCode'])
+ upload_id = create_mpu_dest['UploadId']
+
+ parts = []
+ start = 0
+ # do HEAD?partNumber to get copy range
+ for part_num in range(1, num_parts + 1):
+ part_head_resp = self.client.head_object(
+ Bucket=self.bucket_name, Key=key_src, PartNumber=part_num)
+ end = start + part_head_resp['ContentLength']
+ copy_range = 'bytes=%s-%s' % (start, end - 1)
+ copy_resp = self.client.upload_part_copy(
+ Bucket=self.bucket_name, Key=key_dest, PartNumber=part_num,
+ CopySource={
+ 'Bucket': self.bucket_name,
+ 'Key': key_src,
+ },
+ CopySourceRange=copy_range, UploadId=upload_id)
+ self.assertEqual(200, copy_resp[
+ 'ResponseMetadata']['HTTPStatusCode'])
+ parts.append({
+ 'ETag': copy_resp['CopyPartResult']['ETag'],
+ 'PartNumber': part_num,
+ })
+ start = end
+
+ complete_mpu_resp = self.client.complete_multipart_upload(
+ Bucket=self.bucket_name, Key=key_dest,
+ MultipartUpload={
+ 'Parts': parts,
+ },
+ UploadId=upload_id,
+ )
+ self.assertEqual(200, complete_mpu_resp[
+ 'ResponseMetadata']['HTTPStatusCode'])
+ self.assertEqual(complete_mpu_resp['ETag'], mpu_etag_src)
+
def test_create_list_abort_multipart_uploads(self):
key_name = self.create_name('key')
create_mpu_resp = self.client.create_multipart_upload(
@@ -138,3 +582,36 @@ class TestMultiPartUploads(BaseS3TestCase):
self.assertEqual(complete_mpu_resp['Error']['UploadId'], upload_id)
self.assertIn(complete_mpu_resp['Error']['PartNumber'], ('1', '2'))
self.assertEqual(complete_mpu_resp['Error']['ETag'], None)
+
+
+class TestVersionedMultiPartUpload(BaseMultiPartUploadTestCase):
+
+ def setUp(self):
+ super(TestVersionedMultiPartUpload, self).setUp()
+ resp = self.client.put_bucket_versioning(
+ Bucket=self.bucket_name,
+ VersioningConfiguration={'Status': 'Enabled'})
+ self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
+
+ def tearDown(self):
+ resp = self.client.put_bucket_versioning(
+ Bucket=self.bucket_name,
+ VersioningConfiguration={'Status': 'Suspended'})
+ self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
+ super(TestVersionedMultiPartUpload, self).tearDown()
+
+ def test_get_by_part_number_with_versioning(self):
+ # create 3 version with progressively larger sizes
+ parts_counts = [2, 3, 4]
+ key_name = self.create_name('part-num-versions')
+ version_vars = []
+ for num_parts in parts_counts:
+ self.num_parts = num_parts
+ etag, version_id = self.upload_mpu_version(key_name)
+ version_vars.append((num_parts, etag, version_id))
+ for num_parts, mpu_etag, version in version_vars:
+ self.num_parts = num_parts
+ self._verify_part_num_response(
+ self.client.get_object, key_name, mpu_etag, version)
+ self._verify_part_num_response(
+ self.client.head_object, key_name, mpu_etag, version)
diff --git a/test/unit/common/middleware/s3api/__init__.py b/test/unit/common/middleware/s3api/__init__.py
index 4ec20255c2..83304c3987 100644
--- a/test/unit/common/middleware/s3api/__init__.py
+++ b/test/unit/common/middleware/s3api/__init__.py
@@ -28,19 +28,16 @@ from swift.common.middleware.s3api.etree import fromstring
from swift.common.middleware.s3api.subresource import Owner, encode_acl, \
Grant, User, ACL, PERMISSIONS, AllUsers, AuthenticatedUsers
-from test.debug_logger import debug_logger
from test.unit.common.middleware.helpers import FakeSwift
-class FakeApp(object):
+class FakeAuthApp(object):
container_existence_skip_cache = 0.0
account_existence_skip_cache = 0.0
- def __init__(self):
+ def __init__(self, app):
self.remote_user = 'authorized'
- self._pipeline_final_app = self
- self.swift = FakeSwift()
- self.logger = debug_logger()
+ self.app = app
def _update_s3_path_info(self, env):
"""
@@ -82,7 +79,7 @@ class FakeApp(object):
def __call__(self, env, start_response):
self.handle(env)
- return self.swift(env, start_response)
+ return self.app(env, start_response)
class S3ApiTestCase(unittest.TestCase):
@@ -90,6 +87,9 @@ class S3ApiTestCase(unittest.TestCase):
def __init__(self, name):
unittest.TestCase.__init__(self, name)
+ def _wrap_app(self, app):
+ return FakeAuthApp(app)
+
def setUp(self):
# setup default config dict
self.conf = {
@@ -110,12 +110,13 @@ class S3ApiTestCase(unittest.TestCase):
'log_level': 'debug'
}
- self.app = FakeApp()
- self.swift = self.app.swift
# note: self.conf has no __file__ key so check_pipeline will be skipped
# when constructing self.s3api
+ self.swift = FakeSwift()
+ self.app = self._wrap_app(self.swift)
+ self.app._pipeline_final_app = self.swift
self.s3api = filter_factory({}, **self.conf)(self.app)
- self.logger = self.s3api.logger = self.swift.logger = debug_logger()
+ self.logger = self.s3api.logger = self.swift.logger
# if you change the registered acl response for /bucket or
# /bucket/object tearDown will complain at you; you can set this to
@@ -282,11 +283,11 @@ class S3ApiTestCaseAcl(S3ApiTestCase):
self.swift.register('GET', path, swob.HTTPOk, {}, json.dumps([])),
# setup sticky ACL headers...
- grants = [_gen_grant(perm) for perm in PERMISSIONS]
+ self.grants = [_gen_grant(perm) for perm in PERMISSIONS]
self.default_owner = Owner('test:tester', 'test:tester')
- container_headers = _gen_test_headers(self.default_owner, grants)
+ container_headers = _gen_test_headers(self.default_owner, self.grants)
object_headers = _gen_test_headers(
- self.default_owner, grants, 'object')
+ self.default_owner, self.grants, 'object')
public_headers = _gen_test_headers(
self.default_owner, [Grant(AllUsers(), 'READ')])
authenticated_headers = _gen_test_headers(
diff --git a/test/unit/common/middleware/s3api/test_multi_get.py b/test/unit/common/middleware/s3api/test_multi_get.py
new file mode 100644
index 0000000000..93a8590194
--- /dev/null
+++ b/test/unit/common/middleware/s3api/test_multi_get.py
@@ -0,0 +1,661 @@
+# Copyright (c) 2023 NVIDIA
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import binascii
+import string
+import json
+import mock
+
+from swift.common import swob, utils
+from swift.common.request_helpers import get_reserved_name
+from swift.common.middleware import symlink
+from swift.common.middleware.versioned_writes import object_versioning as ov
+
+from test.unit import make_timestamp_iter
+from test.unit.common.middleware.test_slo import slo, md5hex
+from test.unit.common.middleware.s3api import (
+ S3ApiTestCase, S3ApiTestCaseAcl, _gen_test_headers)
+
+
+def _prepare_mpu(swift, ts_iter, upload_id, num_segments,
+ segment_bucket='bucket+segments', segment_key='mpu'):
+ manifest = []
+ for i, letter in enumerate(string.ascii_lowercase):
+ if len(manifest) >= num_segments:
+ break
+ size = (i + 1) * 5
+ body = letter * size
+ etag = md5hex(body)
+ path = '/%s/%s/%s/%s' % (segment_bucket, segment_key, upload_id, i + 1)
+ swift.register('GET', '/v1/AUTH_test' + path, swob.HTTPOk, {
+ 'Content-Length': len(body),
+ 'Etag': etag,
+ }, body)
+ manifest.append({
+ "name": path,
+ "bytes": size,
+ "hash": etag,
+ "content_type": "application/octet-stream",
+ "last_modified": next(ts_iter).isoformat,
+ })
+ slo_etag = md5hex(''.join(s['hash'] for s in manifest))
+ s3_hash = md5hex(binascii.a2b_hex(''.join(
+ s['hash'] for s in manifest)))
+ s3_etag = "%s-%s" % (s3_hash, len(manifest))
+ manifest_json = json.dumps(manifest)
+ json_md5 = md5hex(manifest_json)
+ manifest_headers = {
+ 'Content-Length': str(len(manifest_json)),
+ 'X-Static-Large-Object': 'true',
+ 'Etag': json_md5,
+ 'Content-Type': 'application/octet-stream',
+ 'X-Object-Sysmeta-Slo-Etag': slo_etag,
+ 'X-Object-Sysmeta-Slo-Size': str(sum(
+ s['bytes'] for s in manifest)),
+ 'X-Object-Sysmeta-S3Api-Etag': s3_etag,
+ 'X-Object-Sysmeta-S3Api-Upload-Id': upload_id,
+ 'X-Object-Sysmeta-Container-Update-Override-Etag':
+ '%s; s3_etag=%s; slo_etag=%s' % (json_md5, s3_etag, slo_etag),
+ }
+ return manifest_headers, manifest_json
+
+
+class TestMpuGETorHEAD(S3ApiTestCase):
+
+ def _wrap_app(self, app):
+ self.slo = slo.filter_factory({'rate_limit_under_size': '0'})(app)
+ return super(TestMpuGETorHEAD, self)._wrap_app(self.slo)
+
+ def setUp(self):
+ # this will call our _wrap_app
+ super(TestMpuGETorHEAD, self).setUp()
+ self.ts = make_timestamp_iter()
+ manifest_headers, manifest_json = _prepare_mpu(
+ self.swift, self.ts, 'X', 3)
+ self.s3_etag = manifest_headers['X-Object-Sysmeta-S3Api-Etag']
+ self.swift.register(
+ 'GET', '/v1/AUTH_test/bucket/mpu',
+ swob.HTTPOk, manifest_headers, manifest_json.encode('ascii'))
+ self.s3_acl = False
+
+ def test_mpu_GET(self):
+ req = swob.Request.blank('/bucket/mpu', headers={
+ 'Authorization': 'AWS test:tester:hmac',
+ 'Date': self.get_date_header()
+ })
+ status, headers, body = self.call_s3api(req)
+ self.assertEqual(status.split()[0], '200')
+ self.assertEqual(body, b'aaaaabbbbbbbbbbccccccccccccccc')
+ expected_calls = [
+ ('GET', '/v1/AUTH_test/bucket/mpu'),
+ ('GET', '/v1/AUTH_test/bucket+segments/mpu/X/1'
+ '?multipart-manifest=get'),
+ ('GET', '/v1/AUTH_test/bucket+segments/mpu/X/2'
+ '?multipart-manifest=get'),
+ ('GET', '/v1/AUTH_test/bucket+segments/mpu/X/3'
+ '?multipart-manifest=get'),
+ ]
+ if self.s3_acl:
+ # pre-flight object ACL check
+ expected_calls.insert(0, ('HEAD', '/v1/AUTH_test/bucket/mpu'))
+ self.assertEqual(self.swift.calls, expected_calls)
+ self.assertEqual(headers['Content-Length'], '30')
+ self.assertEqual(headers['Etag'], '"%s"' % self.s3_etag)
+ self.assertNotIn('X-Amz-Mp-Parts-Count', headers)
+
+ def test_mpu_GET_part_num(self):
+ req = swob.Request.blank('/bucket/mpu', params={
+ 'partNumber': '2',
+ }, headers={
+ 'Authorization': 'AWS test:tester:hmac',
+ 'Date': self.get_date_header()
+ })
+ status, headers, body = self.call_s3api(req)
+ self.assertEqual(status.split()[0], '206')
+ self.assertEqual(body, b'bbbbbbbbbb')
+ expected_calls = [
+ ('GET', '/v1/AUTH_test/bucket/mpu?part-number=2'),
+ ('GET', '/v1/AUTH_test/bucket+segments/mpu/X/2'
+ '?multipart-manifest=get'),
+ ]
+ if self.s3_acl:
+ expected_calls.insert(0, ('HEAD', '/v1/AUTH_test/bucket/mpu'))
+ self.assertEqual(self.swift.calls, expected_calls)
+ self.assertEqual(headers['Content-Length'], '10')
+ self.assertEqual(headers['Content-Range'], 'bytes 5-14/30')
+ self.assertEqual(headers['Etag'], '"%s"' % self.s3_etag)
+ self.assertEqual(headers['X-Amz-Mp-Parts-Count'], '3')
+
+ def test_mpu_GET_invalid_part_num(self):
+ req = swob.Request.blank('/bucket/mpu', params={
+ 'partNumber': 'foo',
+ }, headers={
+ 'Authorization': 'AWS test:tester:hmac',
+ 'Date': self.get_date_header()
+ })
+ status, headers, body = self.call_s3api(req)
+ self.assertEqual(status.split()[0], '400')
+ self.assertEqual(self._get_error_code(body), 'InvalidArgument')
+ self.assertEqual(self.swift.calls, [])
+
+ def test_mpu_GET_zero_part_num(self):
+ req = swob.Request.blank('/bucket/mpu', params={
+ 'partNumber': '0',
+ }, headers={
+ 'Authorization': 'AWS test:tester:hmac',
+ 'Date': self.get_date_header()
+ })
+ status, headers, body = self.call_s3api(req)
+ self.assertEqual(status.split()[0], '400')
+ self.assertEqual(self._get_error_code(body), 'InvalidArgument')
+ self.assertEqual(self.swift.calls, [])
+
+ def _do_test_mpu_GET_out_of_range_part_num(self, part_number):
+ self.swift.clear_calls()
+ req = swob.Request.blank('/bucket/mpu', params={
+ 'partNumber': str(part_number),
+ }, headers={
+ 'Authorization': 'AWS test:tester:hmac',
+ 'Date': self.get_date_header()
+ })
+ status, headers, body = self.call_s3api(req)
+ self.assertEqual(status.split()[0], '416')
+ self.assertEqual(self._get_error_code(body), 'InvalidPartNumber')
+ expected_calls = [
+ # s3api.controller.obj doesn't know yet if it's SLO, we delegate
+ # param validation
+ ('GET', '/v1/AUTH_test/bucket/mpu?part-number=%s' % part_number),
+ ]
+ if self.s3_acl:
+ expected_calls.insert(0, ('HEAD', '/v1/AUTH_test/bucket/mpu'))
+ self.assertEqual(self.swift.calls, expected_calls)
+
+ def test_mpu_GET_out_of_range_part_num(self):
+ self._do_test_mpu_GET_out_of_range_part_num(4)
+ self._do_test_mpu_GET_out_of_range_part_num(10000)
+
+ def test_existing_part_number_greater_than_max_parts_allowed(self):
+ part_number = 3
+ max_parts = 2
+ req = swob.Request.blank('/bucket/mpu', params={
+ 'partNumber': str(part_number),
+ }, headers={
+ 'Authorization': 'AWS test:tester:hmac',
+ 'Date': self.get_date_header()
+ })
+ bad_req = swob.Request.blank('/bucket/mpu', params={
+ 'partNumber': str(part_number + 1),
+ }, headers={
+ 'Authorization': 'AWS test:tester:hmac',
+ 'Date': self.get_date_header()
+ })
+ with mock.patch.object(self.s3api.conf,
+ 'max_upload_part_num', max_parts):
+ # num_parts >= part number > max parts
+ status, headers, body = self.call_s3api(req)
+ self.assertEqual(status.split()[0], '206')
+ # part number > num parts > max parts
+ status, headers, body = self.call_s3api(bad_req)
+ self.assertEqual(status.split()[0], '400')
+ self.assertIn('must be an integer between 1 and 3, inclusive',
+ self._get_error_message(body))
+
+ max_parts = part_number + 1
+ with mock.patch.object(self.s3api.conf,
+ 'max_upload_part_num', max_parts):
+ # max_parts > num_parts >= part number
+ status, headers, body = self.call_s3api(req)
+ self.assertEqual(status.split()[0], '206')
+ # max_parts >= part number > num parts
+ status, headers, body = self.call_s3api(bad_req)
+ self.assertEqual(status.split()[0], '416')
+ self.assertIn('The requested partnumber is not satisfiable',
+ self._get_error_message(body))
+ # part number > max_parts > num parts
+ bad_req.params = {'partNumber': str(max_parts + 1)}
+ status, headers, body = self.call_s3api(bad_req)
+ self.assertEqual(status.split()[0], '400')
+ self.assertIn('must be an integer between 1 and 4, inclusive',
+ self._get_error_message(body))
+
+ def test_mpu_GET_huge_part_num(self):
+ req = swob.Request.blank('/bucket/mpu', params={
+ 'partNumber': '10001',
+ }, headers={
+ 'Authorization': 'AWS test:tester:hmac',
+ 'Date': self.get_date_header()
+ })
+ status, headers, body = self.call_s3api(req)
+ self.assertEqual(status.split()[0], '400')
+ self.assertEqual(self._get_error_code(body), 'InvalidArgument')
+ expected_calls = [
+ # XXX is this value configurable? do we need the SLO request?
+ ('GET', '/v1/AUTH_test/bucket/mpu?part-number=10001'),
+ ]
+ if self.s3_acl:
+ expected_calls.insert(0, ('HEAD', '/v1/AUTH_test/bucket/mpu'))
+ self.assertEqual(self.swift.calls, expected_calls)
+
+ def test_mpu_HEAD_part_num(self):
+ req = swob.Request.blank('/bucket/mpu', params={
+ 'partNumber': '1',
+ }, headers={
+ 'Authorization': 'AWS test:tester:hmac',
+ 'Date': self.get_date_header()
+ }, method='HEAD')
+ status, headers, body = self.call_s3api(req)
+ self.assertEqual(status.split()[0], '206')
+ self.assertEqual(body, b'')
+ self.assertEqual(self.swift.calls, [
+ ('HEAD', '/v1/AUTH_test/bucket/mpu?part-number=1'),
+ ('GET', '/v1/AUTH_test/bucket/mpu?part-number=1'),
+ ])
+ self.assertEqual(headers['Content-Length'], '5')
+ self.assertEqual(headers['Content-Range'], 'bytes 0-4/30')
+ self.assertEqual(headers['Etag'], '"%s"' % self.s3_etag)
+ self.assertEqual(headers['X-Amz-Mp-Parts-Count'], '3')
+
+ def test_mpu_HEAD_invalid_part_num(self):
+ req = swob.Request.blank('/bucket/mpu', method='HEAD', params={
+ 'partNumber': 'foo',
+ }, headers={
+ 'Authorization': 'AWS test:tester:hmac',
+ 'Date': self.get_date_header()
+ })
+ status, headers, _ = self.call_s3api(req)
+ self.assertEqual(status.split()[0], '400')
+ self.assertEqual(self.swift.calls, [])
+
+ def test_mpu_HEAD_zero_part_num(self):
+ req = swob.Request.blank('/bucket/mpu', method='HEAD', params={
+ 'partNumber': '0',
+ }, headers={
+ 'Authorization': 'AWS test:tester:hmac',
+ 'Date': self.get_date_header()
+ })
+ status, headers, _ = self.call_s3api(req)
+ self.assertEqual(status.split()[0], '400')
+ self.assertEqual(self.swift.calls, [])
+
+ def _do_test_mpu_HEAD_out_of_range_part_num(self, part_number):
+ self.swift.clear_calls()
+ req = swob.Request.blank('/bucket/mpu', method='HEAD', params={
+ 'partNumber': str(part_number),
+ }, headers={
+ 'Authorization': 'AWS test:tester:hmac',
+ 'Date': self.get_date_header()
+ })
+ status, headers, _ = self.call_s3api(req)
+ self.assertEqual(status.split()[0], '416')
+ self.assertEqual(self.swift.calls, [
+ ('HEAD', '/v1/AUTH_test/bucket/mpu?part-number=%s' % part_number),
+ # SLO has to refetch to *see* if it's out-of-bounds
+ ('GET', '/v1/AUTH_test/bucket/mpu?part-number=%s' % part_number),
+ ])
+
+ def test_mpu_HEAD_out_of_range_part_num(self):
+ self._do_test_mpu_HEAD_out_of_range_part_num(4)
+ self._do_test_mpu_HEAD_out_of_range_part_num(10000)
+
+ def test_mpu_HEAD_huge_part_num(self):
+ req = swob.Request.blank('/bucket/mpu', method='HEAD', params={
+ 'partNumber': '10001',
+ }, headers={
+ 'Authorization': 'AWS test:tester:hmac',
+ 'Date': self.get_date_header()
+ })
+ status, headers, body = self.call_s3api(req)
+ self.assertEqual(status.split()[0], '400')
+ self.assertEqual(self.swift.calls, [
+ ('HEAD', '/v1/AUTH_test/bucket/mpu?part-number=10001'),
+ # XXX were two requests worth it to 400?
+ # how big can you configure SLO?
+ # do such manifests *exist*?
+ ('GET', '/v1/AUTH_test/bucket/mpu?part-number=10001'),
+ ])
+
+
+class TestMpuGETorHEADAcl(TestMpuGETorHEAD, S3ApiTestCaseAcl):
+
+ def setUp(self):
+ super(TestMpuGETorHEADAcl, self).setUp()
+ object_headers = _gen_test_headers(
+ self.default_owner, self.grants, 'object')
+ self.swift.update_sticky_response_headers(
+ '/v1/AUTH_test/bucket/mpu', object_headers)
+ # this is used to flag insertion of expected HEAD pre-flight request of
+ # object ACLs
+ self.s3_acl = True
+
+
+class TestVersionedMpuGETorHEAD(S3ApiTestCase):
+
+ def _wrap_app(self, app):
+ self.sym = symlink.filter_factory({})(app)
+ self.sym.logger = self.swift.logger
+ self.ov = ov.ObjectVersioningMiddleware(self.sym, {})
+ self.ov.logger = self.swift.logger
+ self.slo = slo.filter_factory({'rate_limit_under_size': '0'})(self.ov)
+ self.slo.logger = self.swift.logger
+ return super(TestVersionedMpuGETorHEAD, self)._wrap_app(self.slo)
+
+ def setUp(self):
+ # this will call our _wrap_app
+ super(TestVersionedMpuGETorHEAD, self).setUp()
+ self.ts = make_timestamp_iter()
+ self.swift.register('HEAD', '/v1/AUTH_test/bucket+segments',
+ swob.HTTPNoContent, {}, None)
+ versions_container = get_reserved_name('versions', 'bucket')
+ self.swift.register(
+ 'HEAD', '/v1/AUTH_test/bucket', swob.HTTPNoContent, {
+ ov.SYSMETA_VERSIONS_CONT: versions_container,
+ ov.SYSMETA_VERSIONS_ENABLED: True,
+ }, None)
+ self.swift.register('HEAD', '/v1/AUTH_test/%s' % versions_container,
+ swob.HTTPNoContent, {}, None)
+ num_versions = 3
+ self.version_ids = []
+ for v in range(num_versions):
+ upload_id = 'X%s' % v
+ num_segments = 3 + v
+ manifest_headers, manifest_json = _prepare_mpu(
+ self.swift, self.ts, upload_id, num_segments)
+ version_ts = next(self.ts)
+ # add in a little user-meta to keep versions stright
+ manifest_version_headers = dict(manifest_headers, **{
+ 'x-object-meta-user-notes': 'version%s' % v,
+ 'x-backend-timestamp': version_ts.internal,
+ })
+ self.version_ids.append(version_ts.normal)
+ obj_version_path = get_reserved_name('mpu', (~version_ts).normal)
+ self.swift.register(
+ 'GET', '/v1/AUTH_test/%s/%s' % (
+ versions_container, obj_version_path),
+ swob.HTTPOk, manifest_version_headers,
+ manifest_json.encode('ascii'))
+ # TODO: make a current version symlink
+ symlink_target = '%s/%s' % (versions_container, obj_version_path)
+ slo_etag = manifest_headers['X-Object-Sysmeta-Slo-Etag']
+ s3_etag = manifest_headers['X-Object-Sysmeta-S3Api-Etag']
+ symlink_target_etag = json_md5 = manifest_headers['Etag']
+ symlink_target_bytes = manifest_headers['X-Object-Sysmeta-Slo-Size']
+ manifest_symlink_headers = dict(manifest_headers, **{
+ 'X-Object-Sysmeta-Container-Update-Override-Etag':
+ '%s; s3_etag=%s; slo_etag=%s; symlink_target=%s; '
+ 'symlink_target_etag=%s; symlink_target_bytes=%s' % (
+ json_md5, s3_etag, slo_etag, symlink_target,
+ symlink_target_etag, symlink_target_bytes),
+ 'X-Object-Sysmeta-Allow-Reserved-Names': 'true',
+ 'X-Object-Sysmeta-Symlink-Target': symlink_target,
+ 'X-Object-Sysmeta-Symlink-Target-Bytes': str(symlink_target_bytes),
+ 'X-Object-Sysmeta-Symlink-Target-Etag': symlink_target_etag,
+ 'X-Object-Sysmeta-Symloop-Extend': 'true',
+ 'X-Object-Sysmeta-Versions-Symlink': 'true',
+ })
+ self.swift.register(
+ 'GET', '/v1/AUTH_test/bucket/mpu', swob.HTTPOk,
+ manifest_symlink_headers, '')
+ self.s3_acl = False
+
+ def test_mpu_GET_version(self):
+ req = swob.Request.blank('/bucket/mpu', params={
+ 'versionId': self.version_ids[0],
+ }, headers={
+ 'Authorization': 'AWS test:tester:hmac',
+ 'Date': self.get_date_header()
+ })
+ status, headers, body = self.call_s3api(req)
+ self.assertEqual(status, '200 OK')
+ self.assertEqual(headers['x-amz-meta-user-notes'], 'version0')
+ self.assertEqual(headers['Content-Length'], '30')
+ self.assertEqual(body, b'aaaaabbbbbbbbbbccccccccccccccc')
+ expected_calls = [
+ ('HEAD', '/v1/AUTH_test'),
+ ('HEAD', '/v1/AUTH_test/bucket'),
+ ('HEAD', '/v1/AUTH_test/\x00versions\x00bucket'),
+ ('GET', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s'
+ '?version-id=%s' % (
+ (~utils.Timestamp(self.version_ids[0])).normal,
+ self.version_ids[0])),
+ ('HEAD', '/v1/AUTH_test/bucket+segments'),
+ ('GET', '/v1/AUTH_test/bucket+segments/mpu/X0/1'
+ '?multipart-manifest=get'),
+ ('GET', '/v1/AUTH_test/bucket+segments/mpu/X0/2'
+ '?multipart-manifest=get'),
+ ('GET', '/v1/AUTH_test/bucket+segments/mpu/X0/3'
+ '?multipart-manifest=get')
+ ]
+ if self.s3_acl:
+ expected_calls.insert(3, (
+ 'HEAD', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s'
+ '?version-id=%s' % (
+ (~utils.Timestamp(self.version_ids[0])).normal,
+ self.version_ids[0])
+ ))
+ self.assertEqual(self.swift.calls, expected_calls)
+
+ def test_mpu_GET_last_version(self):
+ req = swob.Request.blank('/bucket/mpu', headers={
+ 'Authorization': 'AWS test:tester:hmac',
+ 'Date': self.get_date_header()
+ })
+ status, headers, body = self.call_s3api(req)
+ self.assertEqual(status, '200 OK')
+ self.assertEqual(headers['x-amz-meta-user-notes'], 'version2')
+ self.assertEqual(headers['Content-Length'], '75')
+ expected_calls = [
+ ('HEAD', '/v1/AUTH_test'),
+ ('HEAD', '/v1/AUTH_test/bucket'),
+ ('HEAD', '/v1/AUTH_test/\x00versions\x00bucket'),
+ ('GET', '/v1/AUTH_test/bucket/mpu'),
+ ('GET', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s' % (
+ ~utils.Timestamp(self.version_ids[2])).normal),
+ ('HEAD', '/v1/AUTH_test/bucket+segments'),
+ ('GET', '/v1/AUTH_test/bucket+segments/mpu/X2/1'
+ '?multipart-manifest=get'),
+ ('GET', '/v1/AUTH_test/bucket+segments/mpu/X2/2'
+ '?multipart-manifest=get'),
+ ('GET', '/v1/AUTH_test/bucket+segments/mpu/X2/3'
+ '?multipart-manifest=get'),
+ ('GET', '/v1/AUTH_test/bucket+segments/mpu/X2/4'
+ '?multipart-manifest=get'),
+ ('GET', '/v1/AUTH_test/bucket+segments/mpu/X2/5'
+ '?multipart-manifest=get'),
+ ]
+ if self.s3_acl:
+ # the pre-flight head on version marker get's symlinked; but I
+ # think maybe symlink makes metadata addative?
+ expected_calls = expected_calls[:3] + [
+ ('HEAD', '/v1/AUTH_test/bucket/mpu'),
+ ('HEAD', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00'
+ '%s' % (~utils.Timestamp(self.version_ids[2])).normal),
+ ] + expected_calls[3:]
+ self.assertEqual(expected_calls, self.swift.calls)
+
+ def test_mpu_HEAD_last_version(self):
+ req = swob.Request.blank('/bucket/mpu', method='HEAD', headers={
+ 'Authorization': 'AWS test:tester:hmac',
+ 'Date': self.get_date_header()
+ })
+ status, headers, body = self.call_s3api(req)
+ self.assertEqual(status, '200 OK')
+ self.assertEqual(headers['x-amz-meta-user-notes'], 'version2')
+ self.assertEqual(headers['Content-Length'], '75')
+ self.assertEqual([
+ ('HEAD', '/v1/AUTH_test'),
+ ('HEAD', '/v1/AUTH_test/bucket'),
+ ('HEAD', '/v1/AUTH_test/\x00versions\x00bucket'),
+ ('HEAD', '/v1/AUTH_test/bucket/mpu'),
+ ('HEAD', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s' % (
+ ~utils.Timestamp(self.version_ids[2])).normal),
+ ], self.swift.calls)
+
+ def test_mpu_HEAD_version(self):
+ req = swob.Request.blank('/bucket/mpu', method='HEAD', params={
+ 'versionId': self.version_ids[1],
+ }, headers={
+ 'Authorization': 'AWS test:tester:hmac',
+ 'Date': self.get_date_header()
+ })
+ status, headers, body = self.call_s3api(req)
+ self.assertEqual(status, '200 OK')
+ self.assertEqual(headers['x-amz-meta-user-notes'], 'version1')
+ self.assertEqual(headers['Content-Length'], '50')
+ self.assertEqual(body, b'')
+ self.assertEqual([
+ ('HEAD', '/v1/AUTH_test'),
+ ('HEAD', '/v1/AUTH_test/bucket'),
+ ('HEAD', '/v1/AUTH_test/\x00versions\x00bucket'),
+ ('HEAD', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s'
+ '?version-id=%s' % (
+ (~utils.Timestamp(self.version_ids[1])).normal,
+ self.version_ids[1])),
+ ], self.swift.calls)
+
+ def test_mpu_GET_version_part_num(self):
+ req = swob.Request.blank('/bucket/mpu', params={
+ 'versionId': self.version_ids[2],
+ 'partNumber': 5,
+ }, headers={
+ 'Authorization': 'AWS test:tester:hmac',
+ 'Date': self.get_date_header()
+ })
+ status, headers, body = self.call_s3api(req)
+ self.assertEqual(status, '206 Partial Content')
+ self.assertEqual(headers['x-amz-meta-user-notes'], 'version2')
+ self.assertEqual(headers['Content-Length'], '25')
+ self.assertEqual(body, b'e' * 25)
+ expected_calls = [
+ ('HEAD', '/v1/AUTH_test'),
+ ('HEAD', '/v1/AUTH_test/bucket'),
+ ('HEAD', '/v1/AUTH_test/\x00versions\x00bucket'),
+ ('GET', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s'
+ '?part-number=5&version-id=%s' % (
+ (~utils.Timestamp(self.version_ids[2])).normal,
+ self.version_ids[2])),
+ ('HEAD', '/v1/AUTH_test/bucket+segments'),
+ ('GET', '/v1/AUTH_test/bucket+segments/mpu/X2/5'
+ '?multipart-manifest=get'),
+ ]
+ if self.s3_acl:
+ expected_calls.insert(3, (
+ 'HEAD', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s'
+ '?version-id=%s' % (
+ (~utils.Timestamp(self.version_ids[2])).normal,
+ self.version_ids[2])
+ ))
+ self.assertEqual(expected_calls, self.swift.calls)
+
+ def test_mpu_HEAD_version_part_num(self):
+ req = swob.Request.blank('/bucket/mpu', method='HEAD', params={
+ 'versionId': self.version_ids[2],
+ 'partNumber': 3,
+ }, headers={
+ 'Authorization': 'AWS test:tester:hmac',
+ 'Date': self.get_date_header()
+ })
+ status, headers, body = self.call_s3api(req)
+ self.assertEqual(status, '206 Partial Content')
+ self.assertEqual(headers['x-amz-meta-user-notes'], 'version2')
+ self.assertEqual(headers['Content-Length'], '15')
+ self.assertEqual(body, b'')
+ self.assertEqual(self.swift.calls, [
+ ('HEAD', '/v1/AUTH_test'),
+ ('HEAD', '/v1/AUTH_test/bucket'),
+ ('HEAD', '/v1/AUTH_test/\x00versions\x00bucket'),
+ ('HEAD', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s'
+ '?part-number=3&version-id=%s' % (
+ (~utils.Timestamp(self.version_ids[2])).normal,
+ self.version_ids[2])),
+ ('GET', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s'
+ '?part-number=3&version-id=%s' % (
+ (~utils.Timestamp(self.version_ids[2])).normal,
+ self.version_ids[2])),
+ ])
+
+ def test_mpu_GET_last_version_part_num(self):
+ req = swob.Request.blank('/bucket/mpu', params={
+ 'partNumber': 4,
+ }, headers={
+ 'Authorization': 'AWS test:tester:hmac',
+ 'Date': self.get_date_header()
+ })
+ status, headers, body = self.call_s3api(req)
+ self.assertEqual(status, '206 Partial Content')
+ self.assertEqual(headers['x-amz-meta-user-notes'], 'version2')
+ self.assertEqual(headers['Content-Length'], '20')
+ self.assertEqual(body, b'd' * 20)
+ expected_calls = [
+ ('HEAD', '/v1/AUTH_test'),
+ ('HEAD', '/v1/AUTH_test/bucket'),
+ ('HEAD', '/v1/AUTH_test/\x00versions\x00bucket'),
+ ('GET', '/v1/AUTH_test/bucket/mpu?part-number=4'),
+ ('GET', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s'
+ '?part-number=4' % (
+ ~utils.Timestamp(self.version_ids[2])).normal),
+ ('HEAD', '/v1/AUTH_test/bucket+segments'),
+ ('GET', '/v1/AUTH_test/bucket+segments/mpu/X2/4'
+ '?multipart-manifest=get'),
+ ]
+ if self.s3_acl:
+ expected_calls = expected_calls[:3] + [
+ ('HEAD', '/v1/AUTH_test/bucket/mpu'),
+ ('HEAD', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00'
+ '%s' % (~utils.Timestamp(self.version_ids[2])).normal),
+ ] + expected_calls[3:]
+ self.assertEqual(expected_calls, self.swift.calls)
+
+ def test_mpu_HEAD_last_version_part_num(self):
+ req = swob.Request.blank('/bucket/mpu', method='HEAD', params={
+ 'partNumber': 5,
+ }, headers={
+ 'Authorization': 'AWS test:tester:hmac',
+ 'Date': self.get_date_header()
+ })
+ status, headers, body = self.call_s3api(req)
+ self.assertEqual(status, '206 Partial Content')
+ self.assertEqual(headers['x-amz-meta-user-notes'], 'version2')
+ self.assertEqual(headers['Content-Length'], '25')
+ self.assertEqual(self.swift.calls, [
+ ('HEAD', '/v1/AUTH_test'),
+ ('HEAD', '/v1/AUTH_test/bucket'),
+ ('HEAD', '/v1/AUTH_test/\x00versions\x00bucket'),
+ ('HEAD', '/v1/AUTH_test/bucket/mpu?part-number=5'),
+ ('HEAD', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s'
+ '?part-number=5' % (
+ ~utils.Timestamp(self.version_ids[2])).normal),
+ ('GET', '/v1/AUTH_test/bucket/mpu?part-number=5'),
+ ('GET', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s'
+ '?part-number=5' % (
+ ~utils.Timestamp(self.version_ids[2])).normal),
+ ])
+
+
+class TestVersionedMpuGETorHEADAcl(TestVersionedMpuGETorHEAD,
+ S3ApiTestCaseAcl):
+
+ def setUp(self):
+ super(TestVersionedMpuGETorHEADAcl, self).setUp()
+ object_headers = _gen_test_headers(
+ self.default_owner, self.grants, 'object')
+ for version_id in self.version_ids:
+ # s3acl would add the default object ACL on PUT to each version
+ version_path = '/v1/AUTH_test/\x00versions\x00bucket/' \
+ '\x00mpu\x00%s' % (~utils.Timestamp(version_id)).normal
+ self.swift.update_sticky_response_headers(
+ version_path, object_headers)
+ # this is used to flag insertion of expected HEAD pre-flight request of
+ # object ACLs
+ self.s3_acl = True
diff --git a/test/unit/common/middleware/s3api/test_multi_upload.py b/test/unit/common/middleware/s3api/test_multi_upload.py
index 7a91011848..e2e121ace7 100644
--- a/test/unit/common/middleware/s3api/test_multi_upload.py
+++ b/test/unit/common/middleware/s3api/test_multi_upload.py
@@ -685,6 +685,9 @@ class BaseS3ApiMultiUpload(object):
body='part object')
status, headers, body = self.call_s3api(req)
self.assertEqual(self._get_error_code(body), 'InvalidArgument')
+ self.assertEqual(self._get_error_message(body),
+ 'Part number must be an integer between 1 and 10000, '
+ 'inclusive')
# part number must be > 0
req = Request.blank('/bucket/object?partNumber=0&uploadId=X',
@@ -694,6 +697,9 @@ class BaseS3ApiMultiUpload(object):
body='part object')
status, headers, body = self.call_s3api(req)
self.assertEqual(self._get_error_code(body), 'InvalidArgument')
+ self.assertEqual(self._get_error_message(body),
+ 'Part number must be an integer between 1 and 10000, '
+ 'inclusive')
# part number must be < 10001
req = Request.blank('/bucket/object?partNumber=10001&uploadId=X',
@@ -703,6 +709,23 @@ class BaseS3ApiMultiUpload(object):
body='part object')
status, headers, body = self.call_s3api(req)
self.assertEqual(self._get_error_code(body), 'InvalidArgument')
+ self.assertEqual(self._get_error_message(body),
+ 'Part number must be an integer between 1 and 10000, '
+ 'inclusive')
+
+ with patch.object(self.s3api.conf, 'max_upload_part_num', 1000):
+ # part number must be < 1001
+ req = Request.blank(
+ '/bucket/object?partNumber=1001&uploadId=X',
+ environ={'REQUEST_METHOD': 'PUT'},
+ headers={'Authorization': 'AWS test:tester:hmac',
+ 'Date': self.get_date_header()},
+ body='part object')
+ status, headers, body = self.call_s3api(req)
+ self.assertEqual(self._get_error_code(body), 'InvalidArgument')
+ self.assertEqual(self._get_error_message(body),
+ 'Part number must be an integer between 1 and '
+ '1000, inclusive')
# without target bucket
req = Request.blank('/nobucket/object?partNumber=1&uploadId=X',
diff --git a/test/unit/common/middleware/s3api/test_obj.py b/test/unit/common/middleware/s3api/test_obj.py
index 180de31a88..617322d4c4 100644
--- a/test/unit/common/middleware/s3api/test_obj.py
+++ b/test/unit/common/middleware/s3api/test_obj.py
@@ -119,6 +119,12 @@ class BaseS3ApiObj(object):
if method == 'GET':
self.assertEqual(body, self.object_body)
+ def test_object_GET(self):
+ self._test_object_GETorHEAD('GET')
+
+ def test_object_HEAD(self):
+ self._test_object_GETorHEAD('HEAD')
+
def test_object_HEAD_error(self):
# HEAD does not return the body even an error response in the
# specifications of the REST API.
@@ -331,8 +337,136 @@ class BaseS3ApiObj(object):
expected_status='429 Slow Down')
self.assertEqual(code, 'SlowDown')
- def test_object_GET(self):
- self._test_object_GETorHEAD('GET')
+ def _test_non_slo_object_GETorHEAD_part_num(self, method, part_number):
+ req = Request.blank('/bucket/object?partNumber=%s' % part_number,
+ environ={'REQUEST_METHOD': method},
+ headers={'Authorization': 'AWS test:tester:hmac',
+ 'Date': self.get_date_header()})
+ status, headers, body = self.call_s3api(req)
+ self.assertEqual(status.split()[0], '206')
+ self.assertEqual(headers['content-length'], '5')
+ self.assertTrue('content-range' in headers)
+ self.assertEqual(headers['content-range'], 'bytes 0-4/5')
+ self.assertEqual(headers['content-type'], 'text/html')
+ # we'll want this for logging
+ self._assert_policy_index(req.headers, headers,
+ self.bucket_policy_index)
+ self.assertEqual(headers['etag'],
+ '"%s"' % self.response_headers['etag'])
+
+ if method == 'GET':
+ self.assertEqual(body, self.object_body)
+
+ def test_non_slo_object_GET_part_num(self):
+ self._test_non_slo_object_GETorHEAD_part_num('GET', 1)
+
+ def test_non_slo_object_HEAD_part_num(self):
+ self._test_non_slo_object_GETorHEAD_part_num('HEAD', 1)
+
+ def _do_test_non_slo_object_part_num_not_satisfiable(self, method,
+ part_number):
+ req = Request.blank('/bucket/object',
+ params={'partNumber': part_number},
+ headers={'Authorization': 'AWS test:tester:hmac',
+ 'Date': self.get_date_header()})
+ req.method = method
+ status, headers, body = self.call_s3api(req)
+ self.assertEqual(status.split()[0], '416')
+ return body
+
+ def test_non_slo_object_GET_part_num_not_satisfiable(self):
+ body = self._do_test_non_slo_object_part_num_not_satisfiable(
+ 'GET', '2')
+ self.assertEqual(self._get_error_code(body), 'InvalidPartNumber')
+ body = self._do_test_non_slo_object_part_num_not_satisfiable(
+ 'GET', '10000')
+ self.assertEqual(self._get_error_code(body), 'InvalidPartNumber')
+
+ def test_non_slo_object_HEAD_part_num_not_satisfiable(self):
+ body = self._do_test_non_slo_object_part_num_not_satisfiable(
+ 'HEAD', '2')
+ self.assertEqual(body, b'')
+ body = self._do_test_non_slo_object_part_num_not_satisfiable(
+ 'HEAD', '10000')
+ self.assertEqual(body, b'')
+
+ def _do_test_non_slo_object_part_num_invalid(self, method, part_number):
+ req = Request.blank('/bucket/object',
+ params={'partNumber': part_number},
+ headers={'Authorization': 'AWS test:tester:hmac',
+ 'Date': self.get_date_header()})
+ req.method = method
+ status, headers, body = self.call_s3api(req)
+ self.assertEqual(status.split()[0], '400')
+ return body
+
+ def test_non_slo_object_GET_part_num_invalid(self):
+ body = self._do_test_non_slo_object_part_num_invalid('GET', '0')
+ self.assertEqual(self._get_error_code(body), 'InvalidArgument')
+ body = self._do_test_non_slo_object_part_num_invalid('GET', '-1')
+ self.assertEqual(self._get_error_code(body), 'InvalidArgument')
+ body = self._do_test_non_slo_object_part_num_invalid('GET', '10001')
+ self.assertEqual(self._get_error_code(body), 'InvalidArgument')
+ with patch.object(self.s3api.conf, 'max_upload_part_num', 1000):
+ body = self._do_test_non_slo_object_part_num_invalid('GET', '1001')
+ self.assertEqual(self._get_error_code(body), 'InvalidArgument')
+ self.assertEqual(
+ self._get_error_message(body),
+ 'Part number must be an integer between 1 and 1000, inclusive')
+
+ body = self._do_test_non_slo_object_part_num_invalid('GET', 'foo')
+ self.assertEqual(self._get_error_code(body), 'InvalidArgument')
+ self.assertEqual(
+ self._get_error_message(body),
+ 'Part number must be an integer between 1 and 10000, inclusive')
+
+ def test_non_slo_object_HEAD_part_num_invalid(self):
+ body = self._do_test_non_slo_object_part_num_invalid('HEAD', '0')
+ self.assertEqual(body, b'')
+ body = self._do_test_non_slo_object_part_num_invalid('HEAD', '-1')
+ self.assertEqual(body, b'')
+ body = self._do_test_non_slo_object_part_num_invalid('HEAD', '10001')
+ self.assertEqual(body, b'')
+ body = self._do_test_non_slo_object_part_num_invalid('HEAD', 'foo')
+ self.assertEqual(body, b'')
+
+ def test_non_slo_object_GET_part_num_and_range(self):
+ req = Request.blank('/bucket/object',
+ params={'partNumber': '1'},
+ headers={'Range': 'bytes=1-2',
+ 'Authorization': 'AWS test:tester:hmac',
+ 'Date': self.get_date_header()})
+ req.method = 'GET'
+ status, headers, body = self.call_s3api(req)
+ self.assertEqual(status.split()[0], '400')
+ self.assertEqual(self._get_error_code(body), 'InvalidRequest')
+ self.assertEqual(
+ self._get_error_message(body),
+ 'Cannot specify both Range header and partNumber query parameter')
+
+ # partNumber + Range error trumps bad partNumber
+ req = Request.blank('/bucket/object',
+ params={'partNumber': '0'},
+ headers={'Range': 'bytes=1-2',
+ 'Authorization': 'AWS test:tester:hmac',
+ 'Date': self.get_date_header()})
+ req.method = 'GET'
+ status, headers, body = self.call_s3api(req)
+ self.assertEqual(status.split()[0], '400')
+ self.assertEqual(self._get_error_code(body), 'InvalidRequest')
+ self.assertEqual(
+ self._get_error_message(body),
+ 'Cannot specify both Range header and partNumber query parameter')
+
+ def test_non_slo_object_HEAD_part_num_and_range(self):
+ req = Request.blank('/bucket/object',
+ params={'partNumber': '1'},
+ headers={'Range': 'bytes=1-2',
+ 'Authorization': 'AWS test:tester:hmac',
+ 'Date': self.get_date_header()})
+ req.method = 'HEAD'
+ status, headers, body = self.call_s3api(req)
+ self.assertEqual(status.split()[0], '400')
def test_object_GET_Range(self):
req = Request.blank('/bucket/object',
@@ -1105,9 +1239,6 @@ class TestS3ApiObj(BaseS3ApiObj, S3ApiTestCase):
swob.HTTPRequestedRangeNotSatisfiable)
self.assertEqual(code, 'InvalidRange')
- def test_object_HEAD(self):
- self._test_object_GETorHEAD('HEAD')
-
@patch_policies([
StoragePolicy(0, 'gold', is_default=True),
StoragePolicy(1, 'silver')])
diff --git a/test/unit/common/middleware/s3api/test_s3request.py b/test/unit/common/middleware/s3api/test_s3request.py
index 7391b35f82..0b2216e741 100644
--- a/test/unit/common/middleware/s3api/test_s3request.py
+++ b/test/unit/common/middleware/s3api/test_s3request.py
@@ -32,7 +32,8 @@ from swift.common.middleware.s3api.s3request import S3Request, \
S3AclRequest, SigV4Request, SIGV4_X_AMZ_DATE_FORMAT, HashingInput
from swift.common.middleware.s3api.s3response import InvalidArgument, \
NoSuchBucket, InternalError, ServiceUnavailable, \
- AccessDenied, SignatureDoesNotMatch, RequestTimeTooSkewed, BadDigest
+ AccessDenied, SignatureDoesNotMatch, RequestTimeTooSkewed, BadDigest, \
+ InvalidPartArgument, InvalidPartNumber, InvalidRequest
from swift.common.utils import md5
from test.debug_logger import debug_logger
@@ -998,6 +999,104 @@ class TestRequest(S3ApiTestCase):
sigv4_req._canonical_request().endswith(sha256_of_nothing.upper()))
self.assertTrue(sigv4_req.check_signature('secret'))
+ def test_validate_part_number(self):
+ sw_req = Request.blank('/nojunk',
+ environ={'REQUEST_METHOD': 'GET'},
+ headers={
+ 'Authorization': 'AWS test:tester:hmac',
+ 'Date': self.get_date_header()})
+ req = S3Request(sw_req.environ)
+ self.assertIsNone(req.validate_part_number())
+
+ # ok
+ sw_req = Request.blank('/nojunk?partNumber=102',
+ environ={'REQUEST_METHOD': 'GET'},
+ headers={
+ 'Authorization': 'AWS test:tester:hmac',
+ 'Date': self.get_date_header()})
+ req = S3Request(sw_req.environ)
+ self.assertEqual(102, req.validate_part_number())
+ req = S3Request(sw_req.environ,
+ conf=Config({'max_upload_part_num': 100}))
+ self.assertEqual(102, req.validate_part_number(102))
+ req = S3Request(sw_req.environ,
+ conf=Config({'max_upload_part_num': 102}))
+ self.assertEqual(102, req.validate_part_number(102))
+
+ def test_validate_part_number_invalid_argument(self):
+ def check_invalid_argument(part_num, max_parts, parts_count, exp_max):
+ sw_req = Request.blank('/nojunk?partNumber=%s' % part_num,
+ environ={'REQUEST_METHOD': 'GET'},
+ headers={
+ 'Authorization': 'AWS test:tester:hmac',
+ 'Date': self.get_date_header()})
+ req = S3Request(sw_req.environ,
+ conf=Config({'max_upload_part_num': max_parts}))
+ with self.assertRaises(InvalidPartArgument) as cm:
+ req.validate_part_number(parts_count=parts_count)
+ self.assertEqual('400 Bad Request', str(cm.exception))
+ self.assertIn(
+ b'Part number must be an integer between 1 and %d' % exp_max,
+ cm.exception.body)
+
+ check_invalid_argument(102, 99, None, 99)
+ check_invalid_argument(102, 100, 99, 100)
+ check_invalid_argument(102, 100, 101, 101)
+ check_invalid_argument(102, 101, 100, 101)
+ check_invalid_argument(102, 101, 101, 101)
+ check_invalid_argument('banana', 1000, None, 1000)
+ check_invalid_argument(0, 10000, None, 10000)
+
+ def test_validate_part_number_invalid_part_number(self):
+ def check_invalid_part_num(part_num, max_parts, parts_count):
+ sw_req = Request.blank('/nojunk?partNumber=%s' % part_num,
+ environ={'REQUEST_METHOD': 'GET'},
+ headers={
+ 'Authorization': 'AWS test:tester:hmac',
+ 'Date': self.get_date_header()})
+ req = S3Request(sw_req.environ,
+ conf=Config({'max_upload_part_num': max_parts}))
+ with self.assertRaises(InvalidPartNumber) as cm:
+ req.validate_part_number(parts_count=parts_count)
+ self.assertEqual('416 Requested Range Not Satisfiable',
+ str(cm.exception))
+ self.assertIn(b'The requested partnumber is not satisfiable',
+ cm.exception.body)
+
+ check_invalid_part_num(102, 10000, 1)
+ check_invalid_part_num(102, 102, 101)
+ check_invalid_part_num(102, 10000, 101)
+
+ def test_validate_part_number_with_range_header(self):
+ sw_req = Request.blank('/nojunk?partNumber=1',
+ environ={'REQUEST_METHOD': 'GET'},
+ headers={
+ 'Range': 'bytes=1-2',
+ 'Authorization': 'AWS test:tester:hmac',
+ 'Date': self.get_date_header()})
+ req = S3Request(sw_req.environ)
+ with self.assertRaises(InvalidRequest) as cm:
+ req.validate_part_number()
+ self.assertEqual('400 Bad Request',
+ str(cm.exception))
+ self.assertIn(b'Cannot specify both Range header and partNumber query '
+ b'parameter', cm.exception.body)
+
+ # bad part number AND Range header
+ sw_req = Request.blank('/nojunk?partNumber=0',
+ environ={'REQUEST_METHOD': 'GET'},
+ headers={
+ 'Range': 'bytes=1-2',
+ 'Authorization': 'AWS test:tester:hmac',
+ 'Date': self.get_date_header()})
+ req = S3Request(sw_req.environ)
+ with self.assertRaises(InvalidRequest) as cm:
+ req.validate_part_number()
+ self.assertEqual('400 Bad Request',
+ str(cm.exception))
+ self.assertIn(b'Cannot specify both Range header and partNumber query '
+ b'parameter', cm.exception.body)
+
class TestSigV4Request(S3ApiTestCase):
def setUp(self):
@@ -1204,7 +1303,7 @@ class TestSigV4Request(S3ApiTestCase):
return SigV4Request(req.environ, None, config)
s3req = make_s3req(Config(), '/bkt', {'partNumber': '3'})
- self.assertEqual(controllers.multi_upload.PartController,
+ self.assertEqual(controllers.ObjectController,
s3req.controller)
s3req = make_s3req(Config(), '/bkt', {'uploadId': '4'})
@@ -1235,6 +1334,41 @@ class TestSigV4Request(S3ApiTestCase):
self.assertEqual(controllers.ServiceController,
s3req.controller)
+ def test_controller_for_multipart_upload_requests(self):
+ environ = {
+ 'HTTP_HOST': 'bucket.s3.test.com',
+ 'REQUEST_METHOD': 'PUT'}
+ x_amz_date = self.get_v4_amz_date_header()
+ auth = ('AWS4-HMAC-SHA256 '
+ 'Credential=test/%s/us-east-1/s3/aws4_request,'
+ 'SignedHeaders=host;x-amz-content-sha256;x-amz-date,'
+ 'Signature=X' % self.get_v4_amz_date_header().split('T', 1)[0])
+ headers = {
+ 'Authorization': auth,
+ 'X-Amz-Content-SHA256': '0123456789',
+ 'Date': self.get_date_header(),
+ 'X-Amz-Date': x_amz_date}
+
+ def make_s3req(config, path, params):
+ req = Request.blank(path, environ=environ, headers=headers,
+ params=params)
+ return SigV4Request(req.environ, None, config)
+
+ s3req = make_s3req(Config(), '/bkt', {'partNumber': '3',
+ 'uploadId': '4'})
+ self.assertEqual(controllers.multi_upload.PartController,
+ s3req.controller)
+
+ s3req = make_s3req(Config(), '/bkt', {'partNumber': '3'})
+ self.assertEqual(controllers.multi_upload.PartController,
+ s3req.controller)
+
+ s3req = make_s3req(Config(), '/bkt', {'uploadId': '4',
+ 'partNumber': '3',
+ 'copySource': 'bkt2/obj2'})
+ self.assertEqual(controllers.multi_upload.PartController,
+ s3req.controller)
+
class TestHashingInput(S3ApiTestCase):
def test_good(self):
diff --git a/test/unit/common/middleware/s3api/test_utils.py b/test/unit/common/middleware/s3api/test_utils.py
index 9fc0854f52..01cc07117f 100644
--- a/test/unit/common/middleware/s3api/test_utils.py
+++ b/test/unit/common/middleware/s3api/test_utils.py
@@ -181,6 +181,7 @@ class TestConfig(unittest.TestCase):
del conf.allow_no_owner
del conf.allowable_clock_skew
del conf.ratelimit_as_client_error
+ del conf.max_upload_part_num
self.assertEqual({}, conf)
def test_update(self):