Merge "s3api: add more MPU cross-compat tests"
This commit is contained in:
Zuul
Gerrit Code Review
commit
c2f2521400
@ -144,6 +144,18 @@ def get_s3_client(user=1, signature_version='s3v4', addressing_style='path'):
|
||||
)
|
||||
|
||||
|
||||
def etag_from_resp(response):
|
||||
return response['ETag']
|
||||
|
||||
|
||||
def code_from_error(error):
|
||||
return error.response['Error']['Code']
|
||||
|
||||
|
||||
def status_from_error(error):
|
||||
return error.response['ResponseMetadata']['HTTPStatusCode']
|
||||
|
||||
|
||||
TEST_PREFIX = 's3api-test-'
|
||||
|
||||
|
||||
|
||||
@ -13,12 +13,12 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from test.s3api import BaseS3TestCase
|
||||
from test.s3api import BaseS3TestCase, status_from_error, code_from_error, \
|
||||
etag_from_resp
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
|
||||
class BaseMultiPartUploadTestCase(BaseS3TestCase):
|
||||
|
||||
maxDiff = None
|
||||
|
||||
def setUp(self):
|
||||
@ -178,6 +178,100 @@ class TestMultiPartUpload(BaseMultiPartUploadTestCase):
|
||||
self.assertIn(preamble, err_msg)
|
||||
return int(err_msg[len(preamble):].split(',')[0])
|
||||
|
||||
def create_mpu(self, key_name):
|
||||
create_mpu_resp = self.client.create_multipart_upload(
|
||||
Bucket=self.bucket_name, Key=key_name)
|
||||
self.assertEqual(200, create_mpu_resp[
|
||||
'ResponseMetadata']['HTTPStatusCode'])
|
||||
return create_mpu_resp['UploadId']
|
||||
|
||||
def list_mpus(self):
|
||||
list_mpu_resp = self.client.list_multipart_uploads(
|
||||
Bucket=self.bucket_name)
|
||||
self.assertEqual(200, list_mpu_resp[
|
||||
'ResponseMetadata']['HTTPStatusCode'])
|
||||
mpus = list_mpu_resp.get('Uploads', [])
|
||||
return [(mpu['Key'], mpu['UploadId']) for mpu in mpus]
|
||||
|
||||
def list_parts(self, key_name, upload_id):
|
||||
list_parts_resp = self.client.list_parts(
|
||||
Bucket=self.bucket_name, Key=key_name,
|
||||
UploadId=upload_id,
|
||||
)
|
||||
self.assertEqual(200, list_parts_resp[
|
||||
'ResponseMetadata']['HTTPStatusCode'])
|
||||
return [{k: p[k] for k in ('ETag', 'PartNumber')}
|
||||
for p in list_parts_resp.get('Parts', [])]
|
||||
|
||||
def upload_part_indexes(self, key_name, upload_id, part_indexes):
|
||||
parts = []
|
||||
for i in part_indexes:
|
||||
body = ('%d' % i) * 5 * (2 ** 20)
|
||||
part_resp = self.client.upload_part(
|
||||
Body=body, Bucket=self.bucket_name, Key=key_name,
|
||||
PartNumber=i, UploadId=upload_id)
|
||||
self.assertEqual(200, part_resp[
|
||||
'ResponseMetadata']['HTTPStatusCode'])
|
||||
parts.append({
|
||||
'ETag': part_resp['ETag'],
|
||||
'PartNumber': i,
|
||||
})
|
||||
self.assertEqual(parts, self.list_parts(key_name, upload_id))
|
||||
return parts
|
||||
|
||||
def upload_parts(self, key_name, upload_id, num_parts):
|
||||
return self.upload_part_indexes(key_name, upload_id,
|
||||
range(1, num_parts + 1))
|
||||
|
||||
def complete_mpu(self, key_name, upload_id, parts):
|
||||
complete_mpu_resp = self.client.complete_multipart_upload(
|
||||
Bucket=self.bucket_name, Key=key_name,
|
||||
MultipartUpload={
|
||||
'Parts': parts,
|
||||
},
|
||||
UploadId=upload_id,
|
||||
)
|
||||
self.assertEqual(200, complete_mpu_resp[
|
||||
'ResponseMetadata']['HTTPStatusCode'])
|
||||
return complete_mpu_resp
|
||||
|
||||
def abort_mpu(self, key_name, upload_id):
|
||||
abort_resp = self.client.abort_multipart_upload(
|
||||
Bucket=self.bucket_name,
|
||||
Key=key_name,
|
||||
UploadId=upload_id,
|
||||
)
|
||||
self.assertEqual(204, abort_resp[
|
||||
'ResponseMetadata']['HTTPStatusCode'])
|
||||
return abort_resp
|
||||
|
||||
def head_part(self, key_name, part_number):
|
||||
resp = self.client.head_object(Bucket=self.bucket_name, Key=key_name,
|
||||
PartNumber=part_number)
|
||||
self.assertEqual(206, resp[
|
||||
'ResponseMetadata']['HTTPStatusCode'])
|
||||
return resp
|
||||
|
||||
def get_part(self, key_name, part_number):
|
||||
resp = self.client.get_object(Bucket=self.bucket_name, Key=key_name,
|
||||
PartNumber=part_number)
|
||||
self.assertEqual(206, resp[
|
||||
'ResponseMetadata']['HTTPStatusCode'])
|
||||
return resp
|
||||
|
||||
def delete_object(self, key_name):
|
||||
delete_resp = self.client.delete_object(
|
||||
Bucket=self.bucket_name, Key=key_name)
|
||||
self.assertEqual(204, delete_resp[
|
||||
'ResponseMetadata']['HTTPStatusCode'])
|
||||
|
||||
def assert_object_not_found(self, key_name):
|
||||
with self.assertRaises(ClientError) as cm:
|
||||
self.client.head_object(
|
||||
Bucket=self.bucket_name, Key=key_name,
|
||||
)
|
||||
self.assertEqual(404, status_from_error(cm.exception))
|
||||
|
||||
def test_basic_upload(self):
|
||||
key_name = self.create_name('key')
|
||||
create_mpu_resp = self.client.create_multipart_upload(
|
||||
@ -185,6 +279,15 @@ class TestMultiPartUpload(BaseMultiPartUploadTestCase):
|
||||
self.assertEqual(200, create_mpu_resp[
|
||||
'ResponseMetadata']['HTTPStatusCode'])
|
||||
upload_id = create_mpu_resp['UploadId']
|
||||
|
||||
list_mpu_resp = self.client.list_multipart_uploads(
|
||||
Bucket=self.bucket_name)
|
||||
self.assertEqual(200, list_mpu_resp[
|
||||
'ResponseMetadata']['HTTPStatusCode'])
|
||||
found_uploads = list_mpu_resp.get('Uploads', [])
|
||||
self.assertEqual(1, len(found_uploads), found_uploads)
|
||||
self.assertEqual(upload_id, found_uploads[0]['UploadId'])
|
||||
|
||||
parts = []
|
||||
for i in range(1, 3):
|
||||
body = ('%d' % i) * 5 * (2 ** 20)
|
||||
@ -578,6 +681,141 @@ class TestMultiPartUpload(BaseMultiPartUploadTestCase):
|
||||
'ResponseMetadata']['HTTPStatusCode'])
|
||||
self.assertEqual([], list_mpu_resp.get('Uploads', []))
|
||||
|
||||
def test_create_upload_complete_complete(self):
|
||||
key_name = self.create_name('key')
|
||||
upload_id = self.create_mpu(key_name)
|
||||
parts = self.upload_parts(key_name, upload_id, 2)
|
||||
self.complete_mpu(key_name, upload_id, parts)
|
||||
# repeat complete gets 200
|
||||
self.complete_mpu(key_name, upload_id, parts)
|
||||
|
||||
def test_create_upload_complete_delete_complete(self):
|
||||
key_name = self.create_name('key')
|
||||
upload_id = self.create_mpu(key_name)
|
||||
parts = self.upload_parts(key_name, upload_id, 2)
|
||||
self.complete_mpu(key_name, upload_id, parts)
|
||||
self.delete_object(key_name)
|
||||
self.assert_object_not_found(key_name)
|
||||
# repeat complete gets 404
|
||||
with self.assertRaises(ClientError) as cm:
|
||||
self.complete_mpu(key_name, upload_id, parts)
|
||||
self.assertEqual(404, status_from_error(cm.exception))
|
||||
self.assertEqual('NoSuchUpload', code_from_error(cm.exception))
|
||||
|
||||
def test_create_upload_abort_complete(self):
|
||||
key_name = self.create_name('key')
|
||||
upload_id = self.create_mpu(key_name)
|
||||
parts = self.upload_parts(key_name, upload_id, 1)
|
||||
self.abort_mpu(key_name, upload_id)
|
||||
with self.assertRaises(ClientError) as cm:
|
||||
self.complete_mpu(key_name, upload_id, parts)
|
||||
self.assertEqual(404, status_from_error(cm.exception))
|
||||
self.assertEqual('NoSuchUpload', code_from_error(cm.exception))
|
||||
|
||||
def test_abort_bogus_id(self):
|
||||
key_name = self.create_name('key')
|
||||
upload_id = self.create_mpu(key_name)
|
||||
with self.assertRaises(ClientError) as cm:
|
||||
self.abort_mpu(key_name, upload_id + 'x')
|
||||
self.assertEqual(404, status_from_error(cm.exception))
|
||||
self.assertEqual('NoSuchUpload', code_from_error(cm.exception))
|
||||
|
||||
def test_create_upload_abort_list_parts(self):
|
||||
key_name = self.create_name('key')
|
||||
upload_id = self.create_mpu(key_name)
|
||||
self.upload_parts(key_name, upload_id, 1)
|
||||
self.abort_mpu(key_name, upload_id)
|
||||
with self.assertRaises(ClientError) as cm:
|
||||
self.list_parts(key_name, upload_id)
|
||||
self.assertEqual(404, status_from_error(cm.exception))
|
||||
self.assertEqual('NoSuchUpload', code_from_error(cm.exception))
|
||||
|
||||
def test_create_upload_abort_upload(self):
|
||||
key_name = self.create_name('key')
|
||||
upload_id = self.create_mpu(key_name)
|
||||
self.upload_parts(key_name, upload_id, 1)
|
||||
self.abort_mpu(key_name, upload_id)
|
||||
with self.assertRaises(ClientError) as cm:
|
||||
self.upload_parts(key_name, upload_id, 1)
|
||||
self.assertEqual(404, status_from_error(cm.exception))
|
||||
self.assertEqual('NoSuchUpload', code_from_error(cm.exception))
|
||||
|
||||
def test_create_upload_complete_subset_of_parts_list(self):
|
||||
key_name = self.create_name('key')
|
||||
upload_id = self.create_mpu(key_name)
|
||||
parts = self.upload_parts(key_name, upload_id, 3)
|
||||
subset_parts = parts[:2]
|
||||
self.complete_mpu(key_name, upload_id, subset_parts)
|
||||
|
||||
response = self.head_part(key_name, 1)
|
||||
self.assertTrue(etag_from_resp(response).endswith('-2"'))
|
||||
response2 = self.head_part(key_name, 2)
|
||||
self.assertEqual(etag_from_resp(response), etag_from_resp(response2))
|
||||
|
||||
with self.assertRaises(ClientError) as cm:
|
||||
self.get_part(key_name, 3)
|
||||
self.assertEqual(416, status_from_error(cm.exception))
|
||||
self.assertEqual('InvalidPartNumber', code_from_error(cm.exception))
|
||||
|
||||
def test_create_upload_complete_subset_of_parts_list_with_gaps(self):
|
||||
# only a subset of uploaded parts are referenced in complete
|
||||
key_name = self.create_name('key')
|
||||
upload_id = self.create_mpu(key_name)
|
||||
parts = self.upload_parts(key_name, upload_id, 3)
|
||||
subset_parts = [parts[0], parts[2]]
|
||||
self.complete_mpu(key_name, upload_id, subset_parts)
|
||||
# GET partNumbers are not same as uploaded part numbers!
|
||||
self.head_part(key_name, 1)
|
||||
response = self.head_part(key_name, 1)
|
||||
self.assertTrue(etag_from_resp(response).endswith('-2"'))
|
||||
response2 = self.head_part(key_name, 2)
|
||||
self.assertEqual(etag_from_resp(response), etag_from_resp(response2))
|
||||
|
||||
with self.assertRaises(ClientError) as cm:
|
||||
self.get_part(key_name, 3)
|
||||
self.assertEqual(416, status_from_error(cm.exception))
|
||||
self.assertEqual('InvalidPartNumber', code_from_error(cm.exception))
|
||||
|
||||
def test_create_upload_complete_parts_list_with_gaps(self):
|
||||
# only a subset of part indexes are uploaded
|
||||
key_name = self.create_name('key')
|
||||
upload_id = self.create_mpu(key_name)
|
||||
parts = self.upload_part_indexes(key_name, upload_id, [1, 1000])
|
||||
actual_parts = self.list_parts(key_name, upload_id)
|
||||
self.assertEqual([1, 1000], [p['PartNumber'] for p in actual_parts])
|
||||
self.complete_mpu(key_name, upload_id, parts)
|
||||
# GET partNumbers are not same as uploaded part numbers!
|
||||
self.head_part(key_name, 1)
|
||||
response = self.head_part(key_name, 1)
|
||||
self.assertTrue(etag_from_resp(response).endswith('-2"'))
|
||||
response2 = self.head_part(key_name, 2)
|
||||
self.assertEqual(etag_from_resp(response), etag_from_resp(response2))
|
||||
|
||||
with self.assertRaises(ClientError) as cm:
|
||||
self.get_part(key_name, 3)
|
||||
self.assertEqual(416, status_from_error(cm.exception))
|
||||
self.assertEqual('InvalidPartNumber', code_from_error(cm.exception))
|
||||
|
||||
def test_create_upload_complete_misordered_parts(self):
|
||||
key_name = self.create_name('key')
|
||||
upload_id = self.create_mpu(key_name)
|
||||
parts = self.upload_parts(key_name, upload_id, 3)
|
||||
with self.assertRaises(ClientError) as cm:
|
||||
self.complete_mpu(key_name, upload_id, list(reversed(parts)))
|
||||
self.assertEqual(400, status_from_error(cm.exception))
|
||||
self.assertEqual('InvalidPartOrder', code_from_error(cm.exception))
|
||||
|
||||
def test_create_list_mpus_abort_list_mpus(self):
|
||||
key_name = self.create_name('key')
|
||||
upload_id = self.create_mpu(key_name)
|
||||
# our upload is in progress
|
||||
found_uploads = self.list_mpus()
|
||||
self.assertEqual([(key_name, upload_id)], found_uploads)
|
||||
self.assertEqual([], self.list_parts(key_name, upload_id))
|
||||
self.abort_mpu(key_name, upload_id)
|
||||
# no more inprogress uploads
|
||||
self.assertEqual([], self.list_mpus())
|
||||
|
||||
def test_complete_multipart_upload_malformed_request(self):
|
||||
key_name = self.create_name('key')
|
||||
create_mpu_resp = self.client.create_multipart_upload(
|
||||
|
||||
Loading…
Reference in New Issue
Block a user