openstack/swift
Code Issues Proposed changes

Merge "Start transition to boto3 in s3api tests."

Browse Source
This commit is contained in:
Zuul 2019-05-22 05:25:13 +00:00 committed by Gerrit Code Review
commit 5be0e9ff09
6 changed files with 322 additions and 310 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
doc/requirements.txt
View File

@ -2,7 +2,8 @@
# of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later.
# this is required for the docs build jobs
sphinx>=1.6.2 # BSD
sphinx>=1.6.2,<2.0.0;python_version=='2.7' # BSD
sphinx>=1.6.2;python_version>='3.4' # BSD
openstackdocstheme>=1.11.0 # Apache-2.0
reno>=1.8.0 # Apache-2.0
os-api-ref>=1.0.0 # Apache-2.0

2
lower-constraints.txt
View File

@ -3,6 +3,8 @@ asn1crypto==0.24.0
Babel==2.5.3
bandit==1.1.0
boto==2.32.1
boto3==1.9
botocore==1.12
castellan==0.13.0
certifi==2018.1.18
cffi==1.11.5

2
test-requirements.txt
View File

@ -15,6 +15,8 @@ python-keystoneclient!=2.1.0,>=2.0.0 # Apache-2.0
reno>=1.8.0 # Apache-2.0
python-openstackclient>=3.12.0
boto>=2.32.1
boto3>=1.9
botocore>=1.12
requests-mock>=1.2.0 # Apache-2.0
fixtures>=3.0.0 # Apache-2.0/BSD
keystonemiddleware>=4.17.0 # Apache-2.0

23
test/functional/s3api/__init__.py
View File

@ -16,7 +16,8 @@
import unittest2
import traceback
import test.functional as tf
from test.functional.s3api.s3_test_client import Connection
from test.functional.s3api.s3_test_client import (
Connection, get_boto3_conn, tear_down_s3)
def setUpModule():
@ -59,3 +60,23 @@ class S3ApiBase(unittest2.TestCase):
if etag is not None:
self.assertTrue('etag' in headers) # sanity
self.assertEqual(etag, headers['etag'].strip('"'))
class S3ApiBaseBoto3(S3ApiBase):
def setUp(self):
if 's3api' not in tf.cluster_info:
raise tf.SkipTest('s3api middleware is not enabled')
try:
self.conn = get_boto3_conn()
self.endpoint_url = self.conn._endpoint.host
self.access_key = self.conn._request_signer._credentials.access_key
self.region = self.conn._client_config.region_name
tear_down_s3(self.conn)
except Exception:
message = '%s got an error during initialize process.\n\n%s' % \
(self.method_name, traceback.format_exc())
# TODO: Find a way to make this go to FAIL instead of Error
self.fail(message)
def tearDown(self):
tear_down_s3(self.conn)

49
test/functional/s3api/s3_test_client.py
View File

@ -15,6 +15,8 @@
import os
import test.functional as tf
import boto3
from botocore.exceptions import ClientError
from boto.s3.connection import S3Connection, OrdinaryCallingFormat, \
S3ResponseError
import six
@ -135,6 +137,53 @@ class Connection(object):
return url, {}
def get_boto3_conn(aws_access_key='test:tester', aws_secret_key='testing'):
host = tf.config['auth_host']
port = int(tf.config['auth_port'])
config = boto3.session.Config(s3={'addressing_style': 'path'})
return boto3.client(
's3', aws_access_key_id=aws_access_key,
aws_secret_access_key=aws_secret_key,
config=config, region_name='us-east-1', use_ssl=False,
endpoint_url='http://{}:{}'.format(host, port))
def tear_down_s3(conn):
"""
Reset all swift environment to keep clean. As a result by calling this
method, we can assume the backend swift keeps no containers and no
objects on this connection's account.
"""
exceptions = []
for i in range(RETRY_COUNT):
try:
resp = conn.list_buckets()
buckets = [bucket['Name'] for bucket in resp.get('Buckets', [])]
for bucket in buckets:
try:
resp = conn.list_multipart_uploads(Bucket=bucket)
for upload in resp.get('Uploads', []):
conn.abort_multipart_upload(
Bucket=bucket,
Key=upload['Key'],
UploadId=upload['UploadId'])
resp = conn.list_objects(Bucket=bucket)
for obj in resp.get('Contents', []):
conn.delete_object(Bucket=bucket, Key=obj['Key'])
conn.delete_bucket(Bucket=bucket)
except ClientError as e:
# 404 means NoSuchBucket, NoSuchKey, or NoSuchUpload
if e.response['ResponseMetadata']['HTTPStatusCode'] != 404:
raise
except Exception as e:
exceptions.append(''.join(
traceback.format_exception(*sys.exc_info())))
if exceptions:
exceptions.insert(0, 'Too many errors to continue:')
raise Exception('\n========\n'.join(exceptions))
# TODO: make sure where this function is used
def get_admin_connection():
"""

553
test/functional/s3api/test_bucket.py
View File

@ -13,16 +13,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import botocore
import datetime
import unittest2
import os
import test.functional as tf
from swift.common.utils import config_true_value
from swift.common.middleware.s3api.etree import fromstring, tostring, Element, \
SubElement
from test.functional.s3api import S3ApiBase
from test.functional.s3api.s3_test_client import Connection
from test.functional.s3api.utils import get_error_code
from test.functional.s3api import S3ApiBaseBoto3
from test.functional.s3api.s3_test_client import get_boto3_conn
def setUpModule():
@ -33,14 +32,21 @@ def tearDownModule():
tf.teardown_package()
class TestS3ApiBucket(S3ApiBase):
def setUp(self):
super(TestS3ApiBucket, self).setUp()
def _gen_location_xml(self, location):
elem = Element('CreateBucketConfiguration')
SubElement(elem, 'LocationConstraint').text = location
return tostring(elem)
class TestS3ApiBucket(S3ApiBaseBoto3):
def _validate_object_listing(self, resp_objects, req_objects,
expect_owner=True):
self.assertEqual(len(resp_objects), len(req_objects))
for i, obj in enumerate(resp_objects):
self.assertEqual(obj['Key'], req_objects[i])
self.assertEqual(type(obj['LastModified']), datetime.datetime)
self.assertIn('ETag', obj)
self.assertIn('Size', obj)
self.assertEqual(obj['StorageClass'], 'STANDARD')
if expect_owner:
self.assertEqual(obj['Owner']['ID'], self.access_key)
self.assertEqual(obj['Owner']['DisplayName'], self.access_key)
else:
self.assertNotIn('Owner', obj)
def test_bucket(self):
bucket = 'bucket'
@ -48,112 +54,95 @@ class TestS3ApiBucket(S3ApiBase):
'max_bucket_listing', 1000)
# PUT Bucket
status, headers, body = self.conn.make_request('PUT', bucket)
self.assertEqual(status, 200)
resp = self.conn.create_bucket(Bucket=bucket)
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
headers = resp['ResponseMetadata']['HTTPHeaders']
self.assertCommonResponseHeaders(headers)
self.assertIn(headers['location'], (
'/' + bucket, # swob won't touch it...
# but webob (which we get because of auth_token) *does*
'http://%s%s/%s' % (
self.conn.host,
'' if self.conn.port == 80 else ':%d' % self.conn.port,
bucket),
# This is all based on the Host header the client provided,
# and boto will double-up ports for sig v4. See
# - https://github.com/boto/boto/issues/2623
# - https://github.com/boto/boto/issues/3716
# with proposed fixes at
# - https://github.com/boto/boto/pull/3513
# - https://github.com/boto/boto/pull/3676
'http://%s%s:%d/%s' % (
self.conn.host,
'' if self.conn.port == 80 else ':%d' % self.conn.port,
self.conn.port,
bucket),
'%s/%s' % (self.endpoint_url, bucket),
))
self.assertEqual(headers['content-length'], '0')
# GET Bucket(Without Object)
status, headers, body = self.conn.make_request('GET', bucket)
self.assertEqual(status, 200)
resp = self.conn.list_objects(Bucket=bucket)
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
headers = resp['ResponseMetadata']['HTTPHeaders']
self.assertCommonResponseHeaders(headers)
self.assertIsNotNone(headers['content-type'])
self.assertEqual(headers['content-length'], str(len(body)))
# TODO; requires consideration
# self.assertEqual(headers['transfer-encoding'], 'chunked')
elem = fromstring(body, 'ListBucketResult')
self.assertEqual(elem.find('Name').text, bucket)
self.assertIsNone(elem.find('Prefix').text)
self.assertIsNone(elem.find('Marker').text)
self.assertEqual(
elem.find('MaxKeys').text, str(max_bucket_listing))
self.assertEqual(elem.find('IsTruncated').text, 'false')
objects = elem.findall('./Contents')
self.assertEqual(list(objects), [])
self.assertEqual(resp['Name'], bucket)
self.assertEqual(resp['Prefix'], '')
self.assertEqual(resp['Marker'], '')
self.assertEqual(resp['MaxKeys'], max_bucket_listing)
self.assertFalse(resp['IsTruncated'])
self.assertNotIn('Contents', bucket)
# GET Bucket(With Object)
req_objects = ('object', 'object2')
req_objects = ['object', 'object2']
for obj in req_objects:
self.conn.make_request('PUT', bucket, obj)
status, headers, body = self.conn.make_request('GET', bucket)
self.assertEqual(status, 200)
self.conn.put_object(Bucket=bucket, Key=obj, Body=b'')
resp = self.conn.list_objects(Bucket=bucket)
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
elem = fromstring(body, 'ListBucketResult')
self.assertEqual(elem.find('Name').text, bucket)
self.assertIsNone(elem.find('Prefix').text)
self.assertIsNone(elem.find('Marker').text)
self.assertEqual(elem.find('MaxKeys').text,
str(max_bucket_listing))
self.assertEqual(elem.find('IsTruncated').text, 'false')
resp_objects = elem.findall('./Contents')
self.assertEqual(len(list(resp_objects)), 2)
for o in resp_objects:
self.assertIn(o.find('Key').text, req_objects)
self.assertIsNotNone(o.find('LastModified').text)
self.assertRegexpMatches(
o.find('LastModified').text,
r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
self.assertIsNotNone(o.find('ETag').text)
self.assertIsNotNone(o.find('Size').text)
self.assertIsNotNone(o.find('StorageClass').text)
self.assertEqual(o.find('Owner/ID').text, self.conn.user_id)
self.assertEqual(o.find('Owner/DisplayName').text,
self.conn.user_id)
self.assertEqual(resp['Name'], bucket)
self.assertEqual(resp['Prefix'], '')
self.assertEqual(resp['Marker'], '')
self.assertEqual(resp['MaxKeys'], max_bucket_listing)
self.assertFalse(resp['IsTruncated'])
self._validate_object_listing(resp['Contents'], req_objects)
# HEAD Bucket
status, headers, body = self.conn.make_request('HEAD', bucket)
self.assertEqual(status, 200)
resp = self.conn.head_bucket(Bucket=bucket)
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
headers = resp['ResponseMetadata']['HTTPHeaders']
self.assertCommonResponseHeaders(headers)
self.assertIsNotNone(headers['content-type'])
self.assertEqual(headers['content-length'], str(len(body)))
# TODO; requires consideration
# self.assertEqual(headers['transfer-encoding'], 'chunked')
# DELETE Bucket
for obj in req_objects:
self.conn.make_request('DELETE', bucket, obj)
status, headers, body = self.conn.make_request('DELETE', bucket)
self.assertEqual(status, 204)
self.conn.delete_object(Bucket=bucket, Key=obj)
resp = self.conn.delete_bucket(Bucket=bucket)
self.assertEqual(204, resp['ResponseMetadata']['HTTPStatusCode'])
self.assertCommonResponseHeaders(headers)
self.assertCommonResponseHeaders(
resp['ResponseMetadata']['HTTPHeaders'])
def test_put_bucket_error(self):
status, headers, body = \
self.conn.make_request('PUT', 'bucket+invalid')
self.assertEqual(get_error_code(body), 'InvalidBucketName')
event_system = self.conn.meta.events
event_system.unregister(
'before-parameter-build.s3',
botocore.handlers.validate_bucket_name)
with self.assertRaises(botocore.exceptions.ClientError) as ctx:
self.conn.create_bucket(Bucket='bucket+invalid')
self.assertEqual(
ctx.exception.response['ResponseMetadata']['HTTPStatusCode'], 400)
self.assertEqual(
ctx.exception.response['Error']['Code'], 'InvalidBucketName')
auth_error_conn = Connection(aws_secret_key='invalid')
status, headers, body = auth_error_conn.make_request('PUT', 'bucket')
self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch')
auth_error_conn = get_boto3_conn(aws_secret_key='invalid')
with self.assertRaises(botocore.exceptions.ClientError) as ctx:
auth_error_conn.create_bucket(Bucket='bucket')
self.assertEqual(
ctx.exception.response['ResponseMetadata']['HTTPStatusCode'], 403)
self.assertEqual(ctx.exception.response['Error']['Code'],
'SignatureDoesNotMatch')
self.conn.make_request('PUT', 'bucket')
status, headers, body = self.conn.make_request('PUT', 'bucket')
self.assertEqual(status, 409)
self.assertEqual(get_error_code(body), 'BucketAlreadyOwnedByYou')
self.conn.create_bucket(Bucket='bucket')
with self.assertRaises(botocore.exceptions.ClientError) as ctx:
self.conn.create_bucket(Bucket='bucket')
self.assertEqual(
ctx.exception.response['ResponseMetadata']['HTTPStatusCode'], 409)
self.assertEqual(
ctx.exception.response['Error']['Code'], 'BucketAlreadyOwnedByYou')
def test_put_bucket_error_key2(self):
if config_true_value(tf.cluster_info['s3api'].get('s3_acl')):
@ -163,15 +152,18 @@ class TestS3ApiBucket(S3ApiBase):
'Cannot test for BucketAlreadyExists with second user; '
'need s3_access_key2 and s3_secret_key2 configured')
self.conn.make_request('PUT', 'bucket')
self.conn.create_bucket(Bucket='bucket')
# Other users of the same account get the same 409 error
conn2 = Connection(tf.config['s3_access_key2'],
tf.config['s3_secret_key2'],
tf.config['s3_access_key2'])
status, headers, body = conn2.make_request('PUT', 'bucket')
self.assertEqual(status, 409)
self.assertEqual(get_error_code(body), 'BucketAlreadyExists')
conn2 = get_boto3_conn(tf.config['s3_access_key2'],
tf.config['s3_secret_key2'])
with self.assertRaises(botocore.exceptions.ClientError) as ctx:
conn2.create_bucket(Bucket='bucket')
self.assertEqual(
ctx.exception.response['ResponseMetadata']['HTTPStatusCode'],
409)
self.assertEqual(
ctx.exception.response['Error']['Code'], 'BucketAlreadyExists')
def test_put_bucket_error_key3(self):
if 's3_access_key3' not in tf.config or \
@ -179,41 +171,51 @@ class TestS3ApiBucket(S3ApiBase):
raise tf.SkipTest('Cannot test for AccessDenied; need '
's3_access_key3 and s3_secret_key3 configured')
self.conn.make_request('PUT', 'bucket')
self.conn.create_bucket(Bucket='bucket')
# If the user can't create buckets, they shouldn't even know
# whether the bucket exists.
conn3 = Connection(tf.config['s3_access_key3'],
tf.config['s3_secret_key3'],
tf.config['s3_access_key3'])
status, headers, body = conn3.make_request('PUT', 'bucket')
self.assertEqual(status, 403)
self.assertEqual(get_error_code(body), 'AccessDenied')
conn3 = get_boto3_conn(tf.config['s3_access_key3'],
tf.config['s3_secret_key3'])
with self.assertRaises(botocore.exceptions.ClientError) as ctx:
conn3.create_bucket(Bucket='bucket')
self.assertEqual(
ctx.exception.response['ResponseMetadata']['HTTPStatusCode'], 403)
self.assertEqual(
ctx.exception.response['Error']['Code'], 'AccessDenied')
def test_put_bucket_with_LocationConstraint(self):
bucket = 'bucket'
xml = self._gen_location_xml(self.conn.conn.auth_region_name)
status, headers, body = \
self.conn.make_request('PUT', bucket, body=xml)
self.assertEqual(status, 200)
resp = self.conn.create_bucket(
Bucket='bucket',
CreateBucketConfiguration={'LocationConstraint': self.region})
self.assertEqual(resp['ResponseMetadata']['HTTPStatusCode'], 200)
def test_get_bucket_error(self):
self.conn.make_request('PUT', 'bucket')
event_system = self.conn.meta.events
event_system.unregister(
'before-parameter-build.s3',
botocore.handlers.validate_bucket_name)
self.conn.create_bucket(Bucket='bucket')
status, headers, body = \
self.conn.make_request('GET', 'bucket+invalid')
self.assertEqual(get_error_code(body), 'InvalidBucketName')
with self.assertRaises(botocore.exceptions.ClientError) as ctx:
self.conn.list_objects(Bucket='bucket+invalid')
self.assertEqual(
ctx.exception.response['Error']['Code'], 'InvalidBucketName')
auth_error_conn = Connection(aws_secret_key='invalid')
status, headers, body = auth_error_conn.make_request('GET', 'bucket')
self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch')
auth_error_conn = get_boto3_conn(aws_secret_key='invalid')
with self.assertRaises(botocore.exceptions.ClientError) as ctx:
auth_error_conn.list_objects(Bucket='bucket')
self.assertEqual(
ctx.exception.response['Error']['Code'], 'SignatureDoesNotMatch')
status, headers, body = self.conn.make_request('GET', 'nothing')
self.assertEqual(get_error_code(body), 'NoSuchBucket')
with self.assertRaises(botocore.exceptions.ClientError) as ctx:
self.conn.list_objects(Bucket='nothing')
self.assertEqual(
ctx.exception.response['Error']['Code'], 'NoSuchBucket')
def _prepare_test_get_bucket(self, bucket, objects):
self.conn.make_request('PUT', bucket)
self.conn.create_bucket(Bucket=bucket)
for obj in objects:
self.conn.make_request('PUT', bucket, obj)
self.conn.put_object(Bucket=bucket, Key=obj, Body=b'')
def test_get_bucket_with_delimiter(self):
bucket = 'bucket'
@ -222,32 +224,16 @@ class TestS3ApiBucket(S3ApiBase):
self._prepare_test_get_bucket(bucket, put_objects)
delimiter = '/'
query = 'delimiter=%s' % delimiter
expect_objects = ('object', 'object2')
expect_prefixes = ('dir/', 'subdir/', 'subdir2/')
status, headers, body = \
self.conn.make_request('GET', bucket, query=query)
self.assertEqual(status, 200)
elem = fromstring(body, 'ListBucketResult')
self.assertEqual(elem.find('Delimiter').text, delimiter)
resp_objects = elem.findall('./Contents')
self.assertEqual(len(list(resp_objects)), len(expect_objects))
for i, o in enumerate(resp_objects):
self.assertEqual(o.find('Key').text, expect_objects[i])
self.assertIsNotNone(o.find('LastModified').text)
self.assertRegexpMatches(
o.find('LastModified').text,
r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
self.assertIsNotNone(o.find('ETag').text)
self.assertIsNotNone(o.find('Size').text)
self.assertEqual(o.find('StorageClass').text, 'STANDARD')
self.assertEqual(o.find('Owner/ID').text, self.conn.user_id)
self.assertEqual(o.find('Owner/DisplayName').text,
self.conn.user_id)
resp_prefixes = elem.findall('CommonPrefixes')
self.assertEqual(len(resp_prefixes), len(expect_prefixes))
for i, p in enumerate(resp_prefixes):
self.assertEqual(p.find('./Prefix').text, expect_prefixes[i])
resp = self.conn.list_objects(Bucket=bucket, Delimiter=delimiter)
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
self.assertEqual(resp['Delimiter'], delimiter)
self._validate_object_listing(resp['Contents'], expect_objects)
resp_prefixes = resp['CommonPrefixes']
self.assertEqual(
resp_prefixes,
[{'Prefix': p} for p in expect_prefixes])
def test_get_bucket_with_encoding_type(self):
bucket = 'bucket'
@ -255,12 +241,10 @@ class TestS3ApiBucket(S3ApiBase):
self._prepare_test_get_bucket(bucket, put_objects)
encoding_type = 'url'
query = 'encoding-type=%s' % encoding_type
status, headers, body = \
self.conn.make_request('GET', bucket, query=query)
self.assertEqual(status, 200)
elem = fromstring(body, 'ListBucketResult')
self.assertEqual(elem.find('EncodingType').text, encoding_type)
resp = self.conn.list_objects(
Bucket=bucket, EncodingType=encoding_type)
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
self.assertEqual(resp['EncodingType'], encoding_type)
def test_get_bucket_with_marker(self):
bucket = 'bucket'
@ -269,27 +253,11 @@ class TestS3ApiBucket(S3ApiBase):
self._prepare_test_get_bucket(bucket, put_objects)
marker = 'object'
query = 'marker=%s' % marker
expect_objects = ('object2', 'subdir/object', 'subdir2/object')
status, headers, body = \
self.conn.make_request('GET', bucket, query=query)
self.assertEqual(status, 200)
elem = fromstring(body, 'ListBucketResult')
self.assertEqual(elem.find('Marker').text, marker)
resp_objects = elem.findall('./Contents')
self.assertEqual(len(list(resp_objects)), len(expect_objects))
for i, o in enumerate(resp_objects):
self.assertEqual(o.find('Key').text, expect_objects[i])
self.assertIsNotNone(o.find('LastModified').text)
self.assertRegexpMatches(
o.find('LastModified').text,
r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
self.assertIsNotNone(o.find('ETag').text)
self.assertIsNotNone(o.find('Size').text)
self.assertEqual(o.find('StorageClass').text, 'STANDARD')
self.assertEqual(o.find('Owner/ID').text, self.conn.user_id)
self.assertEqual(o.find('Owner/DisplayName').text,
self.conn.user_id)
resp = self.conn.list_objects(Bucket=bucket, Marker=marker)
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
self.assertEqual(resp['Marker'], marker)
self._validate_object_listing(resp['Contents'], expect_objects)
def test_get_bucket_with_max_keys(self):
bucket = 'bucket'
@ -297,28 +265,12 @@ class TestS3ApiBucket(S3ApiBase):
'dir/subdir/object')
self._prepare_test_get_bucket(bucket, put_objects)
max_keys = '2'
query = 'max-keys=%s' % max_keys
max_keys = 2
expect_objects = ('dir/subdir/object', 'object')
status, headers, body = \
self.conn.make_request('GET', bucket, query=query)
self.assertEqual(status, 200)
elem = fromstring(body, 'ListBucketResult')
self.assertEqual(elem.find('MaxKeys').text, max_keys)
resp_objects = elem.findall('./Contents')
self.assertEqual(len(list(resp_objects)), len(expect_objects))
for i, o in enumerate(resp_objects):
self.assertEqual(o.find('Key').text, expect_objects[i])
self.assertIsNotNone(o.find('LastModified').text)
self.assertRegexpMatches(
o.find('LastModified').text,
r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
self.assertIsNotNone(o.find('ETag').text)
self.assertIsNotNone(o.find('Size').text)
self.assertEqual(o.find('StorageClass').text, 'STANDARD')
self.assertEqual(o.find('Owner/ID').text, self.conn.user_id)
self.assertEqual(o.find('Owner/DisplayName').text,
self.conn.user_id)
resp = self.conn.list_objects(Bucket=bucket, MaxKeys=max_keys)
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
self.assertEqual(resp['MaxKeys'], max_keys)
self._validate_object_listing(resp['Contents'], expect_objects)
def test_get_bucket_with_prefix(self):
bucket = 'bucket'
@ -327,27 +279,11 @@ class TestS3ApiBucket(S3ApiBase):
self._prepare_test_get_bucket(bucket, req_objects)
prefix = 'object'
query = 'prefix=%s' % prefix
expect_objects = ('object', 'object2')
status, headers, body = \
self.conn.make_request('GET', bucket, query=query)
self.assertEqual(status, 200)
elem = fromstring(body, 'ListBucketResult')
self.assertEqual(elem.find('Prefix').text, prefix)
resp_objects = elem.findall('./Contents')
self.assertEqual(len(list(resp_objects)), len(expect_objects))
for i, o in enumerate(resp_objects):
self.assertEqual(o.find('Key').text, expect_objects[i])
self.assertIsNotNone(o.find('LastModified').text)
self.assertRegexpMatches(
o.find('LastModified').text,
r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
self.assertIsNotNone(o.find('ETag').text)
self.assertIsNotNone(o.find('Size').text)
self.assertEqual(o.find('StorageClass').text, 'STANDARD')
self.assertEqual(o.find('Owner/ID').text, self.conn.user_id)
self.assertEqual(o.find('Owner/DisplayName').text,
self.conn.user_id)
resp = self.conn.list_objects(Bucket=bucket, Prefix=prefix)
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
self.assertEqual(resp['Prefix'], prefix)
self._validate_object_listing(resp['Contents'], expect_objects)
def test_get_bucket_v2_with_start_after(self):
bucket = 'bucket'
@ -356,26 +292,13 @@ class TestS3ApiBucket(S3ApiBase):
self._prepare_test_get_bucket(bucket, put_objects)
marker = 'object'
query = 'list-type=2&start-after=%s' % marker
expect_objects = ('object2', 'subdir/object', 'subdir2/object')
status, headers, body = \
self.conn.make_request('GET', bucket, query=query)
self.assertEqual(status, 200)
elem = fromstring(body, 'ListBucketResult')
self.assertEqual(elem.find('StartAfter').text, marker)
resp_objects = elem.findall('./Contents')
self.assertEqual(len(list(resp_objects)), len(expect_objects))
for i, o in enumerate(resp_objects):
self.assertEqual(o.find('Key').text, expect_objects[i])
self.assertTrue(o.find('LastModified').text is not None)
self.assertRegexpMatches(
o.find('LastModified').text,
r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
self.assertTrue(o.find('ETag').text is not None)
self.assertTrue(o.find('Size').text is not None)
self.assertEqual(o.find('StorageClass').text, 'STANDARD')
self.assertIsNone(o.find('Owner/ID'))
self.assertIsNone(o.find('Owner/DisplayName'))
resp = self.conn.list_objects_v2(Bucket=bucket, StartAfter=marker)
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
self.assertEqual(resp['StartAfter'], marker)
self.assertEqual(resp['KeyCount'], 3)
self._validate_object_listing(resp['Contents'], expect_objects,
expect_owner=False)
def test_get_bucket_v2_with_fetch_owner(self):
bucket = 'bucket'
@ -383,28 +306,12 @@ class TestS3ApiBucket(S3ApiBase):
'dir/subdir/object')
self._prepare_test_get_bucket(bucket, put_objects)
query = 'list-type=2&fetch-owner=true'
expect_objects = ('dir/subdir/object', 'object', 'object2',
'subdir/object', 'subdir2/object')
status, headers, body = \
self.conn.make_request('GET', bucket, query=query)
self.assertEqual(status, 200)
elem = fromstring(body, 'ListBucketResult')
self.assertEqual(elem.find('KeyCount').text, '5')
resp_objects = elem.findall('./Contents')
self.assertEqual(len(list(resp_objects)), len(expect_objects))
for i, o in enumerate(resp_objects):
self.assertEqual(o.find('Key').text, expect_objects[i])
self.assertTrue(o.find('LastModified').text is not None)
self.assertRegexpMatches(
o.find('LastModified').text,
r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
self.assertTrue(o.find('ETag').text is not None)
self.assertTrue(o.find('Size').text is not None)
self.assertEqual(o.find('StorageClass').text, 'STANDARD')
self.assertTrue(o.find('Owner/ID').text, self.conn.user_id)
self.assertTrue(o.find('Owner/DisplayName').text,
self.conn.user_id)
resp = self.conn.list_objects_v2(Bucket=bucket, FetchOwner=True)
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
self.assertEqual(resp['KeyCount'], 5)
self._validate_object_listing(resp['Contents'], expect_objects)
def test_get_bucket_v2_with_continuation_token_and_delimiter(self):
bucket = 'bucket'
@ -421,86 +328,116 @@ class TestS3ApiBucket(S3ApiBase):
'subdirs': []}]
continuation_token = ''
query = 'list-type=2&max-keys=3&delimiter=/&continuation-token=%s'
for i in range(len(expected)):
status, headers, body = self.conn.make_request(
'GET', bucket, query=query % continuation_token)
self.assertEqual(status, 200)
elem = fromstring(body, 'ListBucketResult')
self.assertEqual(elem.find('MaxKeys').text, '3')
resp = self.conn.list_objects_v2(
Bucket=bucket,
MaxKeys=3,
Delimiter='/',
ContinuationToken=continuation_token)
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
self.assertEqual(resp['MaxKeys'], 3)
self.assertEqual(
elem.find('KeyCount').text,
str(len(expected[i]['objects']) + len(expected[i]['subdirs'])))
expect_truncated = 'true' if i < len(expected) - 1 else 'false'
self.assertEqual(elem.find('IsTruncated').text, expect_truncated)
next_cont_token_elem = elem.find('NextContinuationToken')
if expect_truncated == 'true':
self.assertIsNotNone(next_cont_token_elem)
continuation_token = next_cont_token_elem.text
resp_objects = elem.findall('./Contents')
resp['KeyCount'],
len(expected[i]['objects']) + len(expected[i]['subdirs']))
expect_truncated = i < len(expected) - 1
self.assertEqual(resp['IsTruncated'], expect_truncated)
if expect_truncated:
self.assertIsNotNone(resp['NextContinuationToken'])
continuation_token = resp['NextContinuationToken']
self._validate_object_listing(resp['Contents'],
expected[i]['objects'],
expect_owner=False)
resp_subdirs = resp.get('CommonPrefixes', [])
self.assertEqual(
len(list(resp_objects)), len(expected[i]['objects']))
for j, o in enumerate(resp_objects):
self.assertEqual(o.find('Key').text,
expected[i]['objects'][j].encode('utf-8'))
self.assertTrue(o.find('LastModified').text is not None)
self.assertRegexpMatches(
o.find('LastModified').text,
r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
self.assertTrue(o.find('ETag').text is not None)
self.assertTrue(o.find('Size').text is not None)
self.assertEqual(o.find('StorageClass').text, 'STANDARD')
self.assertIsNone(o.find('Owner/ID'))
self.assertIsNone(o.find('Owner/DisplayName'))
resp_subdirs = elem.findall('./CommonPrefixes')
self.assertEqual(
len(list(resp_subdirs)), len(expected[i]['subdirs']))
for j, o in enumerate(resp_subdirs):
self.assertEqual(
o.find('Prefix').text,
expected[i]['subdirs'][j].encode('utf-8'))
resp_subdirs,
[{'Prefix': p} for p in expected[i]['subdirs']])
def test_head_bucket_error(self):
self.conn.make_request('PUT', 'bucket')
event_system = self.conn.meta.events
event_system.unregister(
'before-parameter-build.s3',
botocore.handlers.validate_bucket_name)
status, headers, body = \
self.conn.make_request('HEAD', 'bucket+invalid')
self.assertEqual(status, 400)
self.assertEqual(body, '') # sanity
self.conn.create_bucket(Bucket='bucket')
auth_error_conn = Connection(aws_secret_key='invalid')
status, headers, body = \
auth_error_conn.make_request('HEAD', 'bucket')
self.assertEqual(status, 403)
self.assertEqual(body, '') # sanity
with self.assertRaises(botocore.exceptions.ClientError) as ctx:
self.conn.head_bucket(Bucket='bucket+invalid')
self.assertEqual(
ctx.exception.response['ResponseMetadata']['HTTPStatusCode'], 400)
self.assertEqual(ctx.exception.response['Error']['Code'], '400')
self.assertEqual(
ctx.exception.response[
'ResponseMetadata']['HTTPHeaders']['content-length'], '0')
status, headers, body = self.conn.make_request('HEAD', 'nothing')
self.assertEqual(status, 404)
self.assertEqual(body, '') # sanity
auth_error_conn = get_boto3_conn(aws_secret_key='invalid')
with self.assertRaises(botocore.exceptions.ClientError) as ctx:
auth_error_conn.head_bucket(Bucket='bucket')
self.assertEqual(
ctx.exception.response['ResponseMetadata']['HTTPStatusCode'], 403)
self.assertEqual(
ctx.exception.response['Error']['Code'], '403')
self.assertEqual(
ctx.exception.response[
'ResponseMetadata']['HTTPHeaders']['content-length'], '0')
with self.assertRaises(botocore.exceptions.ClientError) as ctx:
self.conn.head_bucket(Bucket='nothing')
self.assertEqual(
ctx.exception.response['ResponseMetadata']['HTTPStatusCode'], 404)
self.assertEqual(
ctx.exception.response['Error']['Code'], '404')
self.assertEqual(
ctx.exception.response[
'ResponseMetadata']['HTTPHeaders']['content-length'], '0')
def test_delete_bucket_error(self):
status, headers, body = \
self.conn.make_request('DELETE', 'bucket+invalid')
self.assertEqual(get_error_code(body), 'InvalidBucketName')
event_system = self.conn.meta.events
event_system.unregister(
'before-parameter-build.s3',
botocore.handlers.validate_bucket_name)
with self.assertRaises(botocore.exceptions.ClientError) as ctx:
self.conn.delete_bucket(Bucket='bucket+invalid')
self.assertEqual(
ctx.exception.response['Error']['Code'], 'InvalidBucketName')
auth_error_conn = Connection(aws_secret_key='invalid')
status, headers, body = \
auth_error_conn.make_request('DELETE', 'bucket')
self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch')
auth_error_conn = get_boto3_conn(aws_secret_key='invalid')
with self.assertRaises(botocore.exceptions.ClientError) as ctx:
auth_error_conn.delete_bucket(Bucket='bucket')
self.assertEqual(
ctx.exception.response['Error']['Code'], 'SignatureDoesNotMatch')
status, headers, body = self.conn.make_request('DELETE', 'bucket')
self.assertEqual(get_error_code(body), 'NoSuchBucket')
with self.assertRaises(botocore.exceptions.ClientError) as ctx:
self.conn.delete_bucket(Bucket='bucket')
self.assertEqual(
ctx.exception.response['Error']['Code'], 'NoSuchBucket')
def test_bucket_invalid_method_error(self):
def _mangle_req_method(request, **kwargs):
request.method = 'GETPUT'
def _mangle_req_controller_method(request, **kwargs):
request.method = '_delete_segments_bucket'
event_system = self.conn.meta.events
event_system.register(
'request-created.s3.CreateBucket',
_mangle_req_method)
# non existed verb in the controller
status, headers, body = \
self.conn.make_request('GETPUT', 'bucket')
self.assertEqual(get_error_code(body), 'MethodNotAllowed')
with self.assertRaises(botocore.exceptions.ClientError) as ctx:
self.conn.create_bucket(Bucket='bucket')
self.assertEqual(
ctx.exception.response['Error']['Code'], 'MethodNotAllowed')
event_system.unregister('request-created.s3.CreateBucket',
_mangle_req_method)
event_system.register('request-created.s3.CreateBucket',
_mangle_req_controller_method)
# the method exists in the controller but deny as MethodNotAllowed
status, headers, body = \
self.conn.make_request('_delete_segments_bucket', 'bucket')
self.assertEqual(get_error_code(body), 'MethodNotAllowed')
with self.assertRaises(botocore.exceptions.ClientError) as ctx:
self.conn.create_bucket(Bucket='bucket')
self.assertEqual(
ctx.exception.response['Error']['Code'], 'MethodNotAllowed')
class TestS3ApiBucketSigV4(TestS3ApiBucket):