openstack/swift
Code Issues Proposed changes

Merge "Handle non-ASCII characters in S3 v2 listings."

Browse Source
This commit is contained in:
Zuul 2018-11-16 13:34:07 +00:00 committed by Gerrit Code Review
commit 78da780722
4 changed files with 68 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
swift/common/middleware/s3api/controllers/bucket.py
View File

@ -67,7 +67,7 @@ class BucketController(Controller):
for seg in segments: for seg in segments:
try: try:
req.get_response(self.app, 'DELETE', container, req.get_response(self.app, 'DELETE', container,
seg['name']) seg['name'].encode('utf8'))
except NoSuchKey: except NoSuchKey:
pass pass
except InternalError: except InternalError:
@ -183,10 +183,10 @@ class BucketController(Controller):
if is_truncated: if is_truncated:
if 'name' in objects[-1]: if 'name' in objects[-1]:
SubElement(elem, 'NextContinuationToken').text = \ SubElement(elem, 'NextContinuationToken').text = \
b64encode(objects[-1]['name']) b64encode(objects[-1]['name'].encode('utf8'))
if 'subdir' in objects[-1]: if 'subdir' in objects[-1]:
SubElement(elem, 'NextContinuationToken').text = \ SubElement(elem, 'NextContinuationToken').text = \
b64encode(objects[-1]['subdir']) b64encode(objects[-1]['subdir'].encode('utf8'))
if 'continuation-token' in req.params: if 'continuation-token' in req.params:
SubElement(elem, 'ContinuationToken').text = \ SubElement(elem, 'ContinuationToken').text = \
req.params['continuation-token'] req.params['continuation-token']
@ -210,7 +210,7 @@ class BucketController(Controller):
if 'subdir' not in o: if 'subdir' not in o:
name = o['name'] name = o['name']
if encoding_type == 'url': if encoding_type == 'url':
name = quote(name) name = quote(name.encode('utf-8'))
if listing_type == 'object-versions': if listing_type == 'object-versions':
contents = SubElement(elem, 'Version') contents = SubElement(elem, 'Version')
@ -240,7 +240,7 @@ class BucketController(Controller):
common_prefixes = SubElement(elem, 'CommonPrefixes') common_prefixes = SubElement(elem, 'CommonPrefixes')
name = o['subdir'] name = o['subdir']
if encoding_type == 'url': if encoding_type == 'url':
name = quote(name) name = quote(name.encode('utf-8'))
SubElement(common_prefixes, 'Prefix').text = name SubElement(common_prefixes, 'Prefix').text = name
body = tostring(elem) body = tostring(elem)

5
test/functional/s3api/s3_test_client.py
View File

@ -17,6 +17,8 @@ import os
import test.functional as tf import test.functional as tf
from boto.s3.connection import S3Connection, OrdinaryCallingFormat, \ from boto.s3.connection import S3Connection, OrdinaryCallingFormat, \
BotoClientError, S3ResponseError BotoClientError, S3ResponseError
import six
RETRY_COUNT = 3 RETRY_COUNT = 3
@ -75,6 +77,9 @@ class Connection(object):
break break
for bucket in buckets: for bucket in buckets:
if not isinstance(bucket.name, six.binary_type):
bucket.name = bucket.name.encode('utf-8')
try: try:
for upload in bucket.list_multipart_uploads(): for upload in bucket.list_multipart_uploads():
upload.cancel_upload() upload.cancel_upload()

103
test/functional/s3api/test_bucket.py
View File

@ -406,63 +406,60 @@ class TestS3ApiBucket(S3ApiBase):
self.assertTrue(o.find('Owner/DisplayName').text, self.assertTrue(o.find('Owner/DisplayName').text,
self.conn.user_id) self.conn.user_id)
def test_get_bucket_v2_with_continuation_token(self): def test_get_bucket_v2_with_continuation_token_and_delimiter(self):
bucket = 'bucket' bucket = 'bucket'
put_objects = ('object', 'object2', 'subdir/object', 'subdir2/object', put_objects = ('object', u'object2-\u062a', 'subdir/object',
'dir/subdir/object') u'subdir2-\u062a/object', 'dir/subdir/object',
'x', 'y', 'z')
self._prepare_test_get_bucket(bucket, put_objects) self._prepare_test_get_bucket(bucket, put_objects)
query = 'list-type=2&max-keys=3' expected = [{'objects': ['object', u'object2-\u062a'],
expect_objects = ('dir/subdir/object', 'object', 'object2') 'subdirs': ['dir/']},
status, headers, body = \ {'objects': ['x'],
self.conn.make_request('GET', bucket, query=query) 'subdirs': ['subdir/', u'subdir2-\u062a/']},
self.assertEqual(status, 200) {'objects': ['y', 'z'],
elem = fromstring(body, 'ListBucketResult') 'subdirs': []}]
self.assertEqual(elem.find('MaxKeys').text, '3')
self.assertEqual(elem.find('KeyCount').text, '3')
self.assertEqual(elem.find('IsTruncated').text, 'true')
next_cont_token_elem = elem.find('NextContinuationToken')
self.assertIsNotNone(next_cont_token_elem)
resp_objects = elem.findall('./Contents')
self.assertEqual(len(list(resp_objects)), len(expect_objects))
for i, o in enumerate(resp_objects):
self.assertEqual(o.find('Key').text, expect_objects[i])
self.assertTrue(o.find('LastModified').text is not None)
self.assertRegexpMatches(
o.find('LastModified').text,
r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
self.assertTrue(o.find('ETag').text is not None)
self.assertTrue(o.find('Size').text is not None)
self.assertEqual(o.find('StorageClass').text, 'STANDARD')
self.assertIsNone(o.find('Owner/ID'))
self.assertIsNone(o.find('Owner/DisplayName'))
query = 'list-type=2&max-keys=3&continuation-token=%s' % \ continuation_token = ''
next_cont_token_elem.text query = 'list-type=2&max-keys=3&delimiter=/&continuation-token=%s'
expect_objects = ('subdir/object', 'subdir2/object')
status, headers, body = \ for i in range(len(expected)):
self.conn.make_request('GET', bucket, query=query) status, headers, body = self.conn.make_request(
self.assertEqual(status, 200) 'GET', bucket, query=query % continuation_token)
elem = fromstring(body, 'ListBucketResult') self.assertEqual(status, 200)
self.assertEqual(elem.find('MaxKeys').text, '3') elem = fromstring(body, 'ListBucketResult')
self.assertEqual(elem.find('KeyCount').text, '2') self.assertEqual(elem.find('MaxKeys').text, '3')
self.assertEqual(elem.find('IsTruncated').text, 'false') self.assertEqual(
self.assertIsNone(elem.find('NextContinuationToken')) elem.find('KeyCount').text,
cont_token_elem = elem.find('ContinuationToken') str(len(expected[i]['objects']) + len(expected[i]['subdirs'])))
self.assertEqual(cont_token_elem.text, next_cont_token_elem.text) expect_truncated = 'true' if i < len(expected) - 1 else 'false'
resp_objects = elem.findall('./Contents') self.assertEqual(elem.find('IsTruncated').text, expect_truncated)
self.assertEqual(len(list(resp_objects)), len(expect_objects)) next_cont_token_elem = elem.find('NextContinuationToken')
for i, o in enumerate(resp_objects): if expect_truncated == 'true':
self.assertEqual(o.find('Key').text, expect_objects[i]) self.assertIsNotNone(next_cont_token_elem)
self.assertTrue(o.find('LastModified').text is not None) continuation_token = next_cont_token_elem.text
self.assertRegexpMatches( resp_objects = elem.findall('./Contents')
o.find('LastModified').text, self.assertEqual(
r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$') len(list(resp_objects)), len(expected[i]['objects']))
self.assertTrue(o.find('ETag').text is not None) for j, o in enumerate(resp_objects):
self.assertTrue(o.find('Size').text is not None) self.assertEqual(o.find('Key').text,
self.assertEqual(o.find('StorageClass').text, 'STANDARD') expected[i]['objects'][j].encode('utf-8'))
self.assertIsNone(o.find('Owner/ID')) self.assertTrue(o.find('LastModified').text is not None)
self.assertIsNone(o.find('Owner/DisplayName')) self.assertRegexpMatches(
o.find('LastModified').text,
r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
self.assertTrue(o.find('ETag').text is not None)
self.assertTrue(o.find('Size').text is not None)
self.assertEqual(o.find('StorageClass').text, 'STANDARD')
self.assertIsNone(o.find('Owner/ID'))
self.assertIsNone(o.find('Owner/DisplayName'))
resp_subdirs = elem.findall('./CommonPrefixes')
self.assertEqual(
len(list(resp_subdirs)), len(expected[i]['subdirs']))
for j, o in enumerate(resp_subdirs):
self.assertEqual(
o.find('Prefix').text,
expected[i]['subdirs'][j].encode('utf-8'))
def test_head_bucket_error(self): def test_head_bucket_error(self):
self.conn.make_request('PUT', 'bucket') self.conn.make_request('PUT', 'bucket')

15
test/unit/common/middleware/s3api/test_bucket.py
View File

@ -39,13 +39,14 @@ class TestS3ApiBucket(S3ApiTestCase):
self.objects = (('lily', '2011-01-05T02:19:14.275290', '0', '3909'), self.objects = (('lily', '2011-01-05T02:19:14.275290', '0', '3909'),
('rose', '2011-01-05T02:19:14.275290', 0, 303), ('rose', '2011-01-05T02:19:14.275290', 0, 303),
('viola', '2011-01-05T02:19:14.275290', '0', 3909), ('viola', '2011-01-05T02:19:14.275290', '0', 3909),
(u'lily-\u062a', '2011-01-05T02:19:14.275290', 0, 390),
('mu', '2011-01-05T02:19:14.275290', ('mu', '2011-01-05T02:19:14.275290',
'md5-of-the-manifest; s3_etag=0', '3909'), 'md5-of-the-manifest; s3_etag=0', '3909'),
('with space', '2011-01-05T02:19:14.275290', 0, 390), ('with space', '2011-01-05T02:19:14.275290', 0, 390),
('with%20space', '2011-01-05T02:19:14.275290', 0, 390)) ('with%20space', '2011-01-05T02:19:14.275290', 0, 390))
objects = [ objects = [
{'name': str(item[0]), 'last_modified': str(item[1]), {'name': item[0], 'last_modified': str(item[1]),
'hash': str(item[2]), 'bytes': str(item[3])} 'hash': str(item[2]), 'bytes': str(item[3])}
for item in self.objects] for item in self.objects]
object_list = json.dumps(objects) object_list = json.dumps(objects)
@ -57,7 +58,8 @@ class TestS3ApiBucket(S3ApiTestCase):
swob.HTTPNoContent, {}, json.dumps([])) swob.HTTPNoContent, {}, json.dumps([]))
for name, _, _, _ in self.objects: for name, _, _, _ in self.objects:
self.swift.register( self.swift.register(
'DELETE', '/v1/AUTH_test/bucket+segments/' + name, 'DELETE',
'/v1/AUTH_test/bucket+segments/' + name.encode('utf-8'),
swob.HTTPNoContent, {}, json.dumps([])) swob.HTTPNoContent, {}, json.dumps([]))
self.swift.register( self.swift.register(
'GET', 'GET',
@ -164,9 +166,8 @@ class TestS3ApiBucket(S3ApiTestCase):
o.find('./LastModified').text) o.find('./LastModified').text)
self.assertEqual('"0"', o.find('./ETag').text) self.assertEqual('"0"', o.find('./ETag').text)
self.assertEqual(len(names), len(self.objects)) self.assertEqual(
for i in self.objects: names, [obj[0].encode('utf-8') for obj in self.objects])
self.assertIn(i[0], names)
def test_bucket_GET_url_encoded(self): def test_bucket_GET_url_encoded(self):
bucket_name = 'junk' bucket_name = 'junk'
@ -192,7 +193,7 @@ class TestS3ApiBucket(S3ApiTestCase):
self.assertEqual(len(names), len(self.objects)) self.assertEqual(len(names), len(self.objects))
for i in self.objects: for i in self.objects:
self.assertIn(quote(i[0]), names) self.assertIn(quote(i[0].encode('utf-8')), names)
def test_bucket_GET_subdir(self): def test_bucket_GET_subdir(self):
bucket_name = 'junk-subdir' bucket_name = 'junk-subdir'
@ -518,7 +519,7 @@ class TestS3ApiBucket(S3ApiTestCase):
versions = elem.findall('./Version') versions = elem.findall('./Version')
objects = list(self.objects) objects = list(self.objects)
self.assertEqual([v.find('./Key').text for v in versions], self.assertEqual([v.find('./Key').text for v in versions],
[v[0] for v in objects]) [v[0].encode('utf-8') for v in objects])
self.assertEqual([v.find('./IsLatest').text for v in versions], self.assertEqual([v.find('./IsLatest').text for v in versions],
['true' for v in objects]) ['true' for v in objects])
self.assertEqual([v.find('./VersionId').text for v in versions], self.assertEqual([v.find('./VersionId').text for v in versions],