636b922f3b
This attempts to import openstack/swift3 package into swift upstream repository, namespace. This is almost simple porting except following items. 1. Rename swift3 namespace to swift.common.middleware.s3api 1.1 Rename also some conflicted class names (e.g. Request/Response) 2. Port unittests to test/unit/s3api dir to be able to run on the gate. 3. Port functests to test/functional/s3api and setup in-process testing 4. Port docs to doc dir, then address the namespace change. 5. Use get_logger() instead of global logger instance 6. Avoid global conf instance Ex. fix various minor issue on those steps (e.g. packages, dependencies, deprecated things) The details and patch references in the work on feature/s3api are listed at https://trello.com/b/ZloaZ23t/s3api (completed board) Note that, because this is just a porting, no new feature is developed since the last swift3 release, and in the future work, Swift upstream may continue to work on remaining items for further improvements and the best compatibility of Amazon S3. Please read the new docs for your deployment and keep track to know what would be changed in the future releases. Change-Id: Ib803ea89cfee9a53c429606149159dd136c036fd Co-Authored-By: Thiago da Silva <thiago@redhat.com> Co-Authored-By: Tim Burke <tim.burke@gmail.com>
249 lines
9.9 KiB
Python
249 lines
9.9 KiB
Python
# Copyright (c) 2015 OpenStack Foundation
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import unittest2
|
|
import os
|
|
import test.functional as tf
|
|
from swift.common.middleware.s3api.etree import fromstring, tostring, Element, \
|
|
SubElement
|
|
from swift.common.middleware.s3api.controllers.multi_delete import \
|
|
MAX_MULTI_DELETE_BODY_SIZE
|
|
|
|
from test.functional.s3api import S3ApiBase
|
|
from test.functional.s3api.s3_test_client import Connection
|
|
from test.functional.s3api.utils import get_error_code, calculate_md5
|
|
|
|
|
|
def setUpModule():
|
|
tf.setup_package()
|
|
|
|
|
|
def tearDownModule():
|
|
tf.teardown_package()
|
|
|
|
|
|
class TestS3ApiMultiDelete(S3ApiBase):
|
|
def setUp(self):
|
|
super(TestS3ApiMultiDelete, self).setUp()
|
|
|
|
def _prepare_test_delete_multi_objects(self, bucket, objects):
|
|
self.conn.make_request('PUT', bucket)
|
|
for obj in objects:
|
|
self.conn.make_request('PUT', bucket, obj)
|
|
|
|
def _gen_multi_delete_xml(self, objects, quiet=None):
|
|
elem = Element('Delete')
|
|
if quiet:
|
|
SubElement(elem, 'Quiet').text = quiet
|
|
for key in objects:
|
|
obj = SubElement(elem, 'Object')
|
|
SubElement(obj, 'Key').text = key
|
|
|
|
return tostring(elem, use_s3ns=False)
|
|
|
|
def _gen_invalid_multi_delete_xml(self, hasObjectTag=False):
|
|
elem = Element('Delete')
|
|
if hasObjectTag:
|
|
obj = SubElement(elem, 'Object')
|
|
SubElement(obj, 'Key').text = ''
|
|
|
|
return tostring(elem, use_s3ns=False)
|
|
|
|
def test_delete_multi_objects(self):
|
|
bucket = 'bucket'
|
|
put_objects = ['obj%s' % var for var in xrange(4)]
|
|
self._prepare_test_delete_multi_objects(bucket, put_objects)
|
|
query = 'delete'
|
|
|
|
# Delete an object via MultiDelete API
|
|
req_objects = ['obj0']
|
|
xml = self._gen_multi_delete_xml(req_objects)
|
|
content_md5 = calculate_md5(xml)
|
|
status, headers, body = \
|
|
self.conn.make_request('POST', bucket, body=xml,
|
|
headers={'Content-MD5': content_md5},
|
|
query=query)
|
|
self.assertEqual(status, 200)
|
|
self.assertCommonResponseHeaders(headers)
|
|
self.assertTrue(headers['content-type'] is not None)
|
|
self.assertEqual(headers['content-length'], str(len(body)))
|
|
elem = fromstring(body)
|
|
resp_objects = elem.findall('Deleted')
|
|
self.assertEqual(len(resp_objects), len(req_objects))
|
|
for o in resp_objects:
|
|
self.assertTrue(o.find('Key').text in req_objects)
|
|
|
|
# Delete 2 objects via MultiDelete API
|
|
req_objects = ['obj1', 'obj2']
|
|
xml = self._gen_multi_delete_xml(req_objects)
|
|
content_md5 = calculate_md5(xml)
|
|
status, headers, body = \
|
|
self.conn.make_request('POST', bucket, body=xml,
|
|
headers={'Content-MD5': content_md5},
|
|
query=query)
|
|
self.assertEqual(status, 200)
|
|
elem = fromstring(body, 'DeleteResult')
|
|
resp_objects = elem.findall('Deleted')
|
|
self.assertEqual(len(resp_objects), len(req_objects))
|
|
for o in resp_objects:
|
|
self.assertTrue(o.find('Key').text in req_objects)
|
|
|
|
# Delete 2 objects via MultiDelete API but one (obj4) doesn't exist.
|
|
req_objects = ['obj3', 'obj4']
|
|
xml = self._gen_multi_delete_xml(req_objects)
|
|
content_md5 = calculate_md5(xml)
|
|
status, headers, body = \
|
|
self.conn.make_request('POST', bucket, body=xml,
|
|
headers={'Content-MD5': content_md5},
|
|
query=query)
|
|
self.assertEqual(status, 200)
|
|
elem = fromstring(body, 'DeleteResult')
|
|
resp_objects = elem.findall('Deleted')
|
|
# S3 assumes a NoSuchKey object as deleted.
|
|
self.assertEqual(len(resp_objects), len(req_objects))
|
|
for o in resp_objects:
|
|
self.assertTrue(o.find('Key').text in req_objects)
|
|
|
|
# Delete 2 objects via MultiDelete API but no objects exist
|
|
req_objects = ['obj4', 'obj5']
|
|
xml = self._gen_multi_delete_xml(req_objects)
|
|
content_md5 = calculate_md5(xml)
|
|
status, headers, body = \
|
|
self.conn.make_request('POST', bucket, body=xml,
|
|
headers={'Content-MD5': content_md5},
|
|
query=query)
|
|
self.assertEqual(status, 200)
|
|
elem = fromstring(body, 'DeleteResult')
|
|
resp_objects = elem.findall('Deleted')
|
|
self.assertEqual(len(resp_objects), len(req_objects))
|
|
for o in resp_objects:
|
|
self.assertTrue(o.find('Key').text in req_objects)
|
|
|
|
def test_delete_multi_objects_error(self):
|
|
bucket = 'bucket'
|
|
put_objects = ['obj']
|
|
self._prepare_test_delete_multi_objects(bucket, put_objects)
|
|
xml = self._gen_multi_delete_xml(put_objects)
|
|
content_md5 = calculate_md5(xml)
|
|
query = 'delete'
|
|
|
|
auth_error_conn = Connection(aws_secret_key='invalid')
|
|
status, headers, body = \
|
|
auth_error_conn.make_request('POST', bucket, body=xml,
|
|
headers={
|
|
'Content-MD5': content_md5
|
|
},
|
|
query=query)
|
|
self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch')
|
|
|
|
status, headers, body = \
|
|
self.conn.make_request('POST', 'nothing', body=xml,
|
|
headers={'Content-MD5': content_md5},
|
|
query=query)
|
|
self.assertEqual(get_error_code(body), 'NoSuchBucket')
|
|
|
|
# without Object tag
|
|
xml = self._gen_invalid_multi_delete_xml()
|
|
content_md5 = calculate_md5(xml)
|
|
status, headers, body = \
|
|
self.conn.make_request('POST', bucket, body=xml,
|
|
headers={'Content-MD5': content_md5},
|
|
query=query)
|
|
self.assertEqual(get_error_code(body), 'MalformedXML')
|
|
|
|
# without value of Key tag
|
|
xml = self._gen_invalid_multi_delete_xml(hasObjectTag=True)
|
|
content_md5 = calculate_md5(xml)
|
|
status, headers, body = \
|
|
self.conn.make_request('POST', bucket, body=xml,
|
|
headers={'Content-MD5': content_md5},
|
|
query=query)
|
|
self.assertEqual(get_error_code(body), 'UserKeyMustBeSpecified')
|
|
|
|
# specified number of objects are over max_multi_delete_objects
|
|
# (Default 1000), but xml size is smaller than 61365 bytes.
|
|
req_objects = ['obj%s' for var in xrange(1001)]
|
|
xml = self._gen_multi_delete_xml(req_objects)
|
|
self.assertTrue(len(xml.encode('utf-8')) <= MAX_MULTI_DELETE_BODY_SIZE)
|
|
content_md5 = calculate_md5(xml)
|
|
status, headers, body = \
|
|
self.conn.make_request('POST', bucket, body=xml,
|
|
headers={'Content-MD5': content_md5},
|
|
query=query)
|
|
self.assertEqual(get_error_code(body), 'MalformedXML')
|
|
|
|
# specified xml size is over 61365 bytes, but number of objects are
|
|
# smaller than max_multi_delete_objects.
|
|
obj = 'a' * 1024
|
|
req_objects = [obj + str(var) for var in xrange(999)]
|
|
xml = self._gen_multi_delete_xml(req_objects)
|
|
self.assertTrue(len(xml.encode('utf-8')) > MAX_MULTI_DELETE_BODY_SIZE)
|
|
content_md5 = calculate_md5(xml)
|
|
status, headers, body = \
|
|
self.conn.make_request('POST', bucket, body=xml,
|
|
headers={'Content-MD5': content_md5},
|
|
query=query)
|
|
self.assertEqual(get_error_code(body), 'MalformedXML')
|
|
|
|
def test_delete_multi_objects_with_quiet(self):
|
|
bucket = 'bucket'
|
|
put_objects = ['obj']
|
|
query = 'delete'
|
|
|
|
# with Quiet true
|
|
quiet = 'true'
|
|
self._prepare_test_delete_multi_objects(bucket, put_objects)
|
|
xml = self._gen_multi_delete_xml(put_objects, quiet)
|
|
content_md5 = calculate_md5(xml)
|
|
status, headers, body = \
|
|
self.conn.make_request('POST', bucket, body=xml,
|
|
headers={'Content-MD5': content_md5},
|
|
query=query)
|
|
self.assertEqual(status, 200)
|
|
elem = fromstring(body, 'DeleteResult')
|
|
resp_objects = elem.findall('Deleted')
|
|
self.assertEqual(len(resp_objects), 0)
|
|
|
|
# with Quiet false
|
|
quiet = 'false'
|
|
self._prepare_test_delete_multi_objects(bucket, put_objects)
|
|
xml = self._gen_multi_delete_xml(put_objects, quiet)
|
|
content_md5 = calculate_md5(xml)
|
|
status, headers, body = \
|
|
self.conn.make_request('POST', bucket, body=xml,
|
|
headers={'Content-MD5': content_md5},
|
|
query=query)
|
|
self.assertEqual(status, 200)
|
|
elem = fromstring(body, 'DeleteResult')
|
|
resp_objects = elem.findall('Deleted')
|
|
self.assertEqual(len(resp_objects), 1)
|
|
|
|
|
|
class TestS3ApiMultiDeleteSigV4(TestS3ApiMultiDelete):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
os.environ['S3_USE_SIGV4'] = "True"
|
|
|
|
@classmethod
|
|
def tearDownClass(cls):
|
|
del os.environ['S3_USE_SIGV4']
|
|
|
|
def setUp(self):
|
|
super(TestS3ApiMultiDeleteSigV4, self).setUp()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest2.main()
|