From f05119c16fb33140a655b30e76734f481a479412 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Mon, 5 Aug 2019 17:33:23 -0700 Subject: [PATCH] py3: mostly port s3 func tests test_bucket.py is proving somewhat problematic. Change-Id: I5b337ef66a23fc989762801dd6a5ba1ed903f57b --- .zuul.yaml | 18 +++++++ test/functional/s3api/s3_test_client.py | 8 +-- test/functional/s3api/test_acl.py | 3 +- test/functional/s3api/test_multi_upload.py | 61 ++++++++++++---------- test/functional/s3api/test_object.py | 52 +++++++++--------- test/functional/s3api/test_presigned.py | 8 +-- test/functional/s3api/test_service.py | 4 +- test/functional/s3api/utils.py | 3 +- tox.ini | 12 +++++ 9 files changed, 105 insertions(+), 64 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index 69e8a85ef5..432820b09d 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -132,6 +132,18 @@ vars: tox_envlist: func-domain-remap-staticweb-py3 +- job: + name: swift-tox-func-s3api-py37 + parent: swift-tox-func-py37 + description: | + Run functional tests for swift under cPython version 3.7. + + Uses tox with the ``func-s3api`` environment. + It sets TMPDIR to an XFS mount point created via + tools/test-setup.sh. + vars: + tox_envlist: func-s3api-py3 + - job: name: swift-tox-func-centos-7 parent: swift-tox-func @@ -480,6 +492,11 @@ - ^(api-ref|doc|releasenotes)/.*$ - ^test/probe/.*$ - ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$ + - swift-tox-func-s3api-py37: + irrelevant-files: + - ^(api-ref|doc|releasenotes)/.*$ + - ^test/probe/.*$ + - ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$ # Other tests - swift-tox-func-s3api-ceph-s3tests-tempauth: @@ -555,6 +572,7 @@ - swift-tox-func-encryption - swift-tox-func-domain-remap-staticweb-py37 - swift-tox-func-ec-py37 + - swift-tox-func-s3api-py37 - swift-probetests-centos-7: irrelevant-files: - ^(api-ref|releasenotes)/.*$ diff --git a/test/functional/s3api/s3_test_client.py b/test/functional/s3api/s3_test_client.py index 4736e1d240..a6727047d0 100644 --- a/test/functional/s3api/s3_test_client.py +++ b/test/functional/s3api/s3_test_client.py @@ -81,7 +81,7 @@ class Connection(object): break for bucket in buckets: - if not isinstance(bucket.name, six.binary_type): + if six.PY2 and not isinstance(bucket.name, bytes): bucket.name = bucket.name.encode('utf-8') try: @@ -103,7 +103,7 @@ class Connection(object): exceptions.insert(0, 'Too many errors to continue:') raise Exception('\n========\n'.join(exceptions)) - def make_request(self, method, bucket='', obj='', headers=None, body='', + def make_request(self, method, bucket='', obj='', headers=None, body=b'', query=None): """ Wrapper method of S3Connection.make_request. @@ -123,7 +123,9 @@ class Connection(object): query_args=query, sender=None, override_num_retries=RETRY_COUNT, retry_handler=None) - return response.status, dict(response.getheaders()), response.read() + return (response.status, + {h.lower(): v for h, v in response.getheaders()}, + response.read()) def generate_url_and_headers(self, method, bucket='', obj='', expires_in=3600): diff --git a/test/functional/s3api/test_acl.py b/test/functional/s3api/test_acl.py index cc71410ead..1bde54767e 100644 --- a/test/functional/s3api/test_acl.py +++ b/test/functional/s3api/test_acl.py @@ -40,7 +40,8 @@ class TestS3Acl(S3ApiBase): raise tf.SkipTest( 'TestS3Acl requires s3_access_key3 and s3_secret_key3 ' 'configured for reduced-access user') - self.conn.make_request('PUT', self.bucket) + status, headers, body = self.conn.make_request('PUT', self.bucket) + self.assertEqual(status, 200, body) access_key3 = tf.config['s3_access_key3'] secret_key3 = tf.config['s3_secret_key3'] self.conn3 = Connection(access_key3, secret_key3, access_key3) diff --git a/test/functional/s3api/test_multi_upload.py b/test/functional/s3api/test_multi_upload.py index 33b9e322e3..7132bbad58 100644 --- a/test/functional/s3api/test_multi_upload.py +++ b/test/functional/s3api/test_multi_upload.py @@ -14,6 +14,7 @@ # limitations under the License. import base64 +import binascii import unittest2 import os import boto @@ -23,7 +24,7 @@ import boto from distutils.version import StrictVersion from hashlib import md5 -from itertools import izip, izip_longest +from six.moves import zip, zip_longest import test.functional as tf from swift.common.middleware.s3api.etree import fromstring, tostring, Element, \ @@ -67,7 +68,7 @@ class TestS3ApiMultiUpload(S3ApiBase): headers = [None] * len(keys) self.conn.make_request('PUT', bucket) query = 'uploads' - for key, key_headers in izip_longest(keys, headers): + for key, key_headers in zip_longest(keys, headers): for i in range(trials): status, resp_headers, body = \ self.conn.make_request('POST', bucket, key, @@ -76,7 +77,7 @@ class TestS3ApiMultiUpload(S3ApiBase): def _upload_part(self, bucket, key, upload_id, content=None, part_num=1): query = 'partNumber=%s&uploadId=%s' % (part_num, upload_id) - content = content if content else 'a' * self.min_segment_size + content = content if content else b'a' * self.min_segment_size status, headers, body = \ self.conn.make_request('PUT', bucket, key, body=content, query=query) @@ -108,8 +109,9 @@ class TestS3ApiMultiUpload(S3ApiBase): def test_object_multi_upload(self): bucket = 'bucket' keys = ['obj1', 'obj2', 'obj3'] + bad_content_md5 = base64.b64encode(b'a' * 16).strip().decode('ascii') headers = [None, - {'Content-MD5': base64.b64encode('a' * 16).strip()}, + {'Content-MD5': bad_content_md5}, {'Etag': 'nonsense'}] uploads = [] @@ -118,20 +120,20 @@ class TestS3ApiMultiUpload(S3ApiBase): # Initiate Multipart Upload for expected_key, (status, headers, body) in \ - izip(keys, results_generator): - self.assertEqual(status, 200) + zip(keys, results_generator): + self.assertEqual(status, 200, body) self.assertCommonResponseHeaders(headers) - self.assertTrue('content-type' in headers) + self.assertIn('content-type', headers) self.assertEqual(headers['content-type'], 'application/xml') - self.assertTrue('content-length' in headers) + self.assertIn('content-length', headers) self.assertEqual(headers['content-length'], str(len(body))) elem = fromstring(body, 'InitiateMultipartUploadResult') self.assertEqual(elem.find('Bucket').text, bucket) key = elem.find('Key').text self.assertEqual(expected_key, key) upload_id = elem.find('UploadId').text - self.assertTrue(upload_id is not None) - self.assertTrue((key, upload_id) not in uploads) + self.assertIsNotNone(upload_id) + self.assertNotIn((key, upload_id), uploads) uploads.append((key, upload_id)) self.assertEqual(len(uploads), len(keys)) # sanity @@ -157,7 +159,7 @@ class TestS3ApiMultiUpload(S3ApiBase): self.assertEqual(elem.find('IsTruncated').text, 'false') self.assertEqual(len(elem.findall('Upload')), 3) for (expected_key, expected_upload_id), u in \ - izip(uploads, elem.findall('Upload')): + zip(uploads, elem.findall('Upload')): key = u.find('Key').text upload_id = u.find('UploadId').text self.assertEqual(expected_key, key) @@ -174,7 +176,7 @@ class TestS3ApiMultiUpload(S3ApiBase): # Upload Part key, upload_id = uploads[0] - content = 'a' * self.min_segment_size + content = b'a' * self.min_segment_size etag = md5(content).hexdigest() status, headers, body = \ self._upload_part(bucket, key, upload_id, content) @@ -190,7 +192,7 @@ class TestS3ApiMultiUpload(S3ApiBase): key, upload_id = uploads[1] src_bucket = 'bucket2' src_obj = 'obj3' - src_content = 'b' * self.min_segment_size + src_content = b'b' * self.min_segment_size etag = md5(src_content).hexdigest() # prepare src obj @@ -266,7 +268,7 @@ class TestS3ApiMultiUpload(S3ApiBase): # etags will be used to generate xml for Complete Multipart Upload etags = [] for (expected_etag, expected_date), p in \ - izip(expected_parts_list, elem.findall('Part')): + zip(expected_parts_list, elem.findall('Part')): last_modified = p.find('LastModified').text self.assertTrue(last_modified is not None) # TODO: sanity check @@ -295,9 +297,9 @@ class TestS3ApiMultiUpload(S3ApiBase): else: self.assertIn('transfer-encoding', headers) self.assertEqual(headers['transfer-encoding'], 'chunked') - lines = body.split('\n') - self.assertTrue(lines[0].startswith(''), body) + lines = body.split(b'\n') + self.assertTrue(lines[0].startswith(b''), body) elem = fromstring(body, 'CompleteMultipartUploadResult') # TODO: use tf.config value self.assertEqual( @@ -305,9 +307,10 @@ class TestS3ApiMultiUpload(S3ApiBase): elem.find('Location').text) self.assertEqual(elem.find('Bucket').text, bucket) self.assertEqual(elem.find('Key').text, key) - concatted_etags = ''.join(etag.strip('"') for etag in etags) + concatted_etags = b''.join( + etag.strip('"').encode('ascii') for etag in etags) exp_etag = '"%s-%s"' % ( - md5(concatted_etags.decode('hex')).hexdigest(), len(etags)) + md5(binascii.unhexlify(concatted_etags)).hexdigest(), len(etags)) etag = elem.find('ETag').text self.assertEqual(etag, exp_etag) @@ -332,7 +335,7 @@ class TestS3ApiMultiUpload(S3ApiBase): last_modified = elem.find('LastModified').text self.assertIsNotNone(last_modified) - exp_content = 'a' * self.min_segment_size + exp_content = b'a' * self.min_segment_size etag = md5(exp_content).hexdigest() self.assertEqual(resp_etag, etag) @@ -723,7 +726,7 @@ class TestS3ApiMultiUpload(S3ApiBase): query = 'partNumber=%s&uploadId=%s' % (i, upload_id) status, headers, body = \ self.conn.make_request('PUT', bucket, key, query=query, - body='A' * body_size[i]) + body=b'A' * body_size[i]) etags.append(headers['etag']) xml = self._gen_comp_xml(etags) @@ -747,7 +750,7 @@ class TestS3ApiMultiUpload(S3ApiBase): query = 'partNumber=%s&uploadId=%s' % (i, upload_id) status, headers, body = \ self.conn.make_request('PUT', bucket, key, query=query, - body='A' * body_size[i]) + body=b'A' * body_size[i]) etags.append(headers['etag']) xml = self._gen_comp_xml(etags) @@ -770,9 +773,9 @@ class TestS3ApiMultiUpload(S3ApiBase): etags = [] for i in range(1, 4): query = 'partNumber=%s&uploadId=%s' % (2 * i - 1, upload_id) - status, headers, body = \ - self.conn.make_request('PUT', bucket, key, - body='A' * 1024 * 1024 * 5, query=query) + status, headers, body = self.conn.make_request( + 'PUT', bucket, key, body=b'A' * 1024 * 1024 * 5, + query=query) etags.append(headers['etag']) query = 'uploadId=%s' % upload_id xml = self._gen_comp_xml(etags[:-1], step=2) @@ -791,7 +794,7 @@ class TestS3ApiMultiUpload(S3ApiBase): # Initiate Multipart Upload for expected_key, (status, headers, body) in \ - izip(keys, results_generator): + zip(keys, results_generator): self.assertEqual(status, 200) self.assertCommonResponseHeaders(headers) self.assertTrue('content-type' in headers) @@ -813,7 +816,7 @@ class TestS3ApiMultiUpload(S3ApiBase): key, upload_id = uploads[0] src_bucket = 'bucket2' src_obj = 'obj4' - src_content = 'y' * (self.min_segment_size / 2) + 'z' * \ + src_content = b'y' * (self.min_segment_size // 2) + b'z' * \ self.min_segment_size src_range = 'bytes=0-%d' % (self.min_segment_size - 1) etag = md5(src_content[:self.min_segment_size]).hexdigest() @@ -901,7 +904,7 @@ class TestS3ApiMultiUploadSigV4(TestS3ApiMultiUpload): # Initiate Multipart Upload for expected_key, (status, _, body) in \ - izip(keys, results_generator): + zip(keys, results_generator): self.assertEqual(status, 200) # sanity elem = fromstring(body, 'InitiateMultipartUploadResult') key = elem.find('Key').text @@ -915,7 +918,7 @@ class TestS3ApiMultiUploadSigV4(TestS3ApiMultiUpload): # Upload Part key, upload_id = uploads[0] - content = 'a' * self.min_segment_size + content = b'a' * self.min_segment_size status, headers, body = \ self._upload_part(bucket, key, upload_id, content) self.assertEqual(status, 200) diff --git a/test/functional/s3api/test_object.py b/test/functional/s3api/test_object.py index 4463f7689a..3f0318a5bf 100644 --- a/test/functional/s3api/test_object.py +++ b/test/functional/s3api/test_object.py @@ -25,7 +25,8 @@ import email.parser from email.utils import formatdate, parsedate from time import mktime from hashlib import md5 -from urllib import quote +import six +from six.moves.urllib.parse import quote import test.functional as tf @@ -59,7 +60,7 @@ class TestS3ApiObject(S3ApiBase): def test_object(self): obj = 'object name with %-sign' - content = 'abc123' + content = b'abc123' etag = md5(content).hexdigest() # PUT Object @@ -219,19 +220,19 @@ class TestS3ApiObject(S3ApiBase): status, headers, body = \ auth_error_conn.make_request('HEAD', self.bucket, obj) self.assertEqual(status, 403) - self.assertEqual(body, '') # sanity + self.assertEqual(body, b'') # sanity self.assertEqual(headers['content-type'], 'application/xml') status, headers, body = \ self.conn.make_request('HEAD', self.bucket, 'invalid') self.assertEqual(status, 404) - self.assertEqual(body, '') # sanity + self.assertEqual(body, b'') # sanity self.assertEqual(headers['content-type'], 'application/xml') status, headers, body = \ self.conn.make_request('HEAD', 'invalid', obj) self.assertEqual(status, 404) - self.assertEqual(body, '') # sanity + self.assertEqual(body, b'') # sanity self.assertEqual(headers['content-type'], 'application/xml') def test_delete_object_error(self): @@ -265,7 +266,7 @@ class TestS3ApiObject(S3ApiBase): def test_put_object_content_md5(self): obj = 'object' - content = 'abcdefghij' + content = b'abcdefghij' etag = md5(content).hexdigest() headers = {'Content-MD5': calculate_md5(content)} status, headers, body = \ @@ -276,7 +277,7 @@ class TestS3ApiObject(S3ApiBase): def test_put_object_content_type(self): obj = 'object' - content = 'abcdefghij' + content = b'abcdefghij' etag = md5(content).hexdigest() headers = {'Content-Type': 'text/plain'} status, headers, body = \ @@ -290,7 +291,7 @@ class TestS3ApiObject(S3ApiBase): def test_put_object_conditional_requests(self): obj = 'object' - content = 'abcdefghij' + content = b'abcdefghij' headers = {'If-None-Match': '*'} status, headers, body = \ self.conn.make_request('PUT', self.bucket, obj, headers, content) @@ -318,7 +319,7 @@ class TestS3ApiObject(S3ApiBase): def test_put_object_expect(self): obj = 'object' - content = 'abcdefghij' + content = b'abcdefghij' etag = md5(content).hexdigest() headers = {'Expect': '100-continue'} status, headers, body = \ @@ -331,7 +332,7 @@ class TestS3ApiObject(S3ApiBase): if expected_headers is None: expected_headers = req_headers obj = 'object' - content = 'abcdefghij' + content = b'abcdefghij' etag = md5(content).hexdigest() status, headers, body = \ self.conn.make_request('PUT', self.bucket, obj, @@ -387,7 +388,7 @@ class TestS3ApiObject(S3ApiBase): def test_put_object_storage_class(self): obj = 'object' - content = 'abcdefghij' + content = b'abcdefghij' etag = md5(content).hexdigest() headers = {'X-Amz-Storage-Class': 'STANDARD'} status, headers, body = \ @@ -399,7 +400,7 @@ class TestS3ApiObject(S3ApiBase): def test_put_object_copy_source_params(self): obj = 'object' src_headers = {'X-Amz-Meta-Test': 'src'} - src_body = 'some content' + src_body = b'some content' dst_bucket = 'dst-bucket' dst_obj = 'dst_object' self.conn.make_request('PUT', self.bucket, obj, src_headers, src_body) @@ -433,7 +434,7 @@ class TestS3ApiObject(S3ApiBase): def test_put_object_copy_source(self): obj = 'object' - content = 'abcdefghij' + content = b'abcdefghij' etag = md5(content).hexdigest() self.conn.make_request('PUT', self.bucket, obj, body=content) @@ -648,7 +649,7 @@ class TestS3ApiObject(S3ApiBase): def test_get_object_range(self): obj = 'object' - content = 'abcdefghij' + content = b'abcdefghij' headers = {'x-amz-meta-test': 'swift'} self.conn.make_request( 'PUT', self.bucket, obj, headers=headers, body=content) @@ -662,7 +663,7 @@ class TestS3ApiObject(S3ApiBase): self.assertEqual(headers['content-length'], '5') self.assertTrue('x-amz-meta-test' in headers) self.assertEqual('swift', headers['x-amz-meta-test']) - self.assertEqual(body, 'bcdef') + self.assertEqual(body, b'bcdef') headers = {'Range': 'bytes=5-'} status, headers, body = \ @@ -673,7 +674,7 @@ class TestS3ApiObject(S3ApiBase): self.assertEqual(headers['content-length'], '5') self.assertTrue('x-amz-meta-test' in headers) self.assertEqual('swift', headers['x-amz-meta-test']) - self.assertEqual(body, 'fghij') + self.assertEqual(body, b'fghij') headers = {'Range': 'bytes=-5'} status, headers, body = \ @@ -684,7 +685,7 @@ class TestS3ApiObject(S3ApiBase): self.assertEqual(headers['content-length'], '5') self.assertTrue('x-amz-meta-test' in headers) self.assertEqual('swift', headers['x-amz-meta-test']) - self.assertEqual(body, 'fghij') + self.assertEqual(body, b'fghij') ranges = ['1-2', '4-5'] @@ -693,9 +694,9 @@ class TestS3ApiObject(S3ApiBase): self.conn.make_request('GET', self.bucket, obj, headers=headers) self.assertEqual(status, 206) self.assertCommonResponseHeaders(headers) - self.assertTrue('content-length' in headers) + self.assertIn('content-length', headers) - self.assertTrue('content-type' in headers) # sanity + self.assertIn('content-type', headers) # sanity content_type, boundary = headers['content-type'].split(';') self.assertEqual('multipart/byteranges', content_type) @@ -704,10 +705,13 @@ class TestS3ApiObject(S3ApiBase): # TODO: Using swift.common.utils.multipart_byteranges_to_document_iters # could be easy enough. - parser = email.parser.FeedParser() + if six.PY2: + parser = email.parser.FeedParser() + else: + parser = email.parser.BytesFeedParser() parser.feed( - "Content-Type: multipart/byterange; boundary=%s\r\n\r\n" % - boundary_str) + b"Content-Type: multipart/byterange; boundary=%s\r\n\r\n" % + boundary_str.encode('ascii')) parser.feed(body) message = parser.close() @@ -727,7 +731,7 @@ class TestS3ApiObject(S3ApiBase): self.assertEqual( expected_range, part.get('Content-Range')) # rest - payload = part.get_payload().strip() + payload = part.get_payload(decode=True).strip() self.assertEqual(content[start:end + 1], payload) def test_get_object_if_modified_since(self): @@ -783,7 +787,7 @@ class TestS3ApiObject(S3ApiBase): def test_head_object_range(self): obj = 'object' - content = 'abcdefghij' + content = b'abcdefghij' self.conn.make_request('PUT', self.bucket, obj, body=content) headers = {'Range': 'bytes=1-5'} diff --git a/test/functional/s3api/test_presigned.py b/test/functional/s3api/test_presigned.py index 426a56ffa5..b13fb76c63 100644 --- a/test/functional/s3api/test_presigned.py +++ b/test/functional/s3api/test_presigned.py @@ -190,7 +190,7 @@ class TestS3ApiPresignedUrls(S3ApiBase): # PUT empty object put_url, headers = self.conn.generate_url_and_headers( 'PUT', bucket, obj) - resp = requests.put(put_url, data='', headers=headers) + resp = requests.put(put_url, data=b'', headers=headers) self.assertEqual(resp.status_code, 200, 'Got %d %s' % (resp.status_code, resp.content)) # GET empty object @@ -199,10 +199,10 @@ class TestS3ApiPresignedUrls(S3ApiBase): resp = requests.get(get_url, headers=headers) self.assertEqual(resp.status_code, 200, 'Got %d %s' % (resp.status_code, resp.content)) - self.assertEqual(resp.content, '') + self.assertEqual(resp.content, b'') # PUT over object - resp = requests.put(put_url, data='foobar', headers=headers) + resp = requests.put(put_url, data=b'foobar', headers=headers) self.assertEqual(resp.status_code, 200, 'Got %d %s' % (resp.status_code, resp.content)) @@ -210,7 +210,7 @@ class TestS3ApiPresignedUrls(S3ApiBase): resp = requests.get(get_url, headers=headers) self.assertEqual(resp.status_code, 200, 'Got %d %s' % (resp.status_code, resp.content)) - self.assertEqual(resp.content, 'foobar') + self.assertEqual(resp.content, b'foobar') # DELETE Object delete_url, headers = self.conn.generate_url_and_headers( diff --git a/test/functional/s3api/test_service.py b/test/functional/s3api/test_service.py index 0508f880ab..1df7ca3de2 100644 --- a/test/functional/s3api/test_service.py +++ b/test/functional/s3api/test_service.py @@ -80,8 +80,8 @@ class TestS3ApiService(S3ApiBase): 'GET', headers={'Date': '', 'x-amz-date': ''}) self.assertEqual(status, 403) self.assertEqual(get_error_code(body), 'AccessDenied') - self.assertIn('AWS authentication requires a valid Date ' - 'or x-amz-date header', body) + self.assertIn(b'AWS authentication requires a valid Date ' + b'or x-amz-date header', body) class TestS3ApiServiceSigV4(TestS3ApiService): diff --git a/test/functional/s3api/utils.py b/test/functional/s3api/utils.py index e14be35e68..5a979fdcb8 100644 --- a/test/functional/s3api/utils.py +++ b/test/functional/s3api/utils.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from base64 import b64encode from hashlib import md5 from swift.common.middleware.s3api.etree import fromstring @@ -28,4 +29,4 @@ def get_error_msg(body): def calculate_md5(body): - return md5(body).digest().encode('base64').strip() + return b64encode(md5(body).digest()).strip().decode('ascii') diff --git a/tox.ini b/tox.ini index 201425e6a7..9caecf3cf2 100644 --- a/tox.ini +++ b/tox.ini @@ -48,6 +48,12 @@ commands = ./.functests {posargs} basepython = python3 commands = nosetests {posargs: \ + test/functional/s3api/test_acl.py \ + test/functional/s3api/test_multi_delete.py \ + test/functional/s3api/test_multi_upload.py \ + test/functional/s3api/test_object.py \ + test/functional/s3api/test_presigned.py \ + test/functional/s3api/test_service.py \ test/functional/test_access_control.py \ test/functional/test_domain_remap.py \ test/functional/test_object.py \ @@ -62,6 +68,12 @@ commands = {[testenv:func-py3]commands} setenv = SWIFT_TEST_IN_PROCESS=1 SWIFT_TEST_IN_PROCESS_CONF_LOADER=ec +[testenv:func-s3api-py3] +basepython = python3 +commands = {[testenv:func-py3]commands} +setenv = SWIFT_TEST_IN_PROCESS=1 + SWIFT_TEST_IN_PROCESS_CONF_LOADER=s3api + [testenv:func-encryption-py3] basepython = python3 commands = {[testenv:func-py3]commands}