py3: port request_helpers

Change-Id: I6be1a1c618e4b4fa03b34dad96f378aca01e8e08

swift/common/request_helpers.py

@@ -35,7 +35,7 @@ from swift.common.exceptions import ListingIterError, SegmentError
 from swift.common.http import is_success
 from swift.common.swob import HTTPBadRequest, \
     HTTPServiceUnavailable, Range, is_chunked, multi_range_iterator, \
-    HTTPPreconditionFailed
+    HTTPPreconditionFailed, wsgi_to_bytes
 from swift.common.utils import split_path, validate_device_partition, \
     close_if_possible, maybe_multipart_byteranges_to_document_iters, \
     multipart_byteranges_to_document_iters, parse_content_type, \
@@ -262,7 +262,7 @@ def remove_items(headers, condition):
     :returns: a dict, possibly empty, of headers that have been removed
     """
     removed = {}
-    keys = filter(condition, headers)
+    keys = [key for key in headers if condition(key)]
     removed.update((key, headers.pop(key)) for key in keys)
     return removed
 
@@ -651,7 +651,7 @@ def http_response_to_document_iters(response, read_chunk_size=4096):
         # extracted from the Content-Type header.
         params = dict(params_list)
         return multipart_byteranges_to_document_iters(
-            response, params['boundary'], read_chunk_size)
+            response, wsgi_to_bytes(params['boundary']), read_chunk_size)
 
 
 def update_etag_is_at_header(req, name):

test/unit/common/test_request_helpers.py

@@ -142,16 +142,13 @@ class TestRequestHelpers(unittest.TestCase):
         self.assertEqual(policy.policy_type, REPL_POLICY)
 
         req.headers['X-Backend-Storage-Policy-Index'] = 'foo'
-        try:
+        with self.assertRaises(HTTPException) as raised:
             device, part, account, container, obj, policy = \
                 get_name_and_placement(req, 5, 5, True)
-        except HTTPException as e:
+        e = raised.exception
         self.assertEqual(e.status_int, 503)
         self.assertEqual(str(e), '503 Service Unavailable')
-            self.assertEqual(e.body, "No policy with index foo")
+        self.assertEqual(e.body, b"No policy with index foo")
-        else:
-            self.fail('get_name_and_placement did not raise error '
-                      'for invalid storage policy index')
 
     @patch_policies(with_ec_default=True)
     def test_get_name_and_placement_object_replication(self):
@@ -195,7 +192,7 @@ class TestHTTPResponseToDocumentIters(unittest.TestCase):
         fr = FakeResponse(
             200,
             {'Content-Length': '10', 'Content-Type': 'application/lunch'},
-            'sandwiches')
+            b'sandwiches')
 
         doc_iters = http_response_to_document_iters(fr)
         first_byte, last_byte, length, headers, body = next(doc_iters)
@@ -205,7 +202,7 @@ class TestHTTPResponseToDocumentIters(unittest.TestCase):
         header_dict = HeaderKeyDict(headers)
         self.assertEqual(header_dict.get('Content-Length'), '10')
         self.assertEqual(header_dict.get('Content-Type'), 'application/lunch')
-        self.assertEqual(body.read(), 'sandwiches')
+        self.assertEqual(body.read(), b'sandwiches')
 
         self.assertRaises(StopIteration, next, doc_iters)
 
@@ -213,7 +210,7 @@ class TestHTTPResponseToDocumentIters(unittest.TestCase):
             200,
             {'Transfer-Encoding': 'chunked',
              'Content-Type': 'application/lunch'},
-            'sandwiches')
+            b'sandwiches')
 
         doc_iters = http_response_to_document_iters(fr)
         first_byte, last_byte, length, headers, body = next(doc_iters)
@@ -223,7 +220,7 @@ class TestHTTPResponseToDocumentIters(unittest.TestCase):
         header_dict = HeaderKeyDict(headers)
         self.assertEqual(header_dict.get('Transfer-Encoding'), 'chunked')
         self.assertEqual(header_dict.get('Content-Type'), 'application/lunch')
-        self.assertEqual(body.read(), 'sandwiches')
+        self.assertEqual(body.read(), b'sandwiches')
 
         self.assertRaises(StopIteration, next, doc_iters)
 
@@ -232,7 +229,7 @@ class TestHTTPResponseToDocumentIters(unittest.TestCase):
             206,
             {'Content-Length': '8', 'Content-Type': 'application/lunch',
              'Content-Range': 'bytes 1-8/10'},
-            'andwiche')
+            b'andwiche')
 
         doc_iters = http_response_to_document_iters(fr)
         first_byte, last_byte, length, headers, body = next(doc_iters)
@@ -242,7 +239,7 @@ class TestHTTPResponseToDocumentIters(unittest.TestCase):
         header_dict = HeaderKeyDict(headers)
         self.assertEqual(header_dict.get('Content-Length'), '8')
         self.assertEqual(header_dict.get('Content-Type'), 'application/lunch')
-        self.assertEqual(body.read(), 'andwiche')
+        self.assertEqual(body.read(), b'andwiche')
 
         self.assertRaises(StopIteration, next, doc_iters)
 
@@ -252,7 +249,7 @@ class TestHTTPResponseToDocumentIters(unittest.TestCase):
             {'Transfer-Encoding': 'chunked',
              'Content-Type': 'application/lunch',
              'Content-Range': 'bytes 1-8/10'},
-            'andwiche')
+            b'andwiche')
 
         doc_iters = http_response_to_document_iters(fr)
         first_byte, last_byte, length, headers, body = next(doc_iters)
@@ -261,7 +258,7 @@ class TestHTTPResponseToDocumentIters(unittest.TestCase):
         self.assertEqual(length, 10)
         header_dict = HeaderKeyDict(headers)
         self.assertEqual(header_dict.get('Content-Type'), 'application/lunch')
-        self.assertEqual(body.read(), 'andwiche')
+        self.assertEqual(body.read(), b'andwiche')
 
         self.assertRaises(StopIteration, next, doc_iters)
 
@@ -269,17 +266,17 @@ class TestHTTPResponseToDocumentIters(unittest.TestCase):
         fr = FakeResponse(
             206,
             {'Content-Type': 'multipart/byteranges; boundary=asdfasdfasdf'},
-            ("--asdfasdfasdf\r\n"
+            (b"--asdfasdfasdf\r\n"
-             "Content-Type: application/lunch\r\n"
+             b"Content-Type: application/lunch\r\n"
-             "Content-Range: bytes 0-3/10\r\n"
+             b"Content-Range: bytes 0-3/10\r\n"
-             "\r\n"
+             b"\r\n"
-             "sand\r\n"
+             b"sand\r\n"
-             "--asdfasdfasdf\r\n"
+             b"--asdfasdfasdf\r\n"
-             "Content-Type: application/lunch\r\n"
+             b"Content-Type: application/lunch\r\n"
-             "Content-Range: bytes 6-9/10\r\n"
+             b"Content-Range: bytes 6-9/10\r\n"
-             "\r\n"
+             b"\r\n"
-             "ches\r\n"
+             b"ches\r\n"
-             "--asdfasdfasdf--"))
+             b"--asdfasdfasdf--"))
 
         doc_iters = http_response_to_document_iters(fr)
 
@@ -289,7 +286,7 @@ class TestHTTPResponseToDocumentIters(unittest.TestCase):
         self.assertEqual(length, 10)
         header_dict = HeaderKeyDict(headers)
         self.assertEqual(header_dict.get('Content-Type'), 'application/lunch')
-        self.assertEqual(body.read(), 'sand')
+        self.assertEqual(body.read(), b'sand')
 
         first_byte, last_byte, length, headers, body = next(doc_iters)
         self.assertEqual(first_byte, 6)
@@ -297,7 +294,7 @@ class TestHTTPResponseToDocumentIters(unittest.TestCase):
         self.assertEqual(length, 10)
         header_dict = HeaderKeyDict(headers)
         self.assertEqual(header_dict.get('Content-Type'), 'application/lunch')
-        self.assertEqual(body.read(), 'ches')
+        self.assertEqual(body.read(), b'ches')
 
         self.assertRaises(StopIteration, next, doc_iters)
 
@@ -314,7 +311,7 @@ class TestHTTPResponseToDocumentIters(unittest.TestCase):
         with self.assertRaises(ValueError) as cm:
             update_etag_is_at_header(req, 'X-Object-Sysmeta-,-Bad')
         self.assertEqual('Header name must not contain commas',
-                         cm.exception.message)
+                         cm.exception.args[0])
 
     def test_resolve_etag_is_at_header(self):
         def do_test():

test/unit/common/test_utils.py

@@ -4329,25 +4329,25 @@ cluster_dfw1 = http://dfw1.host/v1/
     def test_get_redirect_data(self):
         ts_now = utils.Timestamp.now()
         headers = {'X-Backend-Redirect-Timestamp': ts_now.internal}
-        response = FakeResponse(200, headers, '')
+        response = FakeResponse(200, headers, b'')
         self.assertIsNone(utils.get_redirect_data(response))
 
         headers = {'Location': '/a/c/o',
                    'X-Backend-Redirect-Timestamp': ts_now.internal}
-        response = FakeResponse(200, headers, '')
+        response = FakeResponse(200, headers, b'')
         path, ts = utils.get_redirect_data(response)
         self.assertEqual('a/c', path)
         self.assertEqual(ts_now, ts)
 
         headers = {'Location': '/a/c',
                    'X-Backend-Redirect-Timestamp': ts_now.internal}
-        response = FakeResponse(200, headers, '')
+        response = FakeResponse(200, headers, b'')
         path, ts = utils.get_redirect_data(response)
         self.assertEqual('a/c', path)
         self.assertEqual(ts_now, ts)
 
         def do_test(headers):
-            response = FakeResponse(200, headers, '')
+            response = FakeResponse(200, headers, b'')
             with self.assertRaises(ValueError) as cm:
                 utils.get_redirect_data(response)
             return cm.exception
@@ -6470,7 +6470,7 @@ class FakeResponse(object):
     def __init__(self, status, headers, body):
         self.status = status
         self.headers = HeaderKeyDict(headers)
-        self.body = StringIO(body)
+        self.body = BytesIO(body)
 
     def getheader(self, header_name):
         return str(self.headers.get(header_name, ''))

tox.ini

@@ -52,6 +52,7 @@ commands =
       test/unit/common/test_linkat.py \
       test/unit/common/test_manager.py \
       test/unit/common/test_memcached.py \
+      test/unit/common/test_request_helpers.py \
       test/unit/common/test_splice.py \
       test/unit/common/test_storage_policy.py \
       test/unit/common/test_swob.py \