Fix setup of manifest responses in SLO tests

The swift_bytes param is removed from the content-type
in the proxy object controller, so the SLO unit tests should
not be registering GET responses with FakeSwift that have
swift_bytes appended to the content-type.

Nor should submanifest segment dicts have swift_bytes appended to
their content-type values.

Also adds a test for the object controller and container server
handling of SLO swift_bytes.

Change-Id: Icf9bd87eee25002c8d9728b16e60c8347060f320

test/unit/common/middleware/test_slo.py

@@ -349,7 +349,7 @@ class TestSloPutManifest(SloTestCase):
             'GET', '/v1/AUTH_test/checktest/slob',
             swob.HTTPOk,
             {'X-Static-Large-Object': 'true', 'Etag': 'slob-etag',
-             'Content-Type': 'cat/picture;swift_bytes=12345',
+             'Content-Type': 'cat/picture',
              'Content-Length': len(_manifest_json)},
             _manifest_json)
 
@@ -1106,7 +1106,7 @@ class TestSloGetRawManifest(SloTestCase):
               'last_modified': '1970-01-01T00:00:00.000000'},
              {'name': '/gettest/d_10',
               'hash': md5hex(md5hex("e" * 5) + md5hex("f" * 5)), 'bytes': '10',
-              'content_type': 'application/json;swift_bytes=10',
+              'content_type': 'application/json',
               'sub_slo': True,
               'last_modified': '1970-01-01T00:00:00.000000'}])
         self.bc_etag = md5hex(_bc_manifest_json)
@@ -1262,7 +1262,7 @@ class TestSloGetManifest(SloTestCase):
               'content_type': 'text/plain'}])
         self.app.register(
             'GET', '/v1/AUTH_test/gettest/manifest-bc',
-            swob.HTTPOk, {'Content-Type': 'application/json;swift_bytes=25',
+            swob.HTTPOk, {'Content-Type': 'application/json',
                           'X-Static-Large-Object': 'true',
                           'X-Object-Meta-Plant': 'Ficus',
                           'Etag': md5hex(_bc_manifest_json)},
@@ -1272,9 +1272,9 @@ class TestSloGetManifest(SloTestCase):
             [{'name': '/gettest/a_5', 'hash': md5hex("a" * 5),
               'content_type': 'text/plain', 'bytes': '5'},
              {'name': '/gettest/manifest-bc', 'sub_slo': True,
-              'content_type': 'application/json;swift_bytes=25',
+              'content_type': 'application/json',
               'hash': md5hex(md5hex("b" * 10) + md5hex("c" * 15)),
-              'bytes': len(_bc_manifest_json)},
+              'bytes': 25},
              {'name': '/gettest/d_20', 'hash': md5hex("d" * 20),
               'content_type': 'text/plain', 'bytes': '20'}])
         self.app.register(
@@ -1284,6 +1284,34 @@ class TestSloGetManifest(SloTestCase):
                           'Etag': md5(_abcd_manifest_json).hexdigest()},
             _abcd_manifest_json)
 
+        # A submanifest segment is created using the response headers from a
+        # HEAD on the submanifest. That HEAD is passed through SLO which will
+        # modify the response content-length to be equal to the size of the
+        # submanifest's large object. The swift_bytes value appended to the
+        # submanifest's content-type will have been removed. So the sub-slo
+        # segment dict that is written to the parent manifest should have the
+        # correct bytes and content-type values. However, if somehow the
+        # submanifest HEAD response wasn't modified by SLO (maybe
+        # historically?) and we ended up with the parent manifest sub-slo entry
+        # having swift_bytes appended to it's content-type and the actual
+        # submanifest size in its bytes field, then SLO can cope, so we create
+        # a deviant manifest to verify that SLO can deal with it.
+        _abcd_manifest_json_alt = json.dumps(
+            [{'name': '/gettest/a_5', 'hash': md5hex("a" * 5),
+              'content_type': 'text/plain', 'bytes': '5'},
+             {'name': '/gettest/manifest-bc', 'sub_slo': True,
+              'content_type': 'application/json; swift_bytes=25',
+              'hash': md5hex(md5hex("b" * 10) + md5hex("c" * 15)),
+              'bytes': len(_bc_manifest_json)},
+             {'name': '/gettest/d_20', 'hash': md5hex("d" * 20),
+              'content_type': 'text/plain', 'bytes': '20'}])
+        self.app.register(
+            'GET', '/v1/AUTH_test/gettest/manifest-abcd-alt',
+            swob.HTTPOk, {'Content-Type': 'application/json',
+                          'X-Static-Large-Object': 'true',
+                          'Etag': md5(_abcd_manifest_json_alt).hexdigest()},
+            _abcd_manifest_json_alt)
+
         _abcdefghijkl_manifest_json = json.dumps(
             [{'name': '/gettest/a_5', 'hash': md5hex("a" * 5),
               'content_type': 'text/plain', 'bytes': '5'},
@@ -1337,7 +1365,7 @@ class TestSloGetManifest(SloTestCase):
         self.bc_ranges_etag = md5hex(_bc_ranges_manifest_json)
         self.app.register(
             'GET', '/v1/AUTH_test/gettest/manifest-bc-ranges',
-            swob.HTTPOk, {'Content-Type': 'application/json;swift_bytes=16',
+            swob.HTTPOk, {'Content-Type': 'application/json',
                           'X-Static-Large-Object': 'true',
                           'X-Object-Meta-Plant': 'Ficus',
                           'Etag': self.bc_ranges_etag},
@@ -1351,12 +1379,12 @@ class TestSloGetManifest(SloTestCase):
               'content_type': 'text/plain', 'bytes': '5',
               'range': '1-4'},
              {'name': '/gettest/manifest-bc-ranges', 'sub_slo': True,
-              'content_type': 'application/json;swift_bytes=16',
+              'content_type': 'application/json',
               'hash': self.bc_ranges_etag,
-              'bytes': len(_bc_ranges_manifest_json),
+              'bytes': 16,
               'range': '8-15'},
              {'name': '/gettest/manifest-bc-ranges', 'sub_slo': True,
-              'content_type': 'application/json;swift_bytes=16',
+              'content_type': 'application/json',
               'hash': self.bc_ranges_etag,
               'bytes': len(_bc_ranges_manifest_json),
               'range': '0-7'},
@@ -1655,6 +1683,22 @@ class TestSloGetManifest(SloTestCase):
         self.assertEqual(
             body, 'aaaaabbbbbbbbbbcccccccccccccccdddddddddddddddddddd')
 
+    def test_get_manifest_with_submanifest_bytes_in_content_type(self):
+        # verify correct content-length when the sub-slo segment in the
+        # manifest has its actual object content-length appended as swift_bytes
+        # to the content-type, and the submanifest length in the bytes field.
+        req = Request.blank(
+            '/v1/AUTH_test/gettest/manifest-abcd-alt',
+            environ={'REQUEST_METHOD': 'GET'})
+        status, headers, body = self.call_slo(req)
+        headers = HeaderKeyDict(headers)
+
+        self.assertEqual(status, '200 OK')
+        self.assertEqual(headers['Content-Length'], '50')
+        self.assertEqual(headers['Etag'], '"%s"' % self.manifest_abcd_etag)
+        self.assertEqual(
+            body, 'aaaaabbbbbbbbbbcccccccccccccccdddddddddddddddddddd')
+
     def test_range_get_manifest(self):
         req = Request.blank(
             '/v1/AUTH_test/gettest/manifest-abcd',
@@ -2274,8 +2318,7 @@ class TestSloGetManifest(SloTestCase):
                  'hash': 'man%d' % (i + 1),
                  'sub_slo': True,
                  'bytes': len(manifest_json),
-                 'content_type':
+                 'content_type': 'application/json'}]
-                 'application/json;swift_bytes=%d' % ((21 - i) * 6)}]
 
             manifest_json = json.dumps(manifest_data)
             self.app.register(
@@ -2330,9 +2373,8 @@ class TestSloGetManifest(SloTestCase):
                 {'name': '/gettest/man%d' % (i + 1),
                  'hash': 'man%d' % (i + 1),
                  'sub_slo': True,
-                 'bytes': len(manifest_json),
+                 'bytes': (10 - i) * 6,
-                 'content_type':
+                 'content_type': 'application/json'},
-                 'application/json;swift_bytes=%d' % ((10 - i) * 6)},
                 {'name': '/gettest/obj%d' % i,
                  'hash': md5hex('body%02d' % i),
                  'bytes': '6',
@@ -2387,9 +2429,8 @@ class TestSloGetManifest(SloTestCase):
                 {'name': '/gettest/man%d' % (i + 1),
                  'hash': 'man%d' % (i + 1),
                  'sub_slo': True,
-                 'bytes': len(manifest_json),
+                 'bytes': (12 - i) * 6,
-                 'content_type':
+                 'content_type': 'application/json'},
-                 'application/json;swift_bytes=%d' % ((12 - i) * 6)},
                 {'name': '/gettest/obj%d' % i,
                  'hash': md5hex('body%02d' % i),
                  'bytes': '6',
@@ -2479,7 +2520,7 @@ class TestSloGetManifest(SloTestCase):
             swob.HTTPOk, {'Content-Type': 'application/json',
                           'X-Static-Large-Object': 'true'},
             json.dumps([{'name': '/gettest/manifest-a', 'sub_slo': True,
-                         'content_type': 'application/json;swift_bytes=5',
+                         'content_type': 'application/json',
                          'hash': 'manifest-a',
                          'bytes': '12345'}]))
 
@@ -2497,7 +2538,7 @@ class TestSloGetManifest(SloTestCase):
     def test_invalid_json_submanifest(self):
         self.app.register(
             'GET', '/v1/AUTH_test/gettest/manifest-bc',
-            swob.HTTPOk, {'Content-Type': 'application/json;swift_bytes=25',
+            swob.HTTPOk, {'Content-Type': 'application/json',
                           'X-Static-Large-Object': 'true',
                           'X-Object-Meta-Plant': 'Ficus'},
             "[this {isn't (JSON")

test/unit/container/test_server.py

@@ -2299,6 +2299,48 @@ class TestContainerController(unittest.TestCase):
         result = [x['content_type'] for x in json.loads(resp.body)]
         self.assertEqual(result, [u'\u2603', 'text/plain;charset="utf-8"'])
 
+    def test_swift_bytes_in_content_type(self):
+        # create container
+        req = Request.blank(
+            '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT',
+                                    'HTTP_X_TIMESTAMP': '0'})
+        req.get_response(self.controller)
+
+        # regular object update
+        ctype = 'text/plain; charset="utf-8"'
+        req = Request.blank(
+            '/sda1/p/a/c/o1', environ={
+                'REQUEST_METHOD': 'PUT',
+                'HTTP_X_TIMESTAMP': '1', 'HTTP_X_CONTENT_TYPE': ctype,
+                'HTTP_X_ETAG': 'x', 'HTTP_X_SIZE': 99})
+        self._update_object_put_headers(req)
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 201)
+
+        # slo object update
+        ctype = 'text/plain; charset="utf-8"; swift_bytes=12345678'
+        req = Request.blank(
+            '/sda1/p/a/c/o2', environ={
+                'REQUEST_METHOD': 'PUT',
+                'HTTP_X_TIMESTAMP': '1', 'HTTP_X_CONTENT_TYPE': ctype,
+                'HTTP_X_ETAG': 'x', 'HTTP_X_SIZE': 99})
+        self._update_object_put_headers(req)
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 201)
+
+        # verify listing
+        req = Request.blank('/sda1/p/a/c?format=json',
+                            environ={'REQUEST_METHOD': 'GET'})
+        resp = req.get_response(self.controller)
+        listing = json.loads(resp.body)
+        self.assertEqual(2, len(listing))
+        self.assertEqual('text/plain;charset="utf-8"',
+                         listing[0]['content_type'])
+        self.assertEqual(99, listing[0]['bytes'])
+        self.assertEqual('text/plain;charset="utf-8"',
+                         listing[1]['content_type'])
+        self.assertEqual(12345678, listing[1]['bytes'])
+
     def test_GET_accept_not_valid(self):
         req = Request.blank('/sda1/p/a/c', method='PUT', headers={
             'X-Timestamp': Timestamp(0).internal})

test/unit/proxy/controllers/test_obj.py

@@ -747,6 +747,20 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
         self.assertEqual(resp.status_int, 200)
         self.assertEqual(resp.headers['Transfer-Encoding'], 'chunked')
 
+    def _test_removes_swift_bytes(self, method):
+        req = swift.common.swob.Request.blank('/v1/a/c/o', method=method)
+        with set_http_connect(
+                200, headers={'content-type': 'image/jpeg; swift_bytes=99'}):
+            resp = req.get_response(self.app)
+        self.assertEqual(resp.status_int, 200)
+        self.assertEqual(resp.headers['Content-Type'], 'image/jpeg')
+
+    def test_GET_removes_swift_bytes(self):
+        self._test_removes_swift_bytes('GET')
+
+    def test_HEAD_removes_swift_bytes(self):
+        self._test_removes_swift_bytes('HEAD')
+
     def test_GET_error(self):
         req = swift.common.swob.Request.blank('/v1/a/c/o')
         self.app.logger.txn_id = req.environ['swift.trans_id'] = 'my-txn-id'