openstack/swift
Code Issues Proposed changes

Fix setup of manifest responses in SLO tests

Browse Source 
The swift_bytes param is removed from the content-type
in the proxy object controller, so the SLO unit tests should
not be registering GET responses with FakeSwift that have
swift_bytes appended to the content-type.

Nor should submanifest segment dicts have swift_bytes appended to
their content-type values.

Also adds a test for the object controller and container server
handling of SLO swift_bytes.

Change-Id: Icf9bd87eee25002c8d9728b16e60c8347060f320
This commit is contained in:
Alistair Coles 2016-05-23 15:20:06 +01:00
parent d0ec1adb78
commit 7b706926a8
3 changed files with 116 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
test/unit/common/middleware/test_slo.py
View File

@ -349,7 +349,7 @@ class TestSloPutManifest(SloTestCase):
'GET', '/v1/AUTH_test/checktest/slob',
swob.HTTPOk,
{'X-Static-Large-Object': 'true', 'Etag': 'slob-etag',
'Content-Type': 'cat/picture;swift_bytes=12345',
'Content-Type': 'cat/picture',
'Content-Length': len(_manifest_json)},
_manifest_json)
@ -1106,7 +1106,7 @@ class TestSloGetRawManifest(SloTestCase):
'last_modified': '1970-01-01T00:00:00.000000'},
{'name': '/gettest/d_10',
'hash': md5hex(md5hex("e" * 5) + md5hex("f" * 5)), 'bytes': '10',
'content_type': 'application/json;swift_bytes=10',
'content_type': 'application/json',
'sub_slo': True,
'last_modified': '1970-01-01T00:00:00.000000'}])
self.bc_etag = md5hex(_bc_manifest_json)
@ -1262,7 +1262,7 @@ class TestSloGetManifest(SloTestCase):
'content_type': 'text/plain'}])
self.app.register(
'GET', '/v1/AUTH_test/gettest/manifest-bc',
swob.HTTPOk, {'Content-Type': 'application/json;swift_bytes=25',
swob.HTTPOk, {'Content-Type': 'application/json',
'X-Static-Large-Object': 'true',
'X-Object-Meta-Plant': 'Ficus',
'Etag': md5hex(_bc_manifest_json)},
@ -1272,9 +1272,9 @@ class TestSloGetManifest(SloTestCase):
[{'name': '/gettest/a_5', 'hash': md5hex("a" * 5),
'content_type': 'text/plain', 'bytes': '5'},
{'name': '/gettest/manifest-bc', 'sub_slo': True,
'content_type': 'application/json;swift_bytes=25',
'content_type': 'application/json',
'hash': md5hex(md5hex("b" * 10) + md5hex("c" * 15)),
'bytes': len(_bc_manifest_json)},
'bytes': 25},
{'name': '/gettest/d_20', 'hash': md5hex("d" * 20),
'content_type': 'text/plain', 'bytes': '20'}])
self.app.register(
@ -1284,6 +1284,34 @@ class TestSloGetManifest(SloTestCase):
'Etag': md5(_abcd_manifest_json).hexdigest()},
_abcd_manifest_json)
# A submanifest segment is created using the response headers from a
# HEAD on the submanifest. That HEAD is passed through SLO which will
# modify the response content-length to be equal to the size of the
# submanifest's large object. The swift_bytes value appended to the
# submanifest's content-type will have been removed. So the sub-slo
# segment dict that is written to the parent manifest should have the
# correct bytes and content-type values. However, if somehow the
# submanifest HEAD response wasn't modified by SLO (maybe
# historically?) and we ended up with the parent manifest sub-slo entry
# having swift_bytes appended to it's content-type and the actual
# submanifest size in its bytes field, then SLO can cope, so we create
# a deviant manifest to verify that SLO can deal with it.
_abcd_manifest_json_alt = json.dumps(
[{'name': '/gettest/a_5', 'hash': md5hex("a" * 5),
'content_type': 'text/plain', 'bytes': '5'},
{'name': '/gettest/manifest-bc', 'sub_slo': True,
'content_type': 'application/json; swift_bytes=25',
'hash': md5hex(md5hex("b" * 10) + md5hex("c" * 15)),
'bytes': len(_bc_manifest_json)},
{'name': '/gettest/d_20', 'hash': md5hex("d" * 20),
'content_type': 'text/plain', 'bytes': '20'}])
self.app.register(
'GET', '/v1/AUTH_test/gettest/manifest-abcd-alt',
swob.HTTPOk, {'Content-Type': 'application/json',
'X-Static-Large-Object': 'true',
'Etag': md5(_abcd_manifest_json_alt).hexdigest()},
_abcd_manifest_json_alt)
_abcdefghijkl_manifest_json = json.dumps(
[{'name': '/gettest/a_5', 'hash': md5hex("a" * 5),
'content_type': 'text/plain', 'bytes': '5'},
@ -1337,7 +1365,7 @@ class TestSloGetManifest(SloTestCase):
self.bc_ranges_etag = md5hex(_bc_ranges_manifest_json)
self.app.register(
'GET', '/v1/AUTH_test/gettest/manifest-bc-ranges',
swob.HTTPOk, {'Content-Type': 'application/json;swift_bytes=16',
swob.HTTPOk, {'Content-Type': 'application/json',
'X-Static-Large-Object': 'true',
'X-Object-Meta-Plant': 'Ficus',
'Etag': self.bc_ranges_etag},
@ -1351,12 +1379,12 @@ class TestSloGetManifest(SloTestCase):
'content_type': 'text/plain', 'bytes': '5',
'range': '1-4'},
{'name': '/gettest/manifest-bc-ranges', 'sub_slo': True,
'content_type': 'application/json;swift_bytes=16',
'content_type': 'application/json',
'hash': self.bc_ranges_etag,
'bytes': len(_bc_ranges_manifest_json),
'bytes': 16,
'range': '8-15'},
{'name': '/gettest/manifest-bc-ranges', 'sub_slo': True,
'content_type': 'application/json;swift_bytes=16',
'content_type': 'application/json',
'hash': self.bc_ranges_etag,
'bytes': len(_bc_ranges_manifest_json),
'range': '0-7'},
@ -1655,6 +1683,22 @@ class TestSloGetManifest(SloTestCase):
self.assertEqual(
body, 'aaaaabbbbbbbbbbcccccccccccccccdddddddddddddddddddd')
def test_get_manifest_with_submanifest_bytes_in_content_type(self):
# verify correct content-length when the sub-slo segment in the
# manifest has its actual object content-length appended as swift_bytes
# to the content-type, and the submanifest length in the bytes field.
req = Request.blank(
'/v1/AUTH_test/gettest/manifest-abcd-alt',
environ={'REQUEST_METHOD': 'GET'})
status, headers, body = self.call_slo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(status, '200 OK')
self.assertEqual(headers['Content-Length'], '50')
self.assertEqual(headers['Etag'], '"%s"' % self.manifest_abcd_etag)
self.assertEqual(
body, 'aaaaabbbbbbbbbbcccccccccccccccdddddddddddddddddddd')
def test_range_get_manifest(self):
req = Request.blank(
'/v1/AUTH_test/gettest/manifest-abcd',
@ -2274,8 +2318,7 @@ class TestSloGetManifest(SloTestCase):
'hash': 'man%d' % (i + 1),
'sub_slo': True,
'bytes': len(manifest_json),
'content_type':
'application/json;swift_bytes=%d' % ((21 - i) * 6)}]
'content_type': 'application/json'}]
manifest_json = json.dumps(manifest_data)
self.app.register(
@ -2330,9 +2373,8 @@ class TestSloGetManifest(SloTestCase):
{'name': '/gettest/man%d' % (i + 1),
'hash': 'man%d' % (i + 1),
'sub_slo': True,
'bytes': len(manifest_json),
'content_type':
'application/json;swift_bytes=%d' % ((10 - i) * 6)},
'bytes': (10 - i) * 6,
'content_type': 'application/json'},
{'name': '/gettest/obj%d' % i,
'hash': md5hex('body%02d' % i),
'bytes': '6',
@ -2387,9 +2429,8 @@ class TestSloGetManifest(SloTestCase):
{'name': '/gettest/man%d' % (i + 1),
'hash': 'man%d' % (i + 1),
'sub_slo': True,
'bytes': len(manifest_json),
'content_type':
'application/json;swift_bytes=%d' % ((12 - i) * 6)},
'bytes': (12 - i) * 6,
'content_type': 'application/json'},
{'name': '/gettest/obj%d' % i,
'hash': md5hex('body%02d' % i),
'bytes': '6',
@ -2479,7 +2520,7 @@ class TestSloGetManifest(SloTestCase):
swob.HTTPOk, {'Content-Type': 'application/json',
'X-Static-Large-Object': 'true'},
json.dumps([{'name': '/gettest/manifest-a', 'sub_slo': True,
'content_type': 'application/json;swift_bytes=5',
'content_type': 'application/json',
'hash': 'manifest-a',
'bytes': '12345'}]))
@ -2497,7 +2538,7 @@ class TestSloGetManifest(SloTestCase):
def test_invalid_json_submanifest(self):
self.app.register(
'GET', '/v1/AUTH_test/gettest/manifest-bc',
swob.HTTPOk, {'Content-Type': 'application/json;swift_bytes=25',
swob.HTTPOk, {'Content-Type': 'application/json',
'X-Static-Large-Object': 'true',
'X-Object-Meta-Plant': 'Ficus'},
"[this {isn't (JSON")

42
test/unit/container/test_server.py
View File

@ -2299,6 +2299,48 @@ class TestContainerController(unittest.TestCase):
result = [x['content_type'] for x in json.loads(resp.body)]
self.assertEqual(result, [u'\u2603', 'text/plain;charset="utf-8"'])
def test_swift_bytes_in_content_type(self):
# create container
req = Request.blank(
'/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '0'})
req.get_response(self.controller)
# regular object update
ctype = 'text/plain; charset="utf-8"'
req = Request.blank(
'/sda1/p/a/c/o1', environ={
'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '1', 'HTTP_X_CONTENT_TYPE': ctype,
'HTTP_X_ETAG': 'x', 'HTTP_X_SIZE': 99})
self._update_object_put_headers(req)
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 201)
# slo object update
ctype = 'text/plain; charset="utf-8"; swift_bytes=12345678'
req = Request.blank(
'/sda1/p/a/c/o2', environ={
'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '1', 'HTTP_X_CONTENT_TYPE': ctype,
'HTTP_X_ETAG': 'x', 'HTTP_X_SIZE': 99})
self._update_object_put_headers(req)
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 201)
# verify listing
req = Request.blank('/sda1/p/a/c?format=json',
environ={'REQUEST_METHOD': 'GET'})
resp = req.get_response(self.controller)
listing = json.loads(resp.body)
self.assertEqual(2, len(listing))
self.assertEqual('text/plain;charset="utf-8"',
listing[0]['content_type'])
self.assertEqual(99, listing[0]['bytes'])
self.assertEqual('text/plain;charset="utf-8"',
listing[1]['content_type'])
self.assertEqual(12345678, listing[1]['bytes'])
def test_GET_accept_not_valid(self):
req = Request.blank('/sda1/p/a/c', method='PUT', headers={
'X-Timestamp': Timestamp(0).internal})

14
test/unit/proxy/controllers/test_obj.py
View File

@ -747,6 +747,20 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
self.assertEqual(resp.status_int, 200)
self.assertEqual(resp.headers['Transfer-Encoding'], 'chunked')
def _test_removes_swift_bytes(self, method):
req = swift.common.swob.Request.blank('/v1/a/c/o', method=method)
with set_http_connect(
200, headers={'content-type': 'image/jpeg; swift_bytes=99'}):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 200)
self.assertEqual(resp.headers['Content-Type'], 'image/jpeg')
def test_GET_removes_swift_bytes(self):
self._test_removes_swift_bytes('GET')
def test_HEAD_removes_swift_bytes(self):
self._test_removes_swift_bytes('HEAD')
def test_GET_error(self):
req = swift.common.swob.Request.blank('/v1/a/c/o')
self.app.logger.txn_id = req.environ['swift.trans_id'] = 'my-txn-id'