84b85f03b4
Multipart uploads in AWS (seem to) have ETags like: '"' + MD5_hex(MD5(part1) + ... + MD5(partN)) + '-' + N + '"' On the other hand, Swift SLOs have Etags like: MD5_hex(MD5_hex(part1) + ... + MD5_hex(partN)) (In both examples, MD5 gets the raw 16-byte digest while MD5_hex gets the 32-byte hex-encoded digest.) Some clients (such as aws-sdk-java) use the presence of a dash to decide whether to perform client-side validation of downloads. Other clients (like s3cmd) use the presence of a dash *in bucket listings* to decide whether or not to perform additional HEAD requests to look for MD5 metadata that can be used to compare against the MD5s of local files. Now we include a dash as well, to prevent spurious errors like > Unable to verify integrity of data download. Client calculated > content hash didn't match hash calculated by Amazon S3. The data > may be corrupt. or unnecessary uploads/downloads because the client assumes data has changed that hasn't. For new multipart-uploads via the S3 API, the ETag that is stored will be calculated in the same way that AWS uses. This ETag will be used in GET/HEAD responses, bucket listings, and conditional requests via the S3 API. Accessing the same object via the Swift API will use the SLO Etag; however, in JSON container listings the multipart upload etag will be exposed in a new "s3_etag" key. New SLOs and pre-existing multipart-uploads will continue to behave as before; there is no data migration or mitigation as part of this patch. Change-Id: Ibe68c44bef6c17605863e9084503e8f5dc577fab Closes-Bug: 1522578
83 lines
3.7 KiB
Python
83 lines
3.7 KiB
Python
# Copyright (c) 2014 OpenStack Foundation
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import unittest
|
|
|
|
from swift.common.swob import Response
|
|
from swift.common.utils import HeaderKeyDict
|
|
from swift.common.middleware.s3api.s3response import S3Response
|
|
from swift.common.middleware.s3api.utils import sysmeta_prefix
|
|
|
|
|
|
class TestResponse(unittest.TestCase):
|
|
def test_from_swift_resp_slo(self):
|
|
for expected, header_vals in \
|
|
((True, ('true', '1')), (False, ('false', 'ugahhh', None))):
|
|
for val in header_vals:
|
|
resp = Response(headers={'X-Static-Large-Object': val,
|
|
'Etag': 'theetag'})
|
|
s3resp = S3Response.from_swift_resp(resp)
|
|
self.assertEqual(expected, s3resp.is_slo)
|
|
self.assertEqual('"theetag"', s3resp.headers['ETag'])
|
|
|
|
def test_response_s3api_sysmeta_headers(self):
|
|
for _server_type in ('object', 'container'):
|
|
swift_headers = HeaderKeyDict(
|
|
{sysmeta_prefix(_server_type) + 'test': 'ok'})
|
|
resp = Response(headers=swift_headers)
|
|
s3resp = S3Response.from_swift_resp(resp)
|
|
self.assertEqual(swift_headers, s3resp.sysmeta_headers)
|
|
|
|
def test_response_s3api_sysmeta_headers_ignore_other_sysmeta(self):
|
|
for _server_type in ('object', 'container'):
|
|
swift_headers = HeaderKeyDict(
|
|
# sysmeta not leading sysmeta_prefix even including s3api word
|
|
{'x-%s-sysmeta-test-s3api' % _server_type: 'ok',
|
|
sysmeta_prefix(_server_type) + 'test': 'ok'})
|
|
resp = Response(headers=swift_headers)
|
|
s3resp = S3Response.from_swift_resp(resp)
|
|
expected_headers = HeaderKeyDict(
|
|
{sysmeta_prefix(_server_type) + 'test': 'ok'})
|
|
self.assertEqual(expected_headers, s3resp.sysmeta_headers)
|
|
|
|
def test_response_s3api_sysmeta_from_swift3_sysmeta(self):
|
|
for _server_type in ('object', 'container'):
|
|
# swift could return older swift3 sysmeta
|
|
swift_headers = HeaderKeyDict(
|
|
{('x-%s-sysmeta-swift3-' % _server_type) + 'test': 'ok'})
|
|
resp = Response(headers=swift_headers)
|
|
s3resp = S3Response.from_swift_resp(resp)
|
|
expected_headers = HeaderKeyDict(
|
|
{sysmeta_prefix(_server_type) + 'test': 'ok'})
|
|
# but Response class should translates as s3api sysmeta
|
|
self.assertEqual(expected_headers, s3resp.sysmeta_headers)
|
|
|
|
def test_response_swift3_sysmeta_does_not_overwrite_s3api_sysmeta(self):
|
|
for _server_type in ('object', 'container'):
|
|
# same key name except sysmeta prefix
|
|
swift_headers = HeaderKeyDict(
|
|
{('x-%s-sysmeta-swift3-' % _server_type) + 'test': 'ng',
|
|
sysmeta_prefix(_server_type) + 'test': 'ok'})
|
|
resp = Response(headers=swift_headers)
|
|
s3resp = S3Response.from_swift_resp(resp)
|
|
expected_headers = HeaderKeyDict(
|
|
{sysmeta_prefix(_server_type) + 'test': 'ok'})
|
|
# but only s3api sysmeta remains in the response sysmeta_headers
|
|
self.assertEqual(expected_headers, s3resp.sysmeta_headers)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|