openstack/swift
Code Issues Proposed changes

Respect X-Backend-Etag-Is-At headers from left of SLO

Browse Source 
If a middleware left of SLO wants to override the ETag for a large
object, it will need to send a X-Backend-Etag-Is-At on GETs if it wants
to be at all performant. This would work fine coming out of the object
controller (which would look at the headers in the response, figure out
what's the real conditional etag, and pass it to swob.Response), and
even encryption (which would do the same), but at SLO, we'd just replace
the ETag, flag it as a conditional response, and let swob assume the
*SLO* ETag is the conditional one.

Now, SLO will jump through the same resolve_backend_etag_is_at hoops that
other parts of the proxy have to deal with. This allows If-Match and
If-None-Match to work correctly if/when swift3 stores an S3-compatible
multipart-upload ETag.

Change-Id: Ibbf59d38d7bcc9c485b1d5305548144025d77441
This commit is contained in:
Tim Burke 2018-03-26 23:50:39 +00:00
parent 9b4c978f99
commit 42adbe561f
3 changed files with 189 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
swift/common/middleware/slo.py
View File

@ -332,7 +332,7 @@ from swift.common.utils import get_logger, config_true_value, \
register_swift_info, RateLimitedIterator, quote, close_if_possible, \
closing_if_possible, LRUCache, StreamingPile, strict_b64decode
from swift.common.request_helpers import SegmentedIterable, \
get_sys_meta_prefix, update_etag_is_at_header
get_sys_meta_prefix, update_etag_is_at_header, resolve_etag_is_at_header
from swift.common.constraints import check_utf8, MAX_BUFFERED_SLO_SEGMENTS
from swift.common.http import HTTP_NOT_FOUND, HTTP_UNAUTHORIZED, is_success
from swift.common.wsgi import WSGIContext, make_subrequest
@ -792,16 +792,19 @@ class SloGetContext(WSGIContext):
if slo_etag and slo_size and (
req.method == 'HEAD' or is_conditional):
# Since we have length and etag, we can respond immediately
for i, (header, _value) in enumerate(self._response_headers):
lheader = header.lower()
if lheader == 'etag':
self._response_headers[i] = (header, '"%s"' % slo_etag)
elif lheader == 'content-length' and not is_conditional:
self._response_headers[i] = (header, slo_size)
start_response(self._response_status,
self._response_headers,
self._response_exc_info)
return resp_iter
resp = Response(
status=self._response_status,
headers=self._response_headers,
app_iter=resp_iter,
request=req,
conditional_etag=resolve_etag_is_at_header(
req, self._response_headers),
conditional_response=True)
resp.headers.update({
'Etag': '"%s"' % slo_etag,
'Content-Length': slo_size,
})
return resp(req.environ, start_response)
if self._need_to_refetch_manifest(req):
req.environ['swift.non_client_disconnect'] = True
@ -874,14 +877,15 @@ class SloGetContext(WSGIContext):
response_headers = []
for header, value in resp_headers:
lheader = header.lower()
if lheader not in ('etag', 'content-length'):
response_headers.append((header, value))
if lheader == SYSMETA_SLO_ETAG:
slo_etag = value
elif lheader == SYSMETA_SLO_SIZE:
# it's from sysmeta, so we don't worry about non-integer
# values here
content_length = int(value)
elif lheader not in ('etag', 'content-length'):
response_headers.append((header, value))
# Prep to calculate content_length & etag if necessary
if slo_etag is None:
@ -926,7 +930,9 @@ class SloGetContext(WSGIContext):
req, content_length, response_headers, segments)
def _manifest_head_response(self, req, response_headers):
conditional_etag = resolve_etag_is_at_header(req, response_headers)
return HTTPOk(request=req, headers=response_headers, body='',
conditional_etag=conditional_etag,
conditional_response=True)
def _manifest_get_response(self, req, content_length, response_headers,
@ -984,9 +990,11 @@ class SloGetContext(WSGIContext):
# the proxy logs and the user will receive incomplete results.
return HTTPConflict(request=req)
conditional_etag = resolve_etag_is_at_header(req, response_headers)
response = Response(request=req, content_length=content_length,
headers=response_headers,
conditional_response=True,
conditional_etag=conditional_etag,
app_iter=segmented_iter)
return response

9
test/unit/common/middleware/helpers.py
View File

@ -20,7 +20,7 @@ from hashlib import md5
from swift.common import swob
from swift.common.header_key_dict import HeaderKeyDict
from swift.common.request_helpers import is_user_meta, \
is_object_transient_sysmeta
is_object_transient_sysmeta, resolve_etag_is_at_header
from swift.common.swob import HTTPNotImplemented
from swift.common.utils import split_path
@ -154,11 +154,8 @@ class FakeSwift(object):
self._calls.append(
FakeSwiftCall(method, path, HeaderKeyDict(req.headers)))
backend_etag_header = req.headers.get('X-Backend-Etag-Is-At')
conditional_etag = None
if backend_etag_header and backend_etag_header in headers:
# Apply conditional etag overrides
conditional_etag = headers[backend_etag_header]
# Apply conditional etag overrides
conditional_etag = resolve_etag_is_at_header(req, headers)
# range requests ought to work, hence conditional_response=True
if isinstance(body, list):

166
test/unit/common/middleware/test_slo.py
View File

@ -1392,6 +1392,7 @@ class TestSloHeadOldManifest(SloTestCase):
'Content-Length': str(len(manifest_json)),
'Content-Type': 'test/data',
'X-Static-Large-Object': 'true',
'X-Object-Sysmeta-Artisanal-Etag': 'bespoke',
'Etag': md5hex(manifest_json)}
manifest_headers.update(getattr(self, 'extra_manifest_headers', {}))
self.manifest_has_sysmeta = all(h in manifest_headers for h in (
@ -1449,6 +1450,46 @@ class TestSloHeadOldManifest(SloTestCase):
expected_app_calls.append(('GET', '/v1/AUTH_test/headtest/man'))
self.assertEqual(self.app.calls, expected_app_calls)
def test_if_none_match_etag_matching_with_override(self):
req = Request.blank(
'/v1/AUTH_test/headtest/man',
environ={'REQUEST_METHOD': 'HEAD'},
headers={
'If-None-Match': 'bespoke',
'X-Backend-Etag-Is-At': 'X-Object-Sysmeta-Artisanal-Etag'})
status, headers, body = self.call_slo(req)
self.assertEqual(status, '304 Not Modified')
# We *are not* responsible for replacing the etag; whoever set
# x-backend-etag-is-at is responsible
self.assertIn(('Etag', '"%s"' % self.slo_etag), headers)
self.assertIn(('Content-Length', '0'), headers)
self.assertIn(('Content-Type', 'test/data'), headers)
expected_app_calls = [('HEAD', '/v1/AUTH_test/headtest/man')]
if not self.manifest_has_sysmeta:
expected_app_calls.append(('GET', '/v1/AUTH_test/headtest/man'))
self.assertEqual(self.app.calls, expected_app_calls)
def test_if_match_etag_not_matching_with_override(self):
req = Request.blank(
'/v1/AUTH_test/headtest/man',
environ={'REQUEST_METHOD': 'HEAD'},
headers={
'If-Match': self.slo_etag,
'X-Backend-Etag-Is-At': 'X-Object-Sysmeta-Artisanal-Etag'})
status, headers, body = self.call_slo(req)
self.assertEqual(status, '412 Precondition Failed')
# We *are not* responsible for replacing the etag; whoever set
# x-backend-etag-is-at is responsible
self.assertIn(('Etag', '"%s"' % self.slo_etag), headers)
self.assertIn(('Content-Length', '0'), headers)
self.assertIn(('Content-Type', 'test/data'), headers)
expected_app_calls = [('HEAD', '/v1/AUTH_test/headtest/man')]
if not self.manifest_has_sysmeta:
expected_app_calls.append(('GET', '/v1/AUTH_test/headtest/man'))
self.assertEqual(self.app.calls, expected_app_calls)
class TestSloHeadManifest(TestSloHeadOldManifest):
def setUp(self):
@ -3410,7 +3451,8 @@ class TestSloConditionalGetOldManifest(SloTestCase):
'Content-Length': str(len(_abcd_manifest_json)),
'Content-Type': 'application/json',
'X-Static-Large-Object': 'true',
'Etag': md5hex(_abcd_manifest_json)}
'Etag': md5hex(_abcd_manifest_json),
'X-Object-Sysmeta-Custom-Etag': 'a custom etag'}
manifest_headers.update(getattr(self, 'extra_manifest_headers', {}))
self.manifest_has_sysmeta = all(h in manifest_headers for h in (
'X-Object-Sysmeta-Slo-Etag', 'X-Object-Sysmeta-Slo-Size'))
@ -3521,6 +3563,128 @@ class TestSloConditionalGetOldManifest(SloTestCase):
self.assertEqual(self.app.headers[0].get('X-Backend-Etag-Is-At'),
'x-object-sysmeta-slo-etag')
def test_if_none_match_matches_with_override(self):
req = Request.blank(
'/v1/AUTH_test/gettest/manifest-abcd',
environ={'REQUEST_METHOD': 'GET'},
headers={'If-None-Match': '"a custom etag"',
'X-Backend-Etag-Is-At': 'X-Object-Sysmeta-Custom-Etag'})
status, headers, body = self.call_slo(req)
self.assertEqual(status, '304 Not Modified')
self.assertIn(('Content-Length', '0'), headers)
self.assertIn(('Etag', '"%s"' % self.slo_etag), headers)
self.assertIn(('X-Object-Sysmeta-Custom-Etag', 'a custom etag'),
headers)
self.assertEqual(body, '')
expected_app_calls = [('GET', '/v1/AUTH_test/gettest/manifest-abcd')]
if not self.manifest_has_sysmeta:
# NB: no known middleware would have written a custom etag with
# old-style manifests. but if there *was*, here's what'd happen
expected_app_calls.extend([
# 304, so gotta refetch
('GET', '/v1/AUTH_test/gettest/manifest-abcd'),
# Since the "authoritative" etag didn't come from slo, we still
# verify the first segment
('GET', '/v1/AUTH_test/gettest/manifest-bc'),
('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'),
])
self.assertEqual(self.app.calls, expected_app_calls)
self.assertEqual(
self.app.headers[0].get('X-Backend-Etag-Is-At'),
'X-Object-Sysmeta-Custom-Etag,x-object-sysmeta-slo-etag')
def test_if_none_match_does_not_match_with_override(self):
req = Request.blank(
'/v1/AUTH_test/gettest/manifest-abcd',
environ={'REQUEST_METHOD': 'GET'},
headers={'If-None-Match': "%s" % self.slo_etag,
'X-Backend-Etag-Is-At': 'X-Object-Sysmeta-Custom-Etag'})
status, headers, body = self.call_slo(req)
self.assertEqual(status, '200 OK')
self.assertIn(('Content-Length', '50'), headers)
self.assertIn(('Etag', '"%s"' % self.slo_etag), headers)
self.assertIn(('X-Object-Sysmeta-Custom-Etag', 'a custom etag'),
headers)
self.assertEqual(
body, 'aaaaabbbbbbbbbbcccccccccccccccdddddddddddddddddddd')
expected_app_calls = [
('GET', '/v1/AUTH_test/gettest/manifest-abcd'),
('GET', '/v1/AUTH_test/gettest/manifest-bc'),
('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'),
('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get'),
('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get'),
('GET', '/v1/AUTH_test/gettest/d_20?multipart-manifest=get'),
]
self.assertEqual(self.app.calls, expected_app_calls)
self.assertEqual(
self.app.headers[0].get('X-Backend-Etag-Is-At'),
'X-Object-Sysmeta-Custom-Etag,x-object-sysmeta-slo-etag')
def test_if_match_matches_with_override(self):
req = Request.blank(
'/v1/AUTH_test/gettest/manifest-abcd',
environ={'REQUEST_METHOD': 'GET'},
headers={'If-Match': '"a custom etag"',
'X-Backend-Etag-Is-At': 'X-Object-Sysmeta-Custom-Etag'})
status, headers, body = self.call_slo(req)
self.assertEqual(status, '200 OK')
self.assertIn(('Content-Length', '50'), headers)
self.assertIn(('Etag', '"%s"' % self.slo_etag), headers)
self.assertIn(('X-Object-Sysmeta-Custom-Etag', 'a custom etag'),
headers)
self.assertEqual(
body, 'aaaaabbbbbbbbbbcccccccccccccccdddddddddddddddddddd')
expected_app_calls = [
('GET', '/v1/AUTH_test/gettest/manifest-abcd'),
# Match on the override from left of us; no need to refetch
('GET', '/v1/AUTH_test/gettest/manifest-bc'),
('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'),
('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get'),
('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get'),
('GET', '/v1/AUTH_test/gettest/d_20?multipart-manifest=get'),
]
self.assertEqual(self.app.calls, expected_app_calls)
self.assertEqual(
self.app.headers[0].get('X-Backend-Etag-Is-At'),
'X-Object-Sysmeta-Custom-Etag,x-object-sysmeta-slo-etag')
def test_if_match_does_not_match_with_override(self):
req = Request.blank(
'/v1/AUTH_test/gettest/manifest-abcd',
environ={'REQUEST_METHOD': 'GET'},
headers={'If-Match': "%s" % self.slo_etag,
'X-Backend-Etag-Is-At': 'X-Object-Sysmeta-Custom-Etag'})
status, headers, body = self.call_slo(req)
self.assertEqual(status, '412 Precondition Failed')
self.assertIn(('Content-Length', '0'), headers)
self.assertIn(('Etag', '"%s"' % self.slo_etag), headers)
self.assertIn(('X-Object-Sysmeta-Custom-Etag', 'a custom etag'),
headers)
self.assertEqual(body, '')
expected_app_calls = [('GET', '/v1/AUTH_test/gettest/manifest-abcd')]
if not self.manifest_has_sysmeta:
# NB: no known middleware would have written a custom etag with
# old-style manifests. but if there *was*, here's what'd happen
expected_app_calls.extend([
# Manifest never matches -> got back a 412; need to re-fetch
('GET', '/v1/AUTH_test/gettest/manifest-abcd'),
# We *still* verify the first segment, even though we'll 412
('GET', '/v1/AUTH_test/gettest/manifest-bc'),
('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'),
])
self.assertEqual(self.app.calls, expected_app_calls)
self.assertEqual(
self.app.headers[0].get('X-Backend-Etag-Is-At'),
'X-Object-Sysmeta-Custom-Etag,x-object-sysmeta-slo-etag')
def test_if_match_matches_and_range(self):
req = Request.blank(
'/v1/AUTH_test/gettest/manifest-abcd',