Remove all post_as_copy related code and configes

It was deprecated and we discussed on this topic in Denver PTG
for Queen cycle. Main motivation for this work is that deprecated
post_as_copy option and its gate blocks future symlink work.

Change-Id: I411893db1565864ed5beb6ae75c38b982a574476
This commit is contained in:
Kota Tsuyuzaki 2017-09-14 20:57:46 +09:00 committed by Tim Burke
parent 834b733d29
commit 1e79f828ad
17 changed files with 30 additions and 445 deletions

View File

@ -996,8 +996,6 @@ Error count to consider a node error limited. The default is 10.
Whether account PUTs and DELETEs are even callable. If set to 'true' any authorized Whether account PUTs and DELETEs are even callable. If set to 'true' any authorized
user may create and delete accounts; if 'false' no one, even authorized, can. The default user may create and delete accounts; if 'false' no one, even authorized, can. The default
is false. is false.
.IP \fBobject_post_as_copy\fR
Deprecated. The default is False.
.IP \fBaccount_autocreate\fR .IP \fBaccount_autocreate\fR
If set to 'true' authorized accounts that do not yet exist within the Swift cluster If set to 'true' authorized accounts that do not yet exist within the Swift cluster
will be automatically created. The default is set to false. will be automatically created. The default is set to false.

View File

@ -1884,7 +1884,6 @@ error_suppression_limit 10 Error count to consider
node error limited node error limited
allow_account_management false Whether account PUTs and DELETEs allow_account_management false Whether account PUTs and DELETEs
are even callable are even callable
object_post_as_copy false Deprecated.
account_autocreate false If set to 'true' authorized account_autocreate false If set to 'true' authorized
accounts that do not yet exist accounts that do not yet exist
within the Swift cluster will within the Swift cluster will

View File

@ -127,9 +127,6 @@ set using environment variables:
environment variable ``SWIFT_TEST_IN_PROCESS_CONF_LOADER`` to environment variable ``SWIFT_TEST_IN_PROCESS_CONF_LOADER`` to
``ec``. ``ec``.
- the deprecated proxy-server ``object_post_as_copy`` option may be set using
the environment variable ``SWIFT_TEST_IN_PROCESS_OBJECT_POST_AS_COPY``.
- logging to stdout may be enabled by setting ``SWIFT_TEST_DEBUG_LOGS``. - logging to stdout may be enabled by setting ``SWIFT_TEST_DEBUG_LOGS``.
For example, this command would run the in-process mode functional tests with For example, this command would run the in-process mode functional tests with
@ -147,7 +144,6 @@ The ``tox.ini`` file also specifies test environments for running other
in-process functional test configurations, e.g.:: in-process functional test configurations, e.g.::
tox -e func-ec tox -e func-ec
tox -e func-post-as-copy
To debug the functional tests, use the 'in-process test' mode and pass the To debug the functional tests, use the 'in-process test' mode and pass the
``--pdb`` flag to ``tox``:: ``--pdb`` flag to ``tox``::

View File

@ -860,8 +860,6 @@ use = egg:swift#copy
# requests are transformed into COPY requests where source and destination are # requests are transformed into COPY requests where source and destination are
# the same. All client-visible behavior (save response time) should be # the same. All client-visible behavior (save response time) should be
# identical. # identical.
# This option is deprecated and will be ignored in a future release.
# object_post_as_copy = false
# Note: To enable encryption, add the following 2 dependent pieces of crypto # Note: To enable encryption, add the following 2 dependent pieces of crypto
# middleware to the proxy-server pipeline. They should be to the right of all # middleware to the proxy-server pipeline. They should be to the right of all

View File

@ -112,20 +112,6 @@ If a request is sent without the query parameter, an attempt will be made to
copy the whole object but will fail if the object size is copy the whole object but will fail if the object size is
greater than 5GB. greater than 5GB.
-------------------
Object Post as Copy
-------------------
Historically, this has been a feature (and a configurable option with default
set to True) in proxy server configuration. This has been moved to server side
copy middleware and the default changed to False.
When ``object_post_as_copy`` is set to ``true``, an incoming POST request is
morphed into a COPY request where source and destination objects are same.
This feature was necessary because of a previous behavior where POSTS would
update the metadata on the object but not on the container. As a result,
features like container sync would not work correctly. This is no longer the
case and this option is now deprecated. It will be removed in a future release.
""" """
import os import os
@ -137,8 +123,7 @@ from swift.common.utils import get_logger, \
config_true_value, FileLikeIter, read_conf_dir, close_if_possible config_true_value, FileLikeIter, read_conf_dir, close_if_possible
from swift.common.swob import Request, HTTPPreconditionFailed, \ from swift.common.swob import Request, HTTPPreconditionFailed, \
HTTPRequestEntityTooLarge, HTTPBadRequest, HTTPException HTTPRequestEntityTooLarge, HTTPBadRequest, HTTPException
from swift.common.http import HTTP_MULTIPLE_CHOICES, HTTP_CREATED, \ from swift.common.http import HTTP_MULTIPLE_CHOICES, is_success, HTTP_OK
is_success, HTTP_OK
from swift.common.constraints import check_account_format, MAX_FILE_SIZE from swift.common.constraints import check_account_format, MAX_FILE_SIZE
from swift.common.request_helpers import copy_header_subset, remove_items, \ from swift.common.request_helpers import copy_header_subset, remove_items, \
is_sys_meta, is_sys_or_user_meta, is_object_transient_sysmeta is_sys_meta, is_sys_or_user_meta, is_object_transient_sysmeta
@ -238,13 +223,7 @@ class ServerSideCopyWebContext(WSGIContext):
return app_resp return app_resp
def _adjust_put_response(self, req, additional_resp_headers): def _adjust_put_response(self, req, additional_resp_headers):
if 'swift.post_as_copy' in req.environ: if is_success(self._get_status_int()):
# Older editions returned 202 Accepted on object POSTs, so we'll
# convert any 201 Created responses to that for compatibility with
# picky clients.
if self._get_status_int() == HTTP_CREATED:
self._response_status = '202 Accepted'
elif is_success(self._get_status_int()):
for header, value in additional_resp_headers.items(): for header, value in additional_resp_headers.items():
self._response_headers.append((header, value)) self._response_headers.append((header, value))
@ -269,17 +248,12 @@ class ServerSideCopyMiddleware(object):
def __init__(self, app, conf): def __init__(self, app, conf):
self.app = app self.app = app
self.logger = get_logger(conf, log_route="copy") self.logger = get_logger(conf, log_route="copy")
# Read the old object_post_as_copy option from Proxy app just in case
# someone has set it to false (non default). This wouldn't cause
# problems during upgrade.
self._load_object_post_as_copy_conf(conf) self._load_object_post_as_copy_conf(conf)
self.object_post_as_copy = \ self.object_post_as_copy = \
config_true_value(conf.get('object_post_as_copy', 'false')) config_true_value(conf.get('object_post_as_copy', 'false'))
if self.object_post_as_copy: if self.object_post_as_copy:
msg = ('object_post_as_copy=true is deprecated; remove all ' msg = ('object_post_as_copy=true is deprecated; This '
'references to it from %s to disable this warning. This ' 'option is now ignored')
'option will be ignored in a future release' % conf.get(
'__file__', 'proxy-server.conf'))
self.logger.warning(msg) self.logger.warning(msg)
def _load_object_post_as_copy_conf(self, conf): def _load_object_post_as_copy_conf(self, conf):
@ -330,9 +304,6 @@ class ServerSideCopyMiddleware(object):
elif req.method == 'COPY': elif req.method == 'COPY':
req.environ['swift.orig_req_method'] = req.method req.environ['swift.orig_req_method'] = req.method
return self.handle_COPY(req, start_response) return self.handle_COPY(req, start_response)
elif req.method == 'POST' and self.object_post_as_copy:
req.environ['swift.orig_req_method'] = req.method
return self.handle_object_post_as_copy(req, start_response)
elif req.method == 'OPTIONS': elif req.method == 'OPTIONS':
# Does not interfere with OPTIONS response from # Does not interfere with OPTIONS response from
# (account,container) servers and /info response. # (account,container) servers and /info response.
@ -343,21 +314,6 @@ class ServerSideCopyMiddleware(object):
return self.app(env, start_response) return self.app(env, start_response)
def handle_object_post_as_copy(self, req, start_response):
req.method = 'PUT'
req.path_info = '/v1/%s/%s/%s' % (
self.account_name, self.container_name, self.object_name)
req.headers['Content-Length'] = 0
req.headers.pop('Range', None)
req.headers['X-Copy-From'] = quote('/%s/%s' % (self.container_name,
self.object_name))
req.environ['swift.post_as_copy'] = True
params = req.params
# for post-as-copy always copy the manifest itself if source is *LO
params['multipart-manifest'] = 'get'
req.params = params
return self.handle_PUT(req, start_response)
def handle_COPY(self, req, start_response): def handle_COPY(self, req, start_response):
if not req.headers.get('Destination'): if not req.headers.get('Destination'):
return HTTPPreconditionFailed(request=req, return HTTPPreconditionFailed(request=req,
@ -394,11 +350,6 @@ class ServerSideCopyMiddleware(object):
source_req.headers.pop('X-Backend-Storage-Policy-Index', None) source_req.headers.pop('X-Backend-Storage-Policy-Index', None)
source_req.path_info = quote(source_path) source_req.path_info = quote(source_path)
source_req.headers['X-Newest'] = 'true' source_req.headers['X-Newest'] = 'true'
if 'swift.post_as_copy' in req.environ:
# We're COPYing one object over itself because of a POST; rely on
# the PUT for write authorization, don't require read authorization
source_req.environ['swift.authorize'] = lambda req: None
source_req.environ['swift.authorize_override'] = True
# in case we are copying an SLO manifest, set format=raw parameter # in case we are copying an SLO manifest, set format=raw parameter
params = source_req.params params = source_req.params
@ -470,11 +421,7 @@ class ServerSideCopyMiddleware(object):
def is_object_sysmeta(k): def is_object_sysmeta(k):
return is_sys_meta('object', k) return is_sys_meta('object', k)
if 'swift.post_as_copy' in sink_req.environ: if config_true_value(req.headers.get('x-fresh-metadata', 'false')):
# Post-as-copy: ignore new sysmeta, copy existing sysmeta
remove_items(sink_req.headers, is_object_sysmeta)
copy_header_subset(source_resp, sink_req, is_object_sysmeta)
elif config_true_value(req.headers.get('x-fresh-metadata', 'false')):
# x-fresh-metadata only applies to copy, not post-as-copy: ignore # x-fresh-metadata only applies to copy, not post-as-copy: ignore
# existing user metadata, update existing sysmeta with new # existing user metadata, update existing sysmeta with new
copy_header_subset(source_resp, sink_req, is_object_sysmeta) copy_header_subset(source_resp, sink_req, is_object_sysmeta)
@ -497,9 +444,8 @@ class ServerSideCopyMiddleware(object):
params['multipart-manifest'] = 'put' params['multipart-manifest'] = 'put'
if 'X-Object-Manifest' in source_resp.headers: if 'X-Object-Manifest' in source_resp.headers:
del params['multipart-manifest'] del params['multipart-manifest']
if 'swift.post_as_copy' not in sink_req.environ: sink_req.headers['X-Object-Manifest'] = \
sink_req.headers['X-Object-Manifest'] = \ source_resp.headers['X-Object-Manifest']
source_resp.headers['X-Object-Manifest']
sink_req.params = params sink_req.params = params
# Set swift.source, data source, content length and etag # Set swift.source, data source, content length and etag

View File

@ -826,8 +826,7 @@ class VersionedWritesMiddleware(object):
allow_versioned_writes) allow_versioned_writes)
except HTTPException as error_response: except HTTPException as error_response:
return error_response(env, start_response) return error_response(env, start_response)
elif (obj and req.method in ('PUT', 'DELETE') and elif (obj and req.method in ('PUT', 'DELETE')):
not req.environ.get('swift.post_as_copy')):
try: try:
return self.object_request( return self.object_request(
req, api_version, account, container, obj, req, api_version, account, container, obj,

View File

@ -505,15 +505,6 @@ def in_process_setup(the_object_server=object_server):
'password6': 'testing6' 'password6': 'testing6'
}) })
# If an env var explicitly specifies the proxy-server object_post_as_copy
# option then use its value, otherwise leave default config unchanged.
object_post_as_copy = os.environ.get(
'SWIFT_TEST_IN_PROCESS_OBJECT_POST_AS_COPY')
if object_post_as_copy is not None:
object_post_as_copy = config_true_value(object_post_as_copy)
config['object_post_as_copy'] = str(object_post_as_copy)
_debug('Setting object_post_as_copy to %r' % object_post_as_copy)
acc1lis = listen_zero() acc1lis = listen_zero()
acc2lis = listen_zero() acc2lis = listen_zero()
con1lis = listen_zero() con1lis = listen_zero()

View File

@ -448,7 +448,7 @@ class ProbeTest(unittest.TestCase):
else: else:
os.system('sudo mount %s' % device) os.system('sudo mount %s' % device)
def make_internal_client(self, object_post_as_copy=True): def make_internal_client(self):
tempdir = mkdtemp() tempdir = mkdtemp()
try: try:
conf_path = os.path.join(tempdir, 'internal_client.conf') conf_path = os.path.join(tempdir, 'internal_client.conf')
@ -464,14 +464,13 @@ class ProbeTest(unittest.TestCase):
[filter:copy] [filter:copy]
use = egg:swift#copy use = egg:swift#copy
object_post_as_copy = %s
[filter:cache] [filter:cache]
use = egg:swift#memcache use = egg:swift#memcache
[filter:catch_errors] [filter:catch_errors]
use = egg:swift#catch_errors use = egg:swift#catch_errors
""" % object_post_as_copy """
with open(conf_path, 'w') as f: with open(conf_path, 'w') as f:
f.write(dedent(conf_body)) f.write(dedent(conf_body))
return internal_client.InternalClient(conf_path, 'test', 1) return internal_client.InternalClient(conf_path, 'test', 1)

View File

@ -93,7 +93,7 @@ class TestContainerSync(ReplProbeTest):
return source['name'], dest['name'] return source['name'], dest['name']
def _test_sync(self, object_post_as_copy): def test_sync(self):
source_container, dest_container = self._setup_synced_containers() source_container, dest_container = self._setup_synced_containers()
# upload to source # upload to source
@ -111,12 +111,10 @@ class TestContainerSync(ReplProbeTest):
self.assertIn('x-object-meta-test', resp_headers) self.assertIn('x-object-meta-test', resp_headers)
self.assertEqual('put_value', resp_headers['x-object-meta-test']) self.assertEqual('put_value', resp_headers['x-object-meta-test'])
# update metadata with a POST, using an internal client so we can # update metadata with a POST
# vary the object_post_as_copy setting - first use post-as-copy
post_headers = {'Content-Type': 'image/jpeg', post_headers = {'Content-Type': 'image/jpeg',
'X-Object-Meta-Test': 'post_value'} 'X-Object-Meta-Test': 'post_value'}
int_client = self.make_internal_client( int_client = self.make_internal_client()
object_post_as_copy=object_post_as_copy)
int_client.set_object_metadata(self.account, source_container, int_client.set_object_metadata(self.account, source_container,
object_name, post_headers) object_name, post_headers)
# sanity checks... # sanity checks...
@ -154,12 +152,6 @@ class TestContainerSync(ReplProbeTest):
self.url, self.token, dest_container, object_name) self.url, self.token, dest_container, object_name)
self.assertEqual(404, cm.exception.http_status) # sanity check self.assertEqual(404, cm.exception.http_status) # sanity check
def test_sync_with_post_as_copy(self):
self._test_sync(True)
def test_sync_with_fast_post(self):
self._test_sync(False)
def test_sync_slo_manifest(self): def test_sync_slo_manifest(self):
# Verify that SLO manifests are sync'd even if their segments can not # Verify that SLO manifests are sync'd even if their segments can not
# be found in the destination account at time of sync'ing. # be found in the destination account at time of sync'ing.

View File

@ -237,8 +237,7 @@ class TestUpdateOverridesEC(ECProbeTest):
self.assertFalse(direct_client.direct_get_container( self.assertFalse(direct_client.direct_get_container(
cnodes[0], cpart, self.account, 'c1')[1]) cnodes[0], cpart, self.account, 'c1')[1])
# use internal client for POST so we can force fast-post mode int_client = self.make_internal_client()
int_client = self.make_internal_client(object_post_as_copy=False)
int_client.set_object_metadata( int_client.set_object_metadata(
self.account, 'c1', 'o1', {'X-Object-Meta-Fruit': 'Tomato'}) self.account, 'c1', 'o1', {'X-Object-Meta-Fruit': 'Tomato'})
self.assertEqual( self.assertEqual(
@ -296,8 +295,7 @@ class TestUpdateOverridesEC(ECProbeTest):
content_type='test/ctype') content_type='test/ctype')
meta = client.head_object(self.url, self.token, 'c1', 'o1') meta = client.head_object(self.url, self.token, 'c1', 'o1')
# use internal client for POST so we can force fast-post mode int_client = self.make_internal_client()
int_client = self.make_internal_client(object_post_as_copy=False)
int_client.set_object_metadata( int_client.set_object_metadata(
self.account, 'c1', 'o1', {'X-Object-Meta-Fruit': 'Tomato'}) self.account, 'c1', 'o1', {'X-Object-Meta-Fruit': 'Tomato'})
self.assertEqual( self.assertEqual(

View File

@ -47,7 +47,7 @@ class Test(ReplProbeTest):
policy=self.policy) policy=self.policy)
self.container_brain = BrainSplitter(self.url, self.token, self.container_brain = BrainSplitter(self.url, self.token,
self.container_name) self.container_name)
self.int_client = self.make_internal_client(object_post_as_copy=False) self.int_client = self.make_internal_client()
def _get_object_info(self, account, container, obj, number): def _get_object_info(self, account, container, obj, number):
obj_conf = self.configs['object-server'] obj_conf = self.configs['object-server']

View File

@ -618,14 +618,5 @@ class TestCryptoPipelineChanges(unittest.TestCase):
self._check_listing(self.crypto_app) self._check_listing(self.crypto_app)
class TestCryptoPipelineChangesFastPost(TestCryptoPipelineChanges):
@classmethod
def setUpClass(cls):
# set proxy config to use fast post
extra_conf = {'object_post_as_copy': 'False'}
cls._test_context = setup_servers(extra_conf=extra_conf)
cls.proxy_app = cls._test_context["test_servers"][0]
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()

View File

@ -14,7 +14,6 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import time
import mock import mock
import shutil import shutil
import tempfile import tempfile
@ -93,9 +92,7 @@ class TestCopyConstraints(unittest.TestCase):
class TestServerSideCopyMiddleware(unittest.TestCase): class TestServerSideCopyMiddleware(unittest.TestCase):
def setUp(self): def setUp(self):
self.app = FakeSwift() self.app = FakeSwift()
self.ssc = copy.filter_factory({ self.ssc = copy.filter_factory({})(self.app)
'object_post_as_copy': 'yes',
})(self.app)
self.ssc.logger = self.app.logger self.ssc.logger = self.app.logger
def tearDown(self): def tearDown(self):
@ -166,92 +163,6 @@ class TestServerSideCopyMiddleware(unittest.TestCase):
self.assertRequestEqual(req, self.authorized[0]) self.assertRequestEqual(req, self.authorized[0])
self.assertNotIn('swift.orig_req_method', req.environ) self.assertNotIn('swift.orig_req_method', req.environ)
def test_POST_as_COPY_simple(self):
self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
self.app.register('PUT', '/v1/a/c/o', swob.HTTPAccepted, {})
req = Request.blank('/v1/a/c/o', method='POST')
status, headers, body = self.call_ssc(req)
self.assertEqual(status, '202 Accepted')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
# For basic test cases, assert orig_req_method behavior
self.assertEqual(req.environ['swift.orig_req_method'], 'POST')
def test_POST_as_COPY_201_return_202(self):
self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {})
req = Request.blank('/v1/a/c/o', method='POST')
status, headers, body = self.call_ssc(req)
self.assertEqual(status, '202 Accepted')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_POST_delete_at(self):
self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
self.app.register('PUT', '/v1/a/c/o', swob.HTTPAccepted, {})
t = str(int(time.time() + 100))
req = Request.blank('/v1/a/c/o', method='POST',
headers={'Content-Type': 'foo/bar',
'X-Delete-At': t})
status, headers, body = self.call_ssc(req)
self.assertEqual(status, '202 Accepted')
calls = self.app.calls_with_headers
method, path, req_headers = calls[1]
self.assertEqual('PUT', method)
self.assertTrue('X-Delete-At' in req_headers)
self.assertEqual(req_headers['X-Delete-At'], str(t))
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_POST_as_COPY_static_large_object(self):
self.app.register('GET', '/v1/a/c/o', swob.HTTPOk,
{'X-Static-Large-Object': True}, 'passed')
self.app.register('PUT', '/v1/a/c/o', swob.HTTPAccepted, {})
req = Request.blank('/v1/a/c/o', method='POST',
headers={})
status, headers, body = self.call_ssc(req)
self.assertEqual(status, '202 Accepted')
calls = self.app.calls_with_headers
method, path, req_headers = calls[1]
self.assertEqual('PUT', method)
self.assertNotIn('X-Static-Large-Object', req_headers)
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_POST_as_COPY_dynamic_large_object_manifest(self):
self.app.register('GET', '/v1/a/c/o', swob.HTTPOk,
{'X-Object-Manifest': 'orig_manifest'}, 'passed')
self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {})
req = Request.blank('/v1/a/c/o', method='POST',
headers={'X-Object-Manifest': 'new_manifest'})
status, headers, body = self.call_ssc(req)
self.assertEqual(status, '202 Accepted')
calls = self.app.calls_with_headers
method, path, req_headers = calls[1]
self.assertEqual('PUT', method)
self.assertEqual('new_manifest', req_headers['x-object-manifest'])
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_POST_as_COPY_dynamic_large_object_no_manifest(self):
self.app.register('GET', '/v1/a/c/o', swob.HTTPOk,
{'X-Object-Manifest': 'orig_manifest'}, 'passed')
self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {})
req = Request.blank('/v1/a/c/o', method='POST',
headers={})
status, headers, body = self.call_ssc(req)
self.assertEqual(status, '202 Accepted')
calls = self.app.calls_with_headers
method, path, req_headers = calls[1]
self.assertEqual('PUT', method)
self.assertNotIn('X-Object-Manifest', req_headers)
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_basic_put_with_x_copy_from(self): def test_basic_put_with_x_copy_from(self):
self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
self.app.register('PUT', '/v1/a/c/o2', swob.HTTPCreated, {}) self.app.register('PUT', '/v1/a/c/o2', swob.HTTPCreated, {})
@ -1345,100 +1256,6 @@ class TestServerSideCopyMiddleware(unittest.TestCase):
req_headers.get('X-Object-Transient-Sysmeta-Test')) req_headers.get('X-Object-Transient-Sysmeta-Test'))
self.assertEqual('Not Bar', req_headers.get('X-Foo')) self.assertEqual('Not Bar', req_headers.get('X-Foo'))
def _test_POST_source_headers(self, extra_post_headers):
# helper method to perform a POST with metadata headers that should
# always be sent to the destination
post_headers = {'X-Object-Meta-Test2': 'added',
'X-Object-Sysmeta-Test2': 'added',
'X-Object-Transient-Sysmeta-Test2': 'added'}
post_headers.update(extra_post_headers)
get_resp_headers = {
'X-Timestamp': '1234567890.12345',
'X-Backend-Timestamp': '1234567890.12345',
'Content-Type': 'text/original',
'Content-Encoding': 'gzip',
'Content-Disposition': 'attachment; filename=myfile',
'X-Object-Meta-Test': 'original',
'X-Object-Sysmeta-Test': 'original',
'X-Object-Transient-Sysmeta-Test': 'original',
'X-Foo': 'Bar'}
self.app.register(
'GET', '/v1/a/c/o', swob.HTTPOk, headers=get_resp_headers)
self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {})
req = Request.blank('/v1/a/c/o', method='POST', headers=post_headers)
status, headers, body = self.call_ssc(req)
self.assertEqual(status, '202 Accepted')
calls = self.app.calls_with_headers
self.assertEqual(2, len(calls))
method, path, req_headers = calls[1]
self.assertEqual('PUT', method)
# these headers should always be applied to the destination
self.assertEqual('added', req_headers.get('X-Object-Meta-Test2'))
self.assertEqual('added',
req_headers.get('X-Object-Transient-Sysmeta-Test2'))
# POSTed sysmeta should never be applied to the destination
self.assertNotIn('X-Object-Sysmeta-Test2', req_headers)
# existing sysmeta should always be preserved
self.assertEqual('original',
req_headers.get('X-Object-Sysmeta-Test'))
return req_headers
def test_POST_no_updates(self):
post_headers = {}
req_headers = self._test_POST_source_headers(post_headers)
self.assertEqual('text/original', req_headers.get('Content-Type'))
self.assertNotIn('X-Object-Meta-Test', req_headers)
self.assertNotIn('X-Object-Transient-Sysmeta-Test', req_headers)
self.assertNotIn('X-Timestamp', req_headers)
self.assertNotIn('X-Backend-Timestamp', req_headers)
self.assertNotIn('Content-Encoding', req_headers)
self.assertNotIn('Content-Disposition', req_headers)
self.assertNotIn('X-Foo', req_headers)
def test_POST_with_updates(self):
post_headers = {
'Content-Type': 'text/not_original',
'Content-Encoding': 'not_gzip',
'Content-Disposition': 'attachment; filename=notmyfile',
'X-Object-Meta-Test': 'not_original',
'X-Object-Sysmeta-Test': 'not_original',
'X-Object-Transient-Sysmeta-Test': 'not_original',
'X-Foo': 'Not Bar',
}
req_headers = self._test_POST_source_headers(post_headers)
self.assertEqual('text/not_original', req_headers.get('Content-Type'))
self.assertEqual('not_gzip', req_headers.get('Content-Encoding'))
self.assertEqual('attachment; filename=notmyfile',
req_headers.get('Content-Disposition'))
self.assertEqual('not_original', req_headers.get('X-Object-Meta-Test'))
self.assertEqual('not_original',
req_headers.get('X-Object-Transient-Sysmeta-Test'))
self.assertEqual('Not Bar', req_headers.get('X-Foo'))
def test_POST_x_fresh_metadata_with_updates(self):
# post-as-copy trumps x-fresh-metadata i.e. existing user metadata
# should not be copied, sysmeta is copied *and not updated with new*
post_headers = {
'X-Fresh-Metadata': 'true',
'Content-Type': 'text/not_original',
'Content-Encoding': 'not_gzip',
'Content-Disposition': 'attachment; filename=notmyfile',
'X-Object-Meta-Test': 'not_original',
'X-Object-Sysmeta-Test': 'not_original',
'X-Object-Transient-Sysmeta-Test': 'not_original',
'X-Foo': 'Not Bar',
}
req_headers = self._test_POST_source_headers(post_headers)
self.assertEqual('text/not_original', req_headers.get('Content-Type'))
self.assertEqual('not_gzip', req_headers.get('Content-Encoding'))
self.assertEqual('attachment; filename=notmyfile',
req_headers.get('Content-Disposition'))
self.assertEqual('not_original', req_headers.get('X-Object-Meta-Test'))
self.assertEqual('not_original',
req_headers.get('X-Object-Transient-Sysmeta-Test'))
self.assertEqual('Not Bar', req_headers.get('X-Foo'))
self.assertIn('X-Fresh-Metadata', req_headers)
def test_COPY_with_single_range(self): def test_COPY_with_single_range(self):
# verify that source etag is not copied when copying a range # verify that source etag is not copied when copying a range
self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, self.app.register('GET', '/v1/a/c/o', swob.HTTPOk,
@ -1472,67 +1289,6 @@ class TestServerSideCopyConfiguration(unittest.TestCase):
def tearDown(self): def tearDown(self):
shutil.rmtree(self.tmpdir) shutil.rmtree(self.tmpdir)
def test_post_as_copy_defaults_to_false(self):
ssc = copy.filter_factory({})("no app here")
self.assertEqual(ssc.object_post_as_copy, False)
def test_reading_proxy_conf_when_no_middleware_conf_present(self):
proxy_conf = dedent("""
[DEFAULT]
bind_ip = 10.4.5.6
[pipeline:main]
pipeline = catch_errors copy ye-olde-proxy-server
[filter:copy]
use = egg:swift#copy
[app:ye-olde-proxy-server]
use = egg:swift#proxy
object_post_as_copy = no
""")
conffile = tempfile.NamedTemporaryFile()
conffile.write(proxy_conf)
conffile.flush()
ssc = copy.filter_factory({
'__file__': conffile.name
})("no app here")
self.assertEqual(ssc.object_post_as_copy, False)
def test_middleware_conf_precedence(self):
proxy_conf = dedent("""
[DEFAULT]
bind_ip = 10.4.5.6
[pipeline:main]
pipeline = catch_errors copy ye-olde-proxy-server
[filter:copy]
use = egg:swift#copy
object_post_as_copy = no
[app:ye-olde-proxy-server]
use = egg:swift#proxy
object_post_as_copy = yes
""")
conffile = tempfile.NamedTemporaryFile()
conffile.write(proxy_conf)
conffile.flush()
with mock.patch('swift.common.middleware.copy.get_logger',
return_value=debug_logger('copy')):
ssc = copy.filter_factory({
'object_post_as_copy': 'no',
'__file__': conffile.name
})("no app here")
self.assertEqual(ssc.object_post_as_copy, False)
self.assertFalse(ssc.logger.get_lines_for_level('warning'))
def _test_post_as_copy_emits_warning(self, conf): def _test_post_as_copy_emits_warning(self, conf):
with mock.patch('swift.common.middleware.copy.get_logger', with mock.patch('swift.common.middleware.copy.get_logger',
return_value=debug_logger('copy')): return_value=debug_logger('copy')):
@ -1585,9 +1341,7 @@ class TestServerSideCopyMiddlewareWithEC(unittest.TestCase):
self.app = PatchedObjControllerApp( self.app = PatchedObjControllerApp(
None, FakeMemcache(), account_ring=FakeRing(), None, FakeMemcache(), account_ring=FakeRing(),
container_ring=FakeRing(), logger=self.logger) container_ring=FakeRing(), logger=self.logger)
self.ssc = copy.filter_factory({ self.ssc = copy.filter_factory({})(self.app)
'object_post_as_copy': 'yes',
})(self.app)
self.ssc.logger = self.app.logger self.ssc.logger = self.app.logger
self.policy = POLICIES.default self.policy = POLICIES.default
self.app.container_info = dict(self.container_info) self.app.container_info = dict(self.container_info)

View File

@ -117,23 +117,14 @@ class TestSubRequestLogging(unittest.TestCase):
self._test_subrequest_logged('PUT') self._test_subrequest_logged('PUT')
self._test_subrequest_logged('DELETE') self._test_subrequest_logged('DELETE')
def _test_subrequest_logged_POST(self, subrequest_type, def _test_subrequest_logged_POST(self, subrequest_type):
post_as_copy=False): app = FakeApp({'subrequest_type': subrequest_type})
# Test that subrequests made downstream from Copy POST will be logged
# with the request type of the subrequest as opposed to the GET/PUT.
app = FakeApp({'subrequest_type': subrequest_type,
'object_post_as_copy': post_as_copy})
hdrs = {'content-type': 'text/plain'} hdrs = {'content-type': 'text/plain'}
req = Request.blank(self.path, method='POST', headers=hdrs) req = Request.blank(self.path, method='POST', headers=hdrs)
app.register('POST', self.path, HTTPOk, headers=hdrs) app.register('POST', self.path, HTTPOk, headers=hdrs)
expect_lines = 2 expect_lines = 2
if post_as_copy:
app.register('PUT', self.path, HTTPOk, headers=hdrs)
app.register('GET', '/v1/a/c/o', HTTPOk, headers=hdrs)
expect_lines = 4
req.get_response(app) req.get_response(app)
info_log_lines = app.fake_logger.get_lines_for_level('info') info_log_lines = app.fake_logger.get_lines_for_level('info')
@ -142,33 +133,17 @@ class TestSubRequestLogging(unittest.TestCase):
subreq_put_post = '%s %s' % (subrequest_type, SUB_PUT_POST_PATH) subreq_put_post = '%s %s' % (subrequest_type, SUB_PUT_POST_PATH)
origpost = 'POST %s' % self.path origpost = 'POST %s' % self.path
copyget = 'GET %s' % self.path
if post_as_copy: # fast post expect POST subreq, original POST
# post_as_copy expect GET subreq, copy GET, PUT subreq, orig POST self.assertTrue(subreq_put_post in info_log_lines[0])
subreq_get = '%s %s' % (subrequest_type, SUB_GET_PATH) self.assertTrue(origpost in info_log_lines[1])
self.assertTrue(subreq_get in info_log_lines[0])
self.assertTrue(copyget in info_log_lines[1])
self.assertTrue(subreq_put_post in info_log_lines[2])
self.assertTrue(origpost in info_log_lines[3])
else:
# fast post expect POST subreq, original POST
self.assertTrue(subreq_put_post in info_log_lines[0])
self.assertTrue(origpost in info_log_lines[1])
def test_subrequest_logged_post_as_copy_with_POST_fast_post(self): def test_subrequest_logged_with_POST(self):
self._test_subrequest_logged_POST('HEAD', post_as_copy=False) self._test_subrequest_logged_POST('HEAD')
self._test_subrequest_logged_POST('GET', post_as_copy=False) self._test_subrequest_logged_POST('GET')
self._test_subrequest_logged_POST('POST', post_as_copy=False) self._test_subrequest_logged_POST('POST')
self._test_subrequest_logged_POST('PUT', post_as_copy=False) self._test_subrequest_logged_POST('PUT')
self._test_subrequest_logged_POST('DELETE', post_as_copy=False) self._test_subrequest_logged_POST('DELETE')
def test_subrequest_logged_post_as_copy_with_POST(self):
self._test_subrequest_logged_POST('HEAD', post_as_copy=True)
self._test_subrequest_logged_POST('GET', post_as_copy=True)
self._test_subrequest_logged_POST('POST', post_as_copy=True)
self._test_subrequest_logged_POST('PUT', post_as_copy=True)
self._test_subrequest_logged_POST('DELETE', post_as_copy=True)
if __name__ == '__main__': if __name__ == '__main__':

View File

@ -330,23 +330,6 @@ class VersionedWritesTestCase(VersionedWritesBaseTestCase):
self.assertEqual(len(self.authorized), 1) self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0]) self.assertRequestEqual(req, self.authorized[0])
def test_put_object_post_as_copy(self):
# PUTs due to a post-as-copy should NOT cause a versioning op
self.app.register(
'PUT', '/v1/a/c/o', swob.HTTPCreated, {}, 'passed')
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank(
'/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
'CONTENT_LENGTH': '100',
'swift.post_as_copy': True})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '201 Created')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
self.assertEqual(1, self.app.call_count)
def test_put_first_object_success(self): def test_put_first_object_success(self):
self.app.register( self.app.register(
'PUT', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') 'PUT', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')

View File

@ -3192,8 +3192,6 @@ class TestReplicatedObjectController(
def test_POST(self): def test_POST(self):
with save_globals(): with save_globals():
self.app.object_post_as_copy = False
def test_status_map(statuses, expected): def test_status_map(statuses, expected):
set_http_connect(*statuses) set_http_connect(*statuses)
self.app.memcache.store = {} self.app.memcache.store = {}
@ -3218,7 +3216,6 @@ class TestReplicatedObjectController(
def test_POST_backend_headers(self): def test_POST_backend_headers(self):
# reset the router post patch_policies # reset the router post patch_policies
self.app.obj_controller_router = proxy_server.ObjectControllerRouter() self.app.obj_controller_router = proxy_server.ObjectControllerRouter()
self.app.object_post_as_copy = False
self.app.sort_nodes = lambda nodes, *args, **kwargs: nodes self.app.sort_nodes = lambda nodes, *args, **kwargs: nodes
backend_requests = [] backend_requests = []
@ -3436,7 +3433,6 @@ class TestReplicatedObjectController(
def test_POST_meta_val_len(self): def test_POST_meta_val_len(self):
with save_globals(): with save_globals():
limit = constraints.MAX_META_VALUE_LENGTH limit = constraints.MAX_META_VALUE_LENGTH
self.app.object_post_as_copy = False
ReplicatedObjectController( ReplicatedObjectController(
self.app, 'account', 'container', 'object') self.app, 'account', 'container', 'object')
set_http_connect(200, 200, 202, 202, 202) set_http_connect(200, 200, 202, 202, 202)
@ -3462,7 +3458,6 @@ class TestReplicatedObjectController(
return return
with save_globals(): with save_globals():
limit = constraints.MAX_META_VALUE_LENGTH limit = constraints.MAX_META_VALUE_LENGTH
self.app.object_post_as_copy = False
controller = ReplicatedObjectController( controller = ReplicatedObjectController(
self.app, 'account', 'container', 'object') self.app, 'account', 'container', 'object')
set_http_connect(200, 200, 202, 202, 202) set_http_connect(200, 200, 202, 202, 202)
@ -3478,7 +3473,6 @@ class TestReplicatedObjectController(
def test_POST_meta_key_len(self): def test_POST_meta_key_len(self):
with save_globals(): with save_globals():
limit = constraints.MAX_META_NAME_LENGTH limit = constraints.MAX_META_NAME_LENGTH
self.app.object_post_as_copy = False
set_http_connect(200, 200, 202, 202, 202) set_http_connect(200, 200, 202, 202, 202)
# acct cont obj obj obj # acct cont obj obj obj
req = Request.blank( req = Request.blank(
@ -4224,7 +4218,6 @@ class TestReplicatedObjectController(
def test_PUT_POST_requires_container_exist(self): def test_PUT_POST_requires_container_exist(self):
with save_globals(): with save_globals():
self.app.object_post_as_copy = False
self.app.memcache = FakeMemcacheReturnsNone() self.app.memcache = FakeMemcacheReturnsNone()
controller = ReplicatedObjectController( controller = ReplicatedObjectController(
self.app, 'account', 'container', 'object') self.app, 'account', 'container', 'object')
@ -4837,7 +4830,6 @@ class TestReplicatedObjectController(
called[0] = True called[0] = True
return HTTPUnauthorized(request=req) return HTTPUnauthorized(request=req)
with save_globals(): with save_globals():
self.app.object_post_as_copy = False
set_http_connect(200, 200, 201, 201, 201) set_http_connect(200, 200, 201, 201, 201)
controller = ReplicatedObjectController( controller = ReplicatedObjectController(
self.app, 'account', 'container', 'object') self.app, 'account', 'container', 'object')
@ -4868,7 +4860,6 @@ class TestReplicatedObjectController(
def test_POST_converts_delete_after_to_delete_at(self): def test_POST_converts_delete_after_to_delete_at(self):
with save_globals(): with save_globals():
self.app.object_post_as_copy = False
controller = ReplicatedObjectController( controller = ReplicatedObjectController(
self.app, 'account', 'container', 'object') self.app, 'account', 'container', 'object')
set_http_connect(200, 200, 202, 202, 202) set_http_connect(200, 200, 202, 202, 202)
@ -5365,7 +5356,6 @@ class TestReplicatedObjectController(
def test_POST_x_container_headers_with_more_container_replicas(self): def test_POST_x_container_headers_with_more_container_replicas(self):
self.app.container_ring.set_replicas(4) self.app.container_ring.set_replicas(4)
self.app.object_post_as_copy = False
req = Request.blank('/v1/a/c/o', req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'POST'}, environ={'REQUEST_METHOD': 'POST'},
@ -6258,21 +6248,7 @@ class BaseTestECObjectController(BaseTestObjectController):
self.ec_policy.object_ring.replica_count) self.ec_policy.object_ring.replica_count)
if method == 'POST': if method == 'POST':
# Take care fast post here! headers = get_ring_reloaded_response(method)
orig_post_as_copy = getattr(
_test_servers[0], 'object_post_as_copy', None)
try:
_test_servers[0].object_post_as_copy = False
with mock.patch.object(
_test_servers[0],
'object_post_as_copy', False):
headers = get_ring_reloaded_response(method)
finally:
if orig_post_as_copy is None:
del _test_servers[0].object_post_as_copy
else:
_test_servers[0].object_post_as_copy = \
orig_post_as_copy
exp = 'HTTP/1.1 20' exp = 'HTTP/1.1 20'
self.assertEqual(headers[:len(exp)], exp) self.assertEqual(headers[:len(exp)], exp)

View File

@ -307,11 +307,6 @@ class TestObjectSysmeta(unittest.TestCase):
# test fast-post by issuing requests to the proxy app # test fast-post by issuing requests to the proxy app
self._test_sysmeta_not_updated_by_POST(self.app) self._test_sysmeta_not_updated_by_POST(self.app)
def test_sysmeta_not_updated_by_POST_as_copy(self):
# test post-as-copy by issuing requests to the copy middleware app
self.copy_app.object_post_as_copy = True
self._test_sysmeta_not_updated_by_POST(self.copy_app)
def test_sysmeta_updated_by_COPY(self): def test_sysmeta_updated_by_COPY(self):
# check sysmeta is updated by a COPY in same way as user meta by # check sysmeta is updated by a COPY in same way as user meta by
# issuing requests to the copy middleware app # issuing requests to the copy middleware app
@ -482,8 +477,3 @@ class TestObjectSysmeta(unittest.TestCase):
def test_transient_sysmeta_replaced_by_PUT_or_POST(self): def test_transient_sysmeta_replaced_by_PUT_or_POST(self):
self._test_transient_sysmeta_replaced_by_PUT_or_POST(self.app) self._test_transient_sysmeta_replaced_by_PUT_or_POST(self.app)
def test_transient_sysmeta_replaced_by_PUT_or_POST_as_copy(self):
# test post-as-copy by issuing requests to the copy middleware app
self.copy_app.object_post_as_copy = True
self._test_transient_sysmeta_replaced_by_PUT_or_POST(self.copy_app)