From 035a411660ca02983cd486312266c67d78a2359c Mon Sep 17 00:00:00 2001 From: Thiago da Silva Date: Sun, 9 Nov 2014 13:13:27 -0500 Subject: [PATCH] versioned writes middleware Rewrite object versioning as middleware to simplify the PUT method in the object controller. The functionality remains basically the same with the only major difference being the ability to now version slo manifest files. dlo manifests are still not supported as part of this patch. Co-Authored-By: Clay Gerrard DocImpact Change-Id: Ie899290b3312e201979eafefb253d1a60b65b837 Signed-off-by: Thiago da Silva Signed-off-by: Prashanth Pai --- doc/saio/swift/container-server/1.conf | 1 - doc/saio/swift/container-server/2.conf | 1 - doc/saio/swift/container-server/3.conf | 1 - doc/saio/swift/container-server/4.conf | 1 - doc/saio/swift/proxy-server.conf | 6 +- doc/source/logs.rst | 1 + doc/source/middleware.rst | 9 + doc/source/overview_object_versioning.rst | 89 +- etc/proxy-server.conf-sample | 13 +- setup.cfg | 1 + swift/common/constraints.py | 30 +- swift/common/middleware/versioned_writes.py | 490 +++++++ swift/proxy/controllers/obj.py | 191 +-- swift/proxy/server.py | 3 + test/functional/swift_test_client.py | 22 +- test/functional/tests.py | 213 +++- test/unit/common/middleware/helpers.py | 2 +- test/unit/common/middleware/test_dlo.py | 2 +- .../middleware/test_versioned_writes.py | 558 ++++++++ test/unit/common/test_constraints.py | 18 + test/unit/common/test_wsgi.py | 12 +- test/unit/proxy/test_server.py | 1132 +++++++---------- 22 files changed, 1816 insertions(+), 980 deletions(-) create mode 100644 swift/common/middleware/versioned_writes.py create mode 100644 test/unit/common/middleware/test_versioned_writes.py diff --git a/doc/saio/swift/container-server/1.conf b/doc/saio/swift/container-server/1.conf index 3062ca3a5a..176096dbe1 100644 --- a/doc/saio/swift/container-server/1.conf +++ b/doc/saio/swift/container-server/1.conf @@ -9,7 +9,6 @@ user = log_facility = LOG_LOCAL2 recon_cache_path = /var/cache/swift eventlet_debug = true -allow_versions = true [pipeline:main] pipeline = recon container-server diff --git a/doc/saio/swift/container-server/2.conf b/doc/saio/swift/container-server/2.conf index 6365215931..7100710b3c 100644 --- a/doc/saio/swift/container-server/2.conf +++ b/doc/saio/swift/container-server/2.conf @@ -9,7 +9,6 @@ user = log_facility = LOG_LOCAL3 recon_cache_path = /var/cache/swift2 eventlet_debug = true -allow_versions = true [pipeline:main] pipeline = recon container-server diff --git a/doc/saio/swift/container-server/3.conf b/doc/saio/swift/container-server/3.conf index b925427ff0..06ec47414d 100644 --- a/doc/saio/swift/container-server/3.conf +++ b/doc/saio/swift/container-server/3.conf @@ -9,7 +9,6 @@ user = log_facility = LOG_LOCAL4 recon_cache_path = /var/cache/swift3 eventlet_debug = true -allow_versions = true [pipeline:main] pipeline = recon container-server diff --git a/doc/saio/swift/container-server/4.conf b/doc/saio/swift/container-server/4.conf index 16799a524a..1acc3b5c54 100644 --- a/doc/saio/swift/container-server/4.conf +++ b/doc/saio/swift/container-server/4.conf @@ -9,7 +9,6 @@ user = log_facility = LOG_LOCAL5 recon_cache_path = /var/cache/swift4 eventlet_debug = true -allow_versions = true [pipeline:main] pipeline = recon container-server diff --git a/doc/saio/swift/proxy-server.conf b/doc/saio/swift/proxy-server.conf index dd037edb8f..c25e0ed90d 100644 --- a/doc/saio/swift/proxy-server.conf +++ b/doc/saio/swift/proxy-server.conf @@ -9,7 +9,7 @@ eventlet_debug = true [pipeline:main] # Yes, proxy-logging appears twice. This is so that # middleware-originated requests get logged too. -pipeline = catch_errors gatekeeper healthcheck proxy-logging cache bulk tempurl ratelimit crossdomain tempauth staticweb container-quotas account-quotas slo dlo proxy-logging proxy-server +pipeline = catch_errors gatekeeper healthcheck proxy-logging cache bulk tempurl ratelimit crossdomain tempauth staticweb container-quotas account-quotas slo dlo versioned_writes proxy-logging proxy-server [filter:catch_errors] use = egg:swift#catch_errors @@ -60,6 +60,10 @@ use = egg:swift#memcache [filter:gatekeeper] use = egg:swift#gatekeeper +[filter:versioned_writes] +use = egg:swift#versioned_writes +allow_versioned_writes = true + [app:proxy-server] use = egg:swift#proxy allow_account_management = true diff --git a/doc/source/logs.rst b/doc/source/logs.rst index f738861843..75b669f1a5 100644 --- a/doc/source/logs.rst +++ b/doc/source/logs.rst @@ -102,6 +102,7 @@ DLO :ref:`dynamic-large-objects` LE :ref:`list_endpoints` KS :ref:`keystoneauth` RL :ref:`ratelimit` +VW :ref:`versioned_writes` ======================= ============================= diff --git a/doc/source/middleware.rst b/doc/source/middleware.rst index f78dbb1947..4e304ed6fb 100644 --- a/doc/source/middleware.rst +++ b/doc/source/middleware.rst @@ -155,6 +155,15 @@ Name Check (Forbidden Character Filter) :members: :show-inheritance: +.. _versioned_writes: + +Object Versioning +================= + +.. automodule:: swift.common.middleware.versioned_writes + :members: + :show-inheritance: + Proxy Logging ============= diff --git a/doc/source/overview_object_versioning.rst b/doc/source/overview_object_versioning.rst index cac5a898d9..78d0b07ad1 100644 --- a/doc/source/overview_object_versioning.rst +++ b/doc/source/overview_object_versioning.rst @@ -1,89 +1,6 @@ -================= Object Versioning ================= --------- -Overview --------- - -Object versioning in swift is implemented by setting a flag on the container -to tell swift to version all objects in the container. The flag is the -``X-Versions-Location`` header on the container, and its value is the -container where the versions are stored. It is recommended to use a different -``X-Versions-Location`` container for each container that is being versioned. - -When data is ``PUT`` into a versioned container (a container with the -versioning flag turned on), the existing data in the file is redirected to a -new object and the data in the ``PUT`` request is saved as the data for the -versioned object. The new object name (for the previous version) is -``//``, where ``length`` -is the 3-character zero-padded hexadecimal length of the ```` and -```` is the timestamp of when the previous version was created. - -A ``GET`` to a versioned object will return the current version of the object -without having to do any request redirects or metadata lookups. - -A ``POST`` to a versioned object will update the object metadata as normal, -but will not create a new version of the object. In other words, new versions -are only created when the content of the object changes. - -A ``DELETE`` to a versioned object will only remove the current version of the -object. If you have 5 total versions of the object, you must delete the -object 5 times to completely remove the object. - -Note: A large object manifest file cannot be versioned, but a large object -manifest may point to versioned segments. - --------------------------------------------------- -How to Enable Object Versioning in a Swift Cluster --------------------------------------------------- - -Set ``allow_versions`` to ``True`` in the container server config. - ------------------------ -Examples Using ``curl`` ------------------------ - -First, create a container with the ``X-Versions-Location`` header or add the -header to an existing container. Also make sure the container referenced by -the ``X-Versions-Location`` exists. In this example, the name of that -container is "versions":: - - curl -i -XPUT -H "X-Auth-Token: " \ - -H "X-Versions-Location: versions" http:///container - curl -i -XPUT -H "X-Auth-Token: " http:///versions - -Create an object (the first version):: - - curl -i -XPUT --data-binary 1 -H "X-Auth-Token: " \ - http:///container/myobject - -Now create a new version of that object:: - - curl -i -XPUT --data-binary 2 -H "X-Auth-Token: " \ - http:///container/myobject - -See a listing of the older versions of the object:: - - curl -i -H "X-Auth-Token: " \ - http:///versions?prefix=008myobject/ - -Now delete the current version of the object and see that the older version is -gone:: - - curl -i -XDELETE -H "X-Auth-Token: " \ - http:///container/myobject - curl -i -H "X-Auth-Token: " \ - http:///versions?prefix=008myobject/ - ---------------------------------------------------- -How to Disable Object Versioning in a Swift Cluster ---------------------------------------------------- - -If you want to disable all functionality, set ``allow_versions`` back to -``False`` in the container server config. - -Disable versioning a versioned container (x is any value except empty):: - - curl -i -XPOST -H "X-Auth-Token: " \ - -H "X-Remove-Versions-Location: x" http:///container +.. automodule:: swift.common.middleware.versioned_writes + :members: + :show-inheritance: diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index 55b6137ae0..b37101c37a 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -77,7 +77,7 @@ bind_port = 8080 # eventlet_debug = false [pipeline:main] -pipeline = catch_errors gatekeeper healthcheck proxy-logging cache container_sync bulk tempurl ratelimit tempauth container-quotas account-quotas slo dlo proxy-logging proxy-server +pipeline = catch_errors gatekeeper healthcheck proxy-logging cache container_sync bulk tempurl ratelimit tempauth container-quotas account-quotas slo dlo versioned_writes proxy-logging proxy-server [app:proxy-server] use = egg:swift#proxy @@ -703,3 +703,14 @@ use = egg:swift#xprofile # # unwind the iterator of applications # unwind = false + +# Note: Put after slo, dlo in the pipeline. +# If you don't put it in the pipeline, it will be inserted automatically. +[filter:versioned_writes] +use = egg:swift#versioned_writes +# Enables using versioned writes middleware and exposing configuration +# settings via HTTP GET /info. +# WARNING: Setting this option bypasses the "allow_versions" option +# in the container configuration file, which will be eventually +# deprecated. See documentation for more details. +# allow_versioned_writes = false diff --git a/setup.cfg b/setup.cfg index a40fc535ee..a819a57f02 100644 --- a/setup.cfg +++ b/setup.cfg @@ -95,6 +95,7 @@ paste.filter_factory = gatekeeper = swift.common.middleware.gatekeeper:filter_factory container_sync = swift.common.middleware.container_sync:filter_factory xprofile = swift.common.middleware.xprofile:filter_factory + versioned_writes = swift.common.middleware.versioned_writes:filter_factory [build_sphinx] all_files = 1 diff --git a/swift/common/constraints.py b/swift/common/constraints.py index aae5f25aac..36f9d5eae8 100644 --- a/swift/common/constraints.py +++ b/swift/common/constraints.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import functools import os import urllib import time @@ -406,28 +407,33 @@ def check_destination_header(req): '/') -def check_account_format(req, account): +def check_name_format(req, name, target_type): """ - Validate that the header contains valid account name. - We assume the caller ensures that - destination header is present in req.headers. + Validate that the header contains valid account or container name. :param req: HTTP request object - :returns: A properly encoded account name + :param name: header value to validate + :param target_type: which header is being validated (Account or Container) + :returns: A properly encoded account name or container name :raise: HTTPPreconditionFailed if account header is not well formatted. """ - if not account: + if not name: raise HTTPPreconditionFailed( request=req, - body='Account name cannot be empty') - if isinstance(account, unicode): - account = account.encode('utf-8') - if '/' in account: + body='%s name cannot be empty' % target_type) + if isinstance(name, unicode): + name = name.encode('utf-8') + if '/' in name: raise HTTPPreconditionFailed( request=req, - body='Account name cannot contain slashes') - return account + body='%s name cannot contain slashes' % target_type) + return name + +check_account_format = functools.partial(check_name_format, + target_type='Account') +check_container_format = functools.partial(check_name_format, + target_type='Container') def valid_api_version(version): diff --git a/swift/common/middleware/versioned_writes.py b/swift/common/middleware/versioned_writes.py new file mode 100644 index 0000000000..e3f56f6fd1 --- /dev/null +++ b/swift/common/middleware/versioned_writes.py @@ -0,0 +1,490 @@ +# Copyright (c) 2014 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Object versioning in swift is implemented by setting a flag on the container +to tell swift to version all objects in the container. The flag is the +``X-Versions-Location`` header on the container, and its value is the +container where the versions are stored. It is recommended to use a different +``X-Versions-Location`` container for each container that is being versioned. + +When data is ``PUT`` into a versioned container (a container with the +versioning flag turned on), the existing data in the file is redirected to a +new object and the data in the ``PUT`` request is saved as the data for the +versioned object. The new object name (for the previous version) is +``//``, where ``length`` +is the 3-character zero-padded hexadecimal length of the ```` and +```` is the timestamp of when the previous version was created. + +A ``GET`` to a versioned object will return the current version of the object +without having to do any request redirects or metadata lookups. + +A ``POST`` to a versioned object will update the object metadata as normal, +but will not create a new version of the object. In other words, new versions +are only created when the content of the object changes. + +A ``DELETE`` to a versioned object will only remove the current version of the +object. If you have 5 total versions of the object, you must delete the +object 5 times to completely remove the object. + +-------------------------------------------------- +How to Enable Object Versioning in a Swift Cluster +-------------------------------------------------- + +This middleware was written as an effort to refactor parts of the proxy server, +so this functionality was already available in previous releases and every +attempt was made to maintain backwards compatibility. To allow operators to +perform a seamless upgrade, it is not required to add the middleware to the +proxy pipeline and the flag ``allow_versions`` in the container server +configuration files are still valid. In future releases, ``allow_versions`` +will be deprecated in favor of adding this middleware to the pipeline to enable +or disable the feature. + +In case the middleware is added to the proxy pipeline, you must also +set ``allow_versioned_writes`` to ``True`` in the middleware options +to enable the information about this middleware to be returned in a /info +request. + +Upgrade considerations: If ``allow_versioned_writes`` is set in the filter +configuration, you can leave the ``allow_versions`` flag in the container +server configuration files untouched. If you decide to disable or remove the +``allow_versions`` flag, you must re-set any existing containers that had +the 'X-Versions-Location' flag configured so that it can now be tracked by the +versioned_writes middleware. + +----------------------- +Examples Using ``curl`` +----------------------- + +First, create a container with the ``X-Versions-Location`` header or add the +header to an existing container. Also make sure the container referenced by +the ``X-Versions-Location`` exists. In this example, the name of that +container is "versions":: + + curl -i -XPUT -H "X-Auth-Token: " \ +-H "X-Versions-Location: versions" http:///container + curl -i -XPUT -H "X-Auth-Token: " http:///versions + +Create an object (the first version):: + + curl -i -XPUT --data-binary 1 -H "X-Auth-Token: " \ +http:///container/myobject + +Now create a new version of that object:: + + curl -i -XPUT --data-binary 2 -H "X-Auth-Token: " \ +http:///container/myobject + +See a listing of the older versions of the object:: + + curl -i -H "X-Auth-Token: " \ +http:///versions?prefix=008myobject/ + +Now delete the current version of the object and see that the older version is +gone:: + + curl -i -XDELETE -H "X-Auth-Token: " \ +http:///container/myobject + curl -i -H "X-Auth-Token: " \ +http:///versions?prefix=008myobject/ + +--------------------------------------------------- +How to Disable Object Versioning in a Swift Cluster +--------------------------------------------------- + +If you want to disable all functionality, set ``allow_versioned_writes`` to +``False`` in the middleware options. + +Disable versioning from a container (x is any value except empty):: + + curl -i -XPOST -H "X-Auth-Token: " \ +-H "X-Remove-Versions-Location: x" http:///container +""" + +import time +from urllib import quote, unquote +from swift.common.utils import get_logger, Timestamp, json, \ + register_swift_info, config_true_value +from swift.common.request_helpers import get_sys_meta_prefix +from swift.common.wsgi import WSGIContext, make_pre_authed_request +from swift.common.swob import Request +from swift.common.constraints import ( + check_account_format, check_container_format, check_destination_header) +from swift.proxy.controllers.base import get_container_info +from swift.common.http import ( + is_success, is_client_error, HTTP_NOT_FOUND) +from swift.common.swob import HTTPPreconditionFailed, HTTPServiceUnavailable, \ + HTTPServerError +from swift.common.exceptions import ( + ListingIterNotFound, ListingIterError) + + +class VersionedWritesContext(WSGIContext): + + def __init__(self, wsgi_app, logger): + WSGIContext.__init__(self, wsgi_app) + self.logger = logger + + def _listing_iter(self, account_name, lcontainer, lprefix, env): + for page in self._listing_pages_iter(account_name, + lcontainer, lprefix, env): + for item in page: + yield item + + def _listing_pages_iter(self, account_name, lcontainer, lprefix, env): + marker = '' + while True: + lreq = make_pre_authed_request( + env, method='GET', swift_source='VW', + path='/v1/%s/%s' % (account_name, lcontainer)) + lreq.environ['QUERY_STRING'] = \ + 'format=json&prefix=%s&marker=%s' % (quote(lprefix), + quote(marker)) + lresp = lreq.get_response(self.app) + if not is_success(lresp.status_int): + if lresp.status_int == HTTP_NOT_FOUND: + raise ListingIterNotFound() + elif is_client_error(lresp.status_int): + raise HTTPPreconditionFailed() + else: + raise ListingIterError() + + if not lresp.body: + break + + sublisting = json.loads(lresp.body) + if not sublisting: + break + marker = sublisting[-1]['name'].encode('utf-8') + yield sublisting + + def handle_obj_versions_put(self, req, object_versions, + object_name, policy_index): + ret = None + + # do a HEAD request to check object versions + _headers = {'X-Newest': 'True', + 'X-Backend-Storage-Policy-Index': policy_index, + 'x-auth-token': req.headers.get('x-auth-token')} + + # make a pre_auth request in case the user has write access + # to container, but not READ. This was allowed in previous version + # (i.e., before middleware) so keeping the same behavior here + head_req = make_pre_authed_request( + req.environ, path=req.path_info, + headers=_headers, method='HEAD', swift_source='VW') + hresp = head_req.get_response(self.app) + + is_dlo_manifest = 'X-Object-Manifest' in req.headers or \ + 'X-Object-Manifest' in hresp.headers + + # if there's an existing object, then copy it to + # X-Versions-Location + if is_success(hresp.status_int) and not is_dlo_manifest: + lcontainer = object_versions.split('/')[0] + prefix_len = '%03x' % len(object_name) + lprefix = prefix_len + object_name + '/' + ts_source = hresp.environ.get('swift_x_timestamp') + if ts_source is None: + ts_source = time.mktime(time.strptime( + hresp.headers['last-modified'], + '%a, %d %b %Y %H:%M:%S GMT')) + new_ts = Timestamp(ts_source).internal + vers_obj_name = lprefix + new_ts + copy_headers = { + 'Destination': '%s/%s' % (lcontainer, vers_obj_name), + 'x-auth-token': req.headers.get('x-auth-token')} + + # COPY implementation sets X-Newest to True when it internally + # does a GET on source object. So, we don't have to explicity + # set it in request headers here. + copy_req = make_pre_authed_request( + req.environ, path=req.path_info, + headers=copy_headers, method='COPY', swift_source='VW') + copy_resp = copy_req.get_response(self.app) + + if is_success(copy_resp.status_int): + # success versioning previous existing object + # return None and handle original request + ret = None + else: + if is_client_error(copy_resp.status_int): + # missing container or bad permissions + ret = HTTPPreconditionFailed(request=req) + else: + # could not copy the data, bail + ret = HTTPServiceUnavailable(request=req) + + else: + if hresp.status_int == HTTP_NOT_FOUND or is_dlo_manifest: + # nothing to version + # return None and handle original request + ret = None + else: + # if not HTTP_NOT_FOUND, return error immediately + ret = hresp + + return ret + + def handle_obj_versions_delete(self, req, object_versions, + account_name, container_name, object_name): + lcontainer = object_versions.split('/')[0] + prefix_len = '%03x' % len(object_name) + lprefix = prefix_len + object_name + '/' + item_list = [] + try: + for _item in self._listing_iter(account_name, lcontainer, lprefix, + req.environ): + item_list.append(_item) + except ListingIterNotFound: + pass + except HTTPPreconditionFailed: + return HTTPPreconditionFailed(request=req) + except ListingIterError: + return HTTPServerError(request=req) + + if item_list: + # we're about to start making COPY requests - need to validate the + # write access to the versioned container + if 'swift.authorize' in req.environ: + container_info = get_container_info( + req.environ, self.app) + req.acl = container_info.get('write_acl') + aresp = req.environ['swift.authorize'](req) + if aresp: + return aresp + + while len(item_list) > 0: + previous_version = item_list.pop() + + # there are older versions so copy the previous version to the + # current object and delete the previous version + prev_obj_name = previous_version['name'].encode('utf-8') + + copy_path = '/v1/' + account_name + '/' + \ + lcontainer + '/' + prev_obj_name + + copy_headers = {'X-Newest': 'True', + 'Destination': container_name + '/' + object_name, + 'x-auth-token': req.headers.get('x-auth-token')} + + copy_req = make_pre_authed_request( + req.environ, path=copy_path, + headers=copy_headers, method='COPY', swift_source='VW') + copy_resp = copy_req.get_response(self.app) + + # if the version isn't there, keep trying with previous version + if copy_resp.status_int == HTTP_NOT_FOUND: + continue + + if not is_success(copy_resp.status_int): + if is_client_error(copy_resp.status_int): + # some user error, maybe permissions + return HTTPPreconditionFailed(request=req) + else: + # could not copy the data, bail + return HTTPServiceUnavailable(request=req) + + # reset these because the COPY changed them + new_del_req = make_pre_authed_request( + req.environ, path=copy_path, method='DELETE', + swift_source='VW') + req = new_del_req + + # remove 'X-If-Delete-At', since it is not for the older copy + if 'X-If-Delete-At' in req.headers: + del req.headers['X-If-Delete-At'] + break + + # handle DELETE request here in case it was modified + return req.get_response(self.app) + + def handle_container_request(self, env, start_response): + app_resp = self._app_call(env) + if self._response_headers is None: + self._response_headers = [] + sysmeta_version_hdr = get_sys_meta_prefix('container') + \ + 'versions-location' + location = '' + for key, val in self._response_headers: + if key.lower() == sysmeta_version_hdr: + location = val + + if location: + self._response_headers.extend([('X-Versions-Location', location)]) + + start_response(self._response_status, + self._response_headers, + self._response_exc_info) + return app_resp + + +class VersionedWritesMiddleware(object): + + def __init__(self, app, conf): + self.app = app + self.conf = conf + self.logger = get_logger(conf, log_route='versioned_writes') + + def container_request(self, req, start_response, enabled): + sysmeta_version_hdr = get_sys_meta_prefix('container') + \ + 'versions-location' + + # set version location header as sysmeta + if 'X-Versions-Location' in req.headers: + val = req.headers.get('X-Versions-Location') + if val: + # diferently from previous version, we are actually + # returning an error if user tries to set versions location + # while feature is explicitly disabled. + if not config_true_value(enabled) and \ + req.method in ('PUT', 'POST'): + raise HTTPPreconditionFailed( + request=req, content_type='text/plain', + body='Versioned Writes is disabled') + + location = check_container_format(req, val) + req.headers[sysmeta_version_hdr] = location + + # reset original header to maintain sanity + # now only sysmeta is source of Versions Location + req.headers['X-Versions-Location'] = '' + + # if both headers are in the same request + # adding location takes precendence over removing + if 'X-Remove-Versions-Location' in req.headers: + del req.headers['X-Remove-Versions-Location'] + else: + # empty value is the same as X-Remove-Versions-Location + req.headers['X-Remove-Versions-Location'] = 'x' + + # handle removing versions container + val = req.headers.get('X-Remove-Versions-Location') + if val: + req.headers.update({sysmeta_version_hdr: ''}) + req.headers.update({'X-Versions-Location': ''}) + del req.headers['X-Remove-Versions-Location'] + + # send request and translate sysmeta headers from response + vw_ctx = VersionedWritesContext(self.app, self.logger) + return vw_ctx.handle_container_request(req.environ, start_response) + + def object_request(self, req, version, account, container, obj, + allow_versioned_writes): + account_name = unquote(account) + container_name = unquote(container) + object_name = unquote(obj) + container_info = None + resp = None + is_enabled = config_true_value(allow_versioned_writes) + if req.method in ('PUT', 'DELETE'): + container_info = get_container_info( + req.environ, self.app) + elif req.method == 'COPY' and 'Destination' in req.headers: + if 'Destination-Account' in req.headers: + account_name = req.headers.get('Destination-Account') + account_name = check_account_format(req, account_name) + container_name, object_name = check_destination_header(req) + req.environ['PATH_INFO'] = "/%s/%s/%s/%s" % ( + version, account_name, container_name, object_name) + container_info = get_container_info( + req.environ, self.app) + + if not container_info: + return self.app + + # To maintain backwards compatibility, container version + # location could be stored as sysmeta or not, need to check both. + # If stored as sysmeta, check if middleware is enabled. If sysmeta + # is not set, but versions property is set in container_info, then + # for backwards compatibility feature is enabled. + object_versions = container_info.get( + 'sysmeta', {}).get('versions-location') + if object_versions and isinstance(object_versions, unicode): + object_versions = object_versions.encode('utf-8') + elif not object_versions: + object_versions = container_info.get('versions') + # if allow_versioned_writes is not set in the configuration files + # but 'versions' is configured, enable feature to maintain + # backwards compatibility + if not allow_versioned_writes and object_versions: + is_enabled = True + + if is_enabled and object_versions: + object_versions = unquote(object_versions) + vw_ctx = VersionedWritesContext(self.app, self.logger) + if req.method in ('PUT', 'COPY'): + policy_idx = req.headers.get( + 'X-Backend-Storage-Policy-Index', + container_info['storage_policy']) + resp = vw_ctx.handle_obj_versions_put( + req, object_versions, object_name, policy_idx) + else: # handle DELETE + resp = vw_ctx.handle_obj_versions_delete( + req, object_versions, account_name, + container_name, object_name) + + if resp: + return resp + else: + return self.app + + def __call__(self, env, start_response): + # making a duplicate, because if this is a COPY request, we will + # modify the PATH_INFO to find out if the 'Destination' is in a + # versioned container + req = Request(env.copy()) + try: + (version, account, container, obj) = req.split_path(3, 4, True) + except ValueError: + return self.app(env, start_response) + + # In case allow_versioned_writes is set in the filter configuration, + # the middleware becomes the authority on whether object + # versioning is enabled or not. In case it is not set, then + # the option in the container configuration is still checked + # for backwards compatibility + + # For a container request, first just check if option is set, + # can be either true or false. + # If set, check if enabled when actually trying to set container + # header. If not set, let request be handled by container server + # for backwards compatibility. + # For an object request, also check if option is set (either T or F). + # If set, check if enabled when checking versions container in + # sysmeta property. If it is not set check 'versions' property in + # container_info + allow_versioned_writes = self.conf.get('allow_versioned_writes') + if allow_versioned_writes and container and not obj: + return self.container_request(req, start_response, + allow_versioned_writes) + elif obj and req.method in ('PUT', 'COPY', 'DELETE'): + return self.object_request( + req, version, account, container, obj, + allow_versioned_writes)(env, start_response) + else: + return self.app(env, start_response) + + +def filter_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + if config_true_value(conf.get('allow_versioned_writes')): + register_swift_info('versioned_writes') + + def obj_versions_filter(app): + return VersionedWritesMiddleware(app, conf) + + return obj_versions_filter diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py index e86b35debe..78af923124 100644 --- a/swift/proxy/controllers/obj.py +++ b/swift/proxy/controllers/obj.py @@ -51,13 +51,12 @@ from swift.common.constraints import check_metadata, check_object_creation, \ check_account_format from swift.common import constraints from swift.common.exceptions import ChunkReadTimeout, \ - ChunkWriteTimeout, ConnectionTimeout, ListingIterNotFound, \ - ListingIterNotAuthorized, ListingIterError, ResponseTimeout, \ + ChunkWriteTimeout, ConnectionTimeout, ResponseTimeout, \ InsufficientStorage, FooterNotSupported, MultiphasePUTNotSupported, \ PutterConnectError from swift.common.http import ( - is_success, is_client_error, is_server_error, HTTP_CONTINUE, HTTP_CREATED, - HTTP_MULTIPLE_CHOICES, HTTP_NOT_FOUND, HTTP_INTERNAL_SERVER_ERROR, + is_success, is_server_error, HTTP_CONTINUE, HTTP_CREATED, + HTTP_MULTIPLE_CHOICES, HTTP_INTERNAL_SERVER_ERROR, HTTP_SERVICE_UNAVAILABLE, HTTP_INSUFFICIENT_STORAGE, HTTP_PRECONDITION_FAILED, HTTP_CONFLICT, is_informational) from swift.common.storage_policy import (POLICIES, REPL_POLICY, EC_POLICY, @@ -139,46 +138,6 @@ class BaseObjectController(Controller): self.container_name = unquote(container_name) self.object_name = unquote(object_name) - def _listing_iter(self, lcontainer, lprefix, env): - for page in self._listing_pages_iter(lcontainer, lprefix, env): - for item in page: - yield item - - def _listing_pages_iter(self, lcontainer, lprefix, env): - lpartition = self.app.container_ring.get_part( - self.account_name, lcontainer) - marker = '' - while True: - lreq = Request.blank('i will be overridden by env', environ=env) - # Don't quote PATH_INFO, by WSGI spec - lreq.environ['PATH_INFO'] = \ - '/v1/%s/%s' % (self.account_name, lcontainer) - lreq.environ['REQUEST_METHOD'] = 'GET' - lreq.environ['QUERY_STRING'] = \ - 'format=json&prefix=%s&marker=%s' % (quote(lprefix), - quote(marker)) - container_node_iter = self.app.iter_nodes(self.app.container_ring, - lpartition) - lresp = self.GETorHEAD_base( - lreq, _('Container'), container_node_iter, lpartition, - lreq.swift_entity_path) - if 'swift.authorize' in env: - lreq.acl = lresp.headers.get('x-container-read') - aresp = env['swift.authorize'](lreq) - if aresp: - raise ListingIterNotAuthorized(aresp) - if lresp.status_int == HTTP_NOT_FOUND: - raise ListingIterNotFound() - elif not is_success(lresp.status_int): - raise ListingIterError() - if not lresp.body: - break - sublisting = json.loads(lresp.body) - if not sublisting: - break - marker = sublisting[-1]['name'].encode('utf-8') - yield sublisting - def iter_nodes_local_first(self, ring, partition): """ Yields nodes for a ring partition. @@ -548,71 +507,6 @@ class BaseObjectController(Controller): # until copy request handling moves to middleware return None, req, data_source, update_response - def _handle_object_versions(self, req): - """ - This method handles versionining of objects in containers that - have the feature enabled. - - When a new PUT request is sent, the proxy checks for previous versions - of that same object name. If found, it is copied to a different - container and the new version is stored in its place. - - This method was added as part of the PUT method refactoring and the - functionality is expected to be moved to middleware - """ - container_info = self.container_info( - self.account_name, self.container_name, req) - policy_index = req.headers.get('X-Backend-Storage-Policy-Index', - container_info['storage_policy']) - obj_ring = self.app.get_object_ring(policy_index) - partition, nodes = obj_ring.get_nodes( - self.account_name, self.container_name, self.object_name) - object_versions = container_info['versions'] - - # do a HEAD request for checking object versions - if object_versions and not req.environ.get('swift_versioned_copy'): - # make sure proxy-server uses the right policy index - _headers = {'X-Backend-Storage-Policy-Index': policy_index, - 'X-Newest': 'True'} - hreq = Request.blank(req.path_info, headers=_headers, - environ={'REQUEST_METHOD': 'HEAD'}) - hnode_iter = self.app.iter_nodes(obj_ring, partition) - hresp = self.GETorHEAD_base( - hreq, _('Object'), hnode_iter, partition, - hreq.swift_entity_path) - - is_manifest = 'X-Object-Manifest' in req.headers or \ - 'X-Object-Manifest' in hresp.headers - if hresp.status_int != HTTP_NOT_FOUND and not is_manifest: - # This is a version manifest and needs to be handled - # differently. First copy the existing data to a new object, - # then write the data from this request to the version manifest - # object. - lcontainer = object_versions.split('/')[0] - prefix_len = '%03x' % len(self.object_name) - lprefix = prefix_len + self.object_name + '/' - ts_source = hresp.environ.get('swift_x_timestamp') - if ts_source is None: - ts_source = time.mktime(time.strptime( - hresp.headers['last-modified'], - '%a, %d %b %Y %H:%M:%S GMT')) - new_ts = Timestamp(ts_source).internal - vers_obj_name = lprefix + new_ts - copy_headers = { - 'Destination': '%s/%s' % (lcontainer, vers_obj_name)} - copy_environ = {'REQUEST_METHOD': 'COPY', - 'swift_versioned_copy': True - } - copy_req = Request.blank(req.path_info, headers=copy_headers, - environ=copy_environ) - copy_resp = self.COPY(copy_req) - if is_client_error(copy_resp.status_int): - # missing container or bad permissions - raise HTTPPreconditionFailed(request=req) - elif not is_success(copy_resp.status_int): - # could not copy the data, bail - raise HTTPServiceUnavailable(request=req) - def _update_content_type(self, req): # Sometimes the 'content-type' header exists, but is set to None. req.content_type_manually_set = True @@ -819,9 +713,6 @@ class BaseObjectController(Controller): self._update_x_timestamp(req) - # check if versioning is enabled and handle copying previous version - self._handle_object_versions(req) - # check if request is a COPY of an existing object source_header = req.headers.get('X-Copy-From') if source_header: @@ -865,86 +756,10 @@ class BaseObjectController(Controller): containers = container_info['nodes'] req.acl = container_info['write_acl'] req.environ['swift_sync_key'] = container_info['sync_key'] - object_versions = container_info['versions'] if 'swift.authorize' in req.environ: aresp = req.environ['swift.authorize'](req) if aresp: return aresp - if object_versions: - # this is a version manifest and needs to be handled differently - object_versions = unquote(object_versions) - lcontainer = object_versions.split('/')[0] - prefix_len = '%03x' % len(self.object_name) - lprefix = prefix_len + self.object_name + '/' - item_list = [] - try: - for _item in self._listing_iter(lcontainer, lprefix, - req.environ): - item_list.append(_item) - except ListingIterNotFound: - # no worries, last_item is None - pass - except ListingIterNotAuthorized as err: - return err.aresp - except ListingIterError: - return HTTPServerError(request=req) - - while len(item_list) > 0: - previous_version = item_list.pop() - # there are older versions so copy the previous version to the - # current object and delete the previous version - orig_container = self.container_name - orig_obj = self.object_name - self.container_name = lcontainer - self.object_name = previous_version['name'].encode('utf-8') - - copy_path = '/v1/' + self.account_name + '/' + \ - self.container_name + '/' + self.object_name - - copy_headers = {'X-Newest': 'True', - 'Destination': orig_container + '/' + orig_obj - } - copy_environ = {'REQUEST_METHOD': 'COPY', - 'swift_versioned_copy': True - } - creq = Request.blank(copy_path, headers=copy_headers, - environ=copy_environ) - copy_resp = self.COPY(creq) - if copy_resp.status_int == HTTP_NOT_FOUND: - # the version isn't there so we'll try with previous - self.container_name = orig_container - self.object_name = orig_obj - continue - if is_client_error(copy_resp.status_int): - # some user error, maybe permissions - return HTTPPreconditionFailed(request=req) - elif not is_success(copy_resp.status_int): - # could not copy the data, bail - return HTTPServiceUnavailable(request=req) - # reset these because the COPY changed them - self.container_name = lcontainer - self.object_name = previous_version['name'].encode('utf-8') - new_del_req = Request.blank(copy_path, environ=req.environ) - container_info = self.container_info( - self.account_name, self.container_name, req) - policy_idx = container_info['storage_policy'] - obj_ring = self.app.get_object_ring(policy_idx) - # pass the policy index to storage nodes via req header - new_del_req.headers['X-Backend-Storage-Policy-Index'] = \ - policy_idx - container_partition = container_info['partition'] - containers = container_info['nodes'] - new_del_req.acl = container_info['write_acl'] - new_del_req.path_info = copy_path - req = new_del_req - # remove 'X-If-Delete-At', since it is not for the older copy - if 'X-If-Delete-At' in req.headers: - del req.headers['X-If-Delete-At'] - if 'swift.authorize' in req.environ: - aresp = req.environ['swift.authorize'](req) - if aresp: - return aresp - break if not containers: return HTTPNotFound(request=req) partition, nodes = obj_ring.get_nodes( diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 65044a1868..d55dcdab92 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -64,6 +64,9 @@ required_filters = [ if pipe.startswith('catch_errors') else [])}, {'name': 'dlo', 'after_fn': lambda _junk: [ + 'staticweb', 'tempauth', 'keystoneauth', + 'catch_errors', 'gatekeeper', 'proxy_logging']}, + {'name': 'versioned_writes', 'after_fn': lambda _junk: [ 'staticweb', 'tempauth', 'keystoneauth', 'catch_errors', 'gatekeeper', 'proxy_logging']}] diff --git a/test/functional/swift_test_client.py b/test/functional/swift_test_client.py index c93b2eab09..750181bc06 100644 --- a/test/functional/swift_test_client.py +++ b/test/functional/swift_test_client.py @@ -236,6 +236,9 @@ class Connection(object): if not cfg.get('no_auth_token'): headers['X-Auth-Token'] = self.storage_token + if cfg.get('use_token'): + headers['X-Auth-Token'] = cfg.get('use_token') + if isinstance(hdrs, dict): headers.update(hdrs) return headers @@ -507,6 +510,18 @@ class Container(Base): return self.conn.make_request('PUT', self.path, hdrs=hdrs, parms=parms, cfg=cfg) in (201, 202) + def update_metadata(self, hdrs=None, cfg=None): + if hdrs is None: + hdrs = {} + if cfg is None: + cfg = {} + + self.conn.make_request('POST', self.path, hdrs=hdrs, cfg=cfg) + if not 200 <= self.conn.response.status <= 299: + raise ResponseError(self.conn.response, 'POST', + self.conn.make_path(self.path)) + return True + def delete(self, hdrs=None, parms=None): if hdrs is None: hdrs = {} @@ -637,6 +652,9 @@ class File(Base): else: headers['Content-Length'] = 0 + if cfg.get('use_token'): + headers['X-Auth-Token'] = cfg.get('use_token') + if cfg.get('no_content_type'): pass elif self.content_type: @@ -711,13 +729,13 @@ class File(Base): return self.conn.make_request('COPY', self.path, hdrs=headers, parms=parms) == 201 - def delete(self, hdrs=None, parms=None): + def delete(self, hdrs=None, parms=None, cfg=None): if hdrs is None: hdrs = {} if parms is None: parms = {} if self.conn.make_request('DELETE', self.path, hdrs=hdrs, - parms=parms) != 204: + cfg=cfg, parms=parms) != 204: raise ResponseError(self.conn.response, 'DELETE', self.conn.make_path(self.path)) diff --git a/test/functional/tests.py b/test/functional/tests.py index 18b3d4716d..8bc628c7c9 100644 --- a/test/functional/tests.py +++ b/test/functional/tests.py @@ -2598,7 +2598,7 @@ class TestObjectVersioningEnv(object): @classmethod def setUp(cls): cls.conn = Connection(tf.config) - cls.conn.authenticate() + cls.storage_url, cls.storage_token = cls.conn.authenticate() cls.account = Account(cls.conn, tf.config.get('account', tf.config['username'])) @@ -2628,6 +2628,30 @@ class TestObjectVersioningEnv(object): # if versioning is off, then X-Versions-Location won't persist cls.versioning_enabled = 'versions' in container_info + # setup another account to test ACLs + config2 = deepcopy(tf.config) + config2['account'] = tf.config['account2'] + config2['username'] = tf.config['username2'] + config2['password'] = tf.config['password2'] + cls.conn2 = Connection(config2) + cls.storage_url2, cls.storage_token2 = cls.conn2.authenticate() + cls.account2 = cls.conn2.get_account() + cls.account2.delete_containers() + + # setup another account with no access to anything to test ACLs + config3 = deepcopy(tf.config) + config3['account'] = tf.config['account'] + config3['username'] = tf.config['username3'] + config3['password'] = tf.config['password3'] + cls.conn3 = Connection(config3) + cls.storage_url3, cls.storage_token3 = cls.conn3.authenticate() + cls.account3 = cls.conn3.get_account() + + @classmethod + def tearDown(cls): + cls.account.delete_containers() + cls.account2.delete_containers() + class TestCrossPolicyObjectVersioningEnv(object): # tri-state: None initially, then True/False @@ -2650,14 +2674,14 @@ class TestCrossPolicyObjectVersioningEnv(object): cls.multiple_policies_enabled = True else: cls.multiple_policies_enabled = False - # We have to lie here that versioning is enabled. We actually - # don't know, but it does not matter. We know these tests cannot - # run without multiple policies present. If multiple policies are - # present, we won't be setting this field to any value, so it - # should all still work. - cls.versioning_enabled = True + cls.versioning_enabled = False return + if cls.versioning_enabled is None: + cls.versioning_enabled = 'versioned_writes' in cluster_info + if not cls.versioning_enabled: + return + policy = cls.policies.select() version_policy = cls.policies.exclude(name=policy['name']).select() @@ -2691,6 +2715,25 @@ class TestCrossPolicyObjectVersioningEnv(object): # if versioning is off, then X-Versions-Location won't persist cls.versioning_enabled = 'versions' in container_info + # setup another account to test ACLs + config2 = deepcopy(tf.config) + config2['account'] = tf.config['account2'] + config2['username'] = tf.config['username2'] + config2['password'] = tf.config['password2'] + cls.conn2 = Connection(config2) + cls.storage_url2, cls.storage_token2 = cls.conn2.authenticate() + cls.account2 = cls.conn2.get_account() + cls.account2.delete_containers() + + # setup another account with no access to anything to test ACLs + config3 = deepcopy(tf.config) + config3['account'] = tf.config['account'] + config3['username'] = tf.config['username3'] + config3['password'] = tf.config['password3'] + cls.conn3 = Connection(config3) + cls.storage_url3, cls.storage_token3 = cls.conn3.authenticate() + cls.account3 = cls.conn3.get_account() + class TestObjectVersioning(Base): env = TestObjectVersioningEnv @@ -2709,40 +2752,103 @@ class TestObjectVersioning(Base): def tearDown(self): super(TestObjectVersioning, self).tearDown() try: - # delete versions first! + # only delete files and not container + # as they were configured in self.env self.env.versions_container.delete_files() self.env.container.delete_files() except ResponseError: pass + def test_clear_version_option(self): + # sanity + self.assertEqual(self.env.container.info()['versions'], + self.env.versions_container.name) + self.env.container.update_metadata( + hdrs={'X-Versions-Location': ''}) + self.assertEqual(self.env.container.info().get('versions'), None) + + # set location back to the way it was + self.env.container.update_metadata( + hdrs={'X-Versions-Location': self.env.versions_container.name}) + self.assertEqual(self.env.container.info()['versions'], + self.env.versions_container.name) + def test_overwriting(self): container = self.env.container versions_container = self.env.versions_container + cont_info = container.info() + self.assertEquals(cont_info['versions'], versions_container.name) + obj_name = Utils.create_name() versioned_obj = container.file(obj_name) - versioned_obj.write("aaaaa") + versioned_obj.write("aaaaa", hdrs={'Content-Type': 'text/jibberish01'}) + obj_info = versioned_obj.info() + self.assertEqual('text/jibberish01', obj_info['content_type']) self.assertEqual(0, versions_container.info()['object_count']) - - versioned_obj.write("bbbbb") + versioned_obj.write("bbbbb", hdrs={'Content-Type': 'text/jibberish02', + 'X-Object-Meta-Foo': 'Bar'}) + versioned_obj.initialize() + self.assertEqual(versioned_obj.content_type, 'text/jibberish02') + self.assertEqual(versioned_obj.metadata['foo'], 'Bar') # the old version got saved off self.assertEqual(1, versions_container.info()['object_count']) versioned_obj_name = versions_container.files()[0] - self.assertEqual( - "aaaaa", versions_container.file(versioned_obj_name).read()) + prev_version = versions_container.file(versioned_obj_name) + prev_version.initialize() + self.assertEqual("aaaaa", prev_version.read()) + self.assertEqual(prev_version.content_type, 'text/jibberish01') + + # make sure the new obj metadata did not leak to the prev. version + self.assertTrue('foo' not in prev_version.metadata) + + # check that POST does not create a new version + versioned_obj.sync_metadata(metadata={'fu': 'baz'}) + self.assertEqual(1, versions_container.info()['object_count']) # if we overwrite it again, there are two versions versioned_obj.write("ccccc") self.assertEqual(2, versions_container.info()['object_count']) + versioned_obj_name = versions_container.files()[1] + prev_version = versions_container.file(versioned_obj_name) + prev_version.initialize() + self.assertEqual("bbbbb", prev_version.read()) + self.assertEqual(prev_version.content_type, 'text/jibberish02') + self.assertTrue('foo' in prev_version.metadata) + self.assertTrue('fu' in prev_version.metadata) # as we delete things, the old contents return self.assertEqual("ccccc", versioned_obj.read()) + + # test copy from a different container + src_container = self.env.account.container(Utils.create_name()) + self.assertTrue(src_container.create()) + src_name = Utils.create_name() + src_obj = src_container.file(src_name) + src_obj.write("ddddd", hdrs={'Content-Type': 'text/jibberish04'}) + src_obj.copy(container.name, obj_name) + + self.assertEqual("ddddd", versioned_obj.read()) + versioned_obj.initialize() + self.assertEqual(versioned_obj.content_type, 'text/jibberish04') + + # make sure versions container has the previous version + self.assertEqual(3, versions_container.info()['object_count']) + versioned_obj_name = versions_container.files()[2] + prev_version = versions_container.file(versioned_obj_name) + prev_version.initialize() + self.assertEqual("ccccc", prev_version.read()) + + # test delete + versioned_obj.delete() + self.assertEqual("ccccc", versioned_obj.read()) versioned_obj.delete() self.assertEqual("bbbbb", versioned_obj.read()) versioned_obj.delete() self.assertEqual("aaaaa", versioned_obj.read()) + self.assertEqual(0, versions_container.info()['object_count']) versioned_obj.delete() self.assertRaises(ResponseError, versioned_obj.read) @@ -2774,6 +2880,87 @@ class TestObjectVersioning(Base): self.assertEqual(3, versions_container.info()['object_count']) self.assertEqual("112233", man_file.read()) + def test_versioning_container_acl(self): + # create versions container and DO NOT give write access to account2 + versions_container = self.env.account.container(Utils.create_name()) + self.assertTrue(versions_container.create(hdrs={ + 'X-Container-Write': '' + })) + + # check account2 cannot write to versions container + fail_obj_name = Utils.create_name() + fail_obj = versions_container.file(fail_obj_name) + self.assertRaises(ResponseError, fail_obj.write, "should fail", + cfg={'use_token': self.env.storage_token2}) + + # create container and give write access to account2 + # don't set X-Versions-Location just yet + container = self.env.account.container(Utils.create_name()) + self.assertTrue(container.create(hdrs={ + 'X-Container-Write': self.env.conn2.user_acl})) + + # check account2 cannot set X-Versions-Location on container + self.assertRaises(ResponseError, container.update_metadata, hdrs={ + 'X-Versions-Location': versions_container}, + cfg={'use_token': self.env.storage_token2}) + + # good! now let admin set the X-Versions-Location + # p.s.: sticking a 'x-remove' header here to test precedence + # of both headers. Setting the location should succeed. + self.assertTrue(container.update_metadata(hdrs={ + 'X-Remove-Versions-Location': versions_container, + 'X-Versions-Location': versions_container})) + + # write object twice to container and check version + obj_name = Utils.create_name() + versioned_obj = container.file(obj_name) + self.assertTrue(versioned_obj.write("never argue with the data", + cfg={'use_token': self.env.storage_token2})) + self.assertEqual(versioned_obj.read(), "never argue with the data") + + self.assertTrue( + versioned_obj.write("we don't have no beer, just tequila", + cfg={'use_token': self.env.storage_token2})) + self.assertEqual(versioned_obj.read(), + "we don't have no beer, just tequila") + self.assertEqual(1, versions_container.info()['object_count']) + + # read the original uploaded object + for filename in versions_container.files(): + backup_file = versions_container.file(filename) + break + self.assertEqual(backup_file.read(), "never argue with the data") + + # user3 (some random user with no access to anything) + # tries to read from versioned container + self.assertRaises(ResponseError, backup_file.read, + cfg={'use_token': self.env.storage_token3}) + + # user3 cannot write or delete from source container either + self.assertRaises(ResponseError, versioned_obj.write, + "some random user trying to write data", + cfg={'use_token': self.env.storage_token3}) + self.assertRaises(ResponseError, versioned_obj.delete, + cfg={'use_token': self.env.storage_token3}) + + # user2 can't read or delete from versions-location + self.assertRaises(ResponseError, backup_file.read, + cfg={'use_token': self.env.storage_token2}) + self.assertRaises(ResponseError, backup_file.delete, + cfg={'use_token': self.env.storage_token2}) + + # but is able to delete from the source container + # this could be a helpful scenario for dev ops that want to setup + # just one container to hold object versions of multiple containers + # and each one of those containers are owned by different users + self.assertTrue(versioned_obj.delete( + cfg={'use_token': self.env.storage_token2})) + + # tear-down since we create these containers here + # and not in self.env + versions_container.delete_recursive() + container.delete_recursive() + def test_versioning_check_acl(self): container = self.env.container versions_container = self.env.versions_container diff --git a/test/unit/common/middleware/helpers.py b/test/unit/common/middleware/helpers.py index 7c1b45571e..bc6ad50fdd 100644 --- a/test/unit/common/middleware/helpers.py +++ b/test/unit/common/middleware/helpers.py @@ -76,7 +76,7 @@ class FakeSwift(object): path += '?' + env['QUERY_STRING'] if 'swift.authorize' in env: - resp = env['swift.authorize']() + resp = env['swift.authorize'](swob.Request(env)) if resp: return resp(env, start_response) diff --git a/test/unit/common/middleware/test_dlo.py b/test/unit/common/middleware/test_dlo.py index c290430e08..702eb2432d 100644 --- a/test/unit/common/middleware/test_dlo.py +++ b/test/unit/common/middleware/test_dlo.py @@ -793,7 +793,7 @@ class TestDloGetManifest(DloTestCase): def test_get_with_auth_overridden(self): auth_got_called = [0] - def my_auth(): + def my_auth(req): auth_got_called[0] += 1 return None diff --git a/test/unit/common/middleware/test_versioned_writes.py b/test/unit/common/middleware/test_versioned_writes.py new file mode 100644 index 0000000000..1d38b73f68 --- /dev/null +++ b/test/unit/common/middleware/test_versioned_writes.py @@ -0,0 +1,558 @@ +# Copyright (c) 2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from swift.common import swob +from swift.common.middleware import versioned_writes +from swift.common.swob import Request +from test.unit.common.middleware.helpers import FakeSwift + + +class FakeCache(object): + + def __init__(self, val): + if 'status' not in val: + val['status'] = 200 + self.val = val + + def get(self, *args): + return self.val + + +class VersionedWritesTestCase(unittest.TestCase): + def setUp(self): + self.app = FakeSwift() + conf = {'allow_versioned_writes': 'true'} + self.vw = versioned_writes.filter_factory(conf)(self.app) + + def call_app(self, req, app=None, expect_exception=False): + if app is None: + app = self.app + + self.authorized = [] + + def authorize(req): + self.authorized.append(req) + + if 'swift.authorize' not in req.environ: + req.environ['swift.authorize'] = authorize + + req.headers.setdefault("User-Agent", "Marula Kruger") + + status = [None] + headers = [None] + + def start_response(s, h, ei=None): + status[0] = s + headers[0] = h + + body_iter = app(req.environ, start_response) + body = '' + caught_exc = None + try: + for chunk in body_iter: + body += chunk + except Exception as exc: + if expect_exception: + caught_exc = exc + else: + raise + + if expect_exception: + return status[0], headers[0], body, caught_exc + else: + return status[0], headers[0], body + + def call_vw(self, req, **kwargs): + return self.call_app(req, app=self.vw, **kwargs) + + def assertRequestEqual(self, req, other): + self.assertEqual(req.method, other.method) + self.assertEqual(req.path, other.path) + + def test_put_container(self): + self.app.register('PUT', '/v1/a/c', swob.HTTPOk, {}, 'passed') + req = Request.blank('/v1/a/c', + headers={'X-Versions-Location': 'ver_cont'}, + environ={'REQUEST_METHOD': 'PUT'}) + status, headers, body = self.call_vw(req) + self.assertEquals(status, '200 OK') + + # check for sysmeta header + calls = self.app.calls_with_headers + method, path, req_headers = calls[0] + self.assertEquals('PUT', method) + self.assertEquals('/v1/a/c', path) + self.assertTrue('x-container-sysmeta-versions-location' in req_headers) + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + + def test_container_allow_versioned_writes_false(self): + self.vw.conf = {'allow_versioned_writes': 'false'} + + # PUT/POST container must fail as 412 when allow_versioned_writes + # set to false + for method in ('PUT', 'POST'): + req = Request.blank('/v1/a/c', + headers={'X-Versions-Location': 'ver_cont'}, + environ={'REQUEST_METHOD': method}) + try: + status, headers, body = self.call_vw(req) + except swob.HTTPException as e: + pass + self.assertEquals(e.status_int, 412) + + # GET/HEAD performs as normal + self.app.register('GET', '/v1/a/c', swob.HTTPOk, {}, 'passed') + self.app.register('HEAD', '/v1/a/c', swob.HTTPOk, {}, 'passed') + + for method in ('GET', 'HEAD'): + req = Request.blank('/v1/a/c', + headers={'X-Versions-Location': 'ver_cont'}, + environ={'REQUEST_METHOD': method}) + status, headers, body = self.call_vw(req) + self.assertEquals(status, '200 OK') + + def test_remove_versions_location(self): + self.app.register('POST', '/v1/a/c', swob.HTTPOk, {}, 'passed') + req = Request.blank('/v1/a/c', + headers={'X-Remove-Versions-Location': 'x'}, + environ={'REQUEST_METHOD': 'POST'}) + status, headers, body = self.call_vw(req) + self.assertEquals(status, '200 OK') + + # check for sysmeta header + calls = self.app.calls_with_headers + method, path, req_headers = calls[0] + self.assertEquals('POST', method) + self.assertEquals('/v1/a/c', path) + self.assertTrue('x-container-sysmeta-versions-location' in req_headers) + self.assertTrue('x-versions-location' in req_headers) + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + + def test_remove_add_versions_precedence(self): + self.app.register( + 'POST', '/v1/a/c', swob.HTTPOk, + {'x-container-sysmeta-versions-location': 'ver_cont'}, + 'passed') + req = Request.blank('/v1/a/c', + headers={'X-Remove-Versions-Location': 'x', + 'X-Versions-Location': 'ver_cont'}, + environ={'REQUEST_METHOD': 'POST'}) + + status, headers, body = self.call_vw(req) + self.assertEquals(status, '200 OK') + self.assertTrue(('X-Versions-Location', 'ver_cont') in headers) + + # check for sysmeta header + calls = self.app.calls_with_headers + method, path, req_headers = calls[0] + self.assertEquals('POST', method) + self.assertEquals('/v1/a/c', path) + self.assertTrue('x-container-sysmeta-versions-location' in req_headers) + self.assertTrue('x-remove-versions-location' not in req_headers) + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + + def test_get_container(self): + self.app.register( + 'GET', '/v1/a/c', swob.HTTPOk, + {'x-container-sysmeta-versions-location': 'ver_cont'}, None) + req = Request.blank( + '/v1/a/c', + environ={'REQUEST_METHOD': 'GET'}) + status, headers, body = self.call_vw(req) + self.assertEquals(status, '200 OK') + self.assertTrue(('X-Versions-Location', 'ver_cont') in headers) + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + + def test_get_head(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, None) + req = Request.blank( + '/v1/a/c/o', + environ={'REQUEST_METHOD': 'GET'}) + status, headers, body = self.call_vw(req) + self.assertEquals(status, '200 OK') + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + + self.app.register('HEAD', '/v1/a/c/o', swob.HTTPOk, {}, None) + req = Request.blank( + '/v1/a/c/o', + environ={'REQUEST_METHOD': 'HEAD'}) + status, headers, body = self.call_vw(req) + self.assertEquals(status, '200 OK') + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + + def test_put_object_no_versioning(self): + self.app.register( + 'PUT', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') + + cache = FakeCache({}) + req = Request.blank( + '/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache, + 'CONTENT_LENGTH': '100'}) + status, headers, body = self.call_vw(req) + self.assertEquals(status, '200 OK') + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + + def test_put_first_object_success(self): + self.app.register( + 'PUT', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') + self.app.register( + 'HEAD', '/v1/a/c/o', swob.HTTPNotFound, {}, None) + + cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}}) + req = Request.blank( + '/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache, + 'CONTENT_LENGTH': '100'}) + status, headers, body = self.call_vw(req) + self.assertEquals(status, '200 OK') + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + + def test_PUT_versioning_with_nonzero_default_policy(self): + self.app.register( + 'PUT', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') + self.app.register( + 'HEAD', '/v1/a/c/o', swob.HTTPNotFound, {}, None) + + cache = FakeCache({'versions': 'ver_cont', 'storage_policy': '2'}) + req = Request.blank( + '/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache, + 'CONTENT_LENGTH': '100'}) + status, headers, body = self.call_vw(req) + self.assertEquals(status, '200 OK') + + # check for 'X-Backend-Storage-Policy-Index' in HEAD request + calls = self.app.calls_with_headers + method, path, req_headers = calls[0] + self.assertEquals('HEAD', method) + self.assertEquals('/v1/a/c/o', path) + self.assertTrue('X-Backend-Storage-Policy-Index' in req_headers) + self.assertEquals('2', + req_headers.get('X-Backend-Storage-Policy-Index')) + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + + def test_put_object_no_versioning_with_container_config_true(self): + # set False to versions_write obsously and expect no COPY occurred + self.vw.conf = {'allow_versioned_writes': 'false'} + self.app.register( + 'PUT', '/v1/a/c/o', swob.HTTPCreated, {}, 'passed') + self.app.register( + 'HEAD', '/v1/a/c/o', swob.HTTPOk, + {'last-modified': 'Wed, 19 Nov 2014 18:19:02 GMT'}, 'passed') + cache = FakeCache({'versions': 'ver_cont'}) + req = Request.blank( + '/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache, + 'CONTENT_LENGTH': '100'}) + status, headers, body = self.call_vw(req) + self.assertEquals(status, '201 Created') + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + called_method = [method for (method, path, hdrs) in self.app._calls] + self.assertTrue('COPY' not in called_method) + + def test_delete_object_no_versioning_with_container_config_true(self): + # set False to versions_write obviously and expect no GET versioning + # container and COPY called (just delete object as normal) + self.vw.conf = {'allow_versioned_writes': 'false'} + self.app.register( + 'DELETE', '/v1/a/c/o', swob.HTTPNoContent, {}, 'passed') + cache = FakeCache({'versions': 'ver_cont'}) + req = Request.blank( + '/v1/a/c/o', + environ={'REQUEST_METHOD': 'DELETE', 'swift.cache': cache}) + status, headers, body = self.call_vw(req) + self.assertEquals(status, '204 No Content') + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + called_method = \ + [method for (method, path, rheaders) in self.app._calls] + self.assertTrue('COPY' not in called_method) + self.assertTrue('GET' not in called_method) + + def test_copy_object_no_versioning_with_container_config_true(self): + # set False to versions_write obviously and expect no extra + # COPY called (just copy object as normal) + self.vw.conf = {'allow_versioned_writes': 'false'} + self.app.register( + 'COPY', '/v1/a/c/o', swob.HTTPCreated, {}, None) + cache = FakeCache({'versions': 'ver_cont'}) + req = Request.blank( + '/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache}) + status, headers, body = self.call_vw(req) + self.assertEquals(status, '201 Created') + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + called_method = \ + [method for (method, path, rheaders) in self.app._calls] + self.assertTrue('COPY' in called_method) + self.assertEquals(called_method.count('COPY'), 1) + + def test_new_version_success(self): + self.app.register( + 'PUT', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') + self.app.register( + 'HEAD', '/v1/a/c/o', swob.HTTPOk, + {'last-modified': 'Wed, 19 Nov 2014 18:19:02 GMT'}, 'passed') + self.app.register( + 'COPY', '/v1/a/c/o', swob.HTTPCreated, {}, None) + cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}}) + req = Request.blank( + '/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache, + 'CONTENT_LENGTH': '100'}) + status, headers, body = self.call_vw(req) + self.assertEquals(status, '200 OK') + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + + def test_new_version_sysmeta_precedence(self): + self.app.register( + 'PUT', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') + self.app.register( + 'HEAD', '/v1/a/c/o', swob.HTTPOk, + {'last-modified': 'Wed, 19 Nov 2014 18:19:02 GMT'}, 'passed') + self.app.register( + 'COPY', '/v1/a/c/o', swob.HTTPCreated, {}, None) + + # fill cache with two different values for versions location + # new middleware should use sysmeta first + cache = FakeCache({'versions': 'old_ver_cont', + 'sysmeta': {'versions-location': 'ver_cont'}}) + req = Request.blank( + '/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache, + 'CONTENT_LENGTH': '100'}) + status, headers, body = self.call_vw(req) + self.assertEquals(status, '200 OK') + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + + # check that sysmeta header was used + calls = self.app.calls_with_headers + method, path, req_headers = calls[1] + self.assertEquals('COPY', method) + self.assertEquals('/v1/a/c/o', path) + self.assertTrue(req_headers['Destination'].startswith('ver_cont/')) + + def test_copy_first_version(self): + self.app.register( + 'COPY', '/v1/a/src_cont/src_obj', swob.HTTPOk, {}, 'passed') + self.app.register( + 'HEAD', '/v1/a/tgt_cont/tgt_obj', swob.HTTPNotFound, {}, None) + cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}}) + req = Request.blank( + '/v1/a/src_cont/src_obj', + environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache, + 'CONTENT_LENGTH': '100'}, + headers={'Destination': 'tgt_cont/tgt_obj'}) + status, headers, body = self.call_vw(req) + self.assertEquals(status, '200 OK') + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + + def test_copy_new_version(self): + self.app.register( + 'COPY', '/v1/a/src_cont/src_obj', swob.HTTPOk, {}, 'passed') + self.app.register( + 'HEAD', '/v1/a/tgt_cont/tgt_obj', swob.HTTPOk, + {'last-modified': 'Wed, 19 Nov 2014 18:19:02 GMT'}, 'passed') + self.app.register( + 'COPY', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated, {}, None) + cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}}) + req = Request.blank( + '/v1/a/src_cont/src_obj', + environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache, + 'CONTENT_LENGTH': '100'}, + headers={'Destination': 'tgt_cont/tgt_obj'}) + status, headers, body = self.call_vw(req) + self.assertEquals(status, '200 OK') + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + + def test_copy_new_version_different_account(self): + self.app.register( + 'COPY', '/v1/src_a/src_cont/src_obj', swob.HTTPOk, {}, 'passed') + self.app.register( + 'HEAD', '/v1/tgt_a/tgt_cont/tgt_obj', swob.HTTPOk, + {'last-modified': 'Wed, 19 Nov 2014 18:19:02 GMT'}, 'passed') + self.app.register( + 'COPY', '/v1/tgt_a/tgt_cont/tgt_obj', swob.HTTPCreated, {}, None) + cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}}) + req = Request.blank( + '/v1/src_a/src_cont/src_obj', + environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache, + 'CONTENT_LENGTH': '100'}, + headers={'Destination': 'tgt_cont/tgt_obj', + 'Destination-Account': 'tgt_a'}) + status, headers, body = self.call_vw(req) + self.assertEquals(status, '200 OK') + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + + def test_delete_first_object_success(self): + self.app.register( + 'DELETE', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') + self.app.register( + 'GET', '/v1/a/ver_cont?format=json&prefix=001o/&marker=', + swob.HTTPNotFound, {}, None) + + cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}}) + req = Request.blank( + '/v1/a/c/o', + environ={'REQUEST_METHOD': 'DELETE', 'swift.cache': cache, + 'CONTENT_LENGTH': '0'}) + status, headers, body = self.call_vw(req) + self.assertEquals(status, '200 OK') + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + + def test_delete_latest_version_success(self): + self.app.register( + 'DELETE', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') + self.app.register( + 'GET', '/v1/a/ver_cont?format=json&prefix=001o/&marker=', + swob.HTTPOk, {}, + '[{"hash": "x", ' + '"last_modified": "2014-11-21T14:14:27.409100", ' + '"bytes": 3, ' + '"name": "001o/1", ' + '"content_type": "text/plain"}, ' + '{"hash": "y", ' + '"last_modified": "2014-11-21T14:23:02.206740", ' + '"bytes": 3, ' + '"name": "001o/2", ' + '"content_type": "text/plain"}]') + self.app.register( + 'GET', '/v1/a/ver_cont?format=json&prefix=001o/' + '&marker=001o/2', + swob.HTTPNotFound, {}, None) + self.app.register( + 'COPY', '/v1/a/ver_cont/001o/2', swob.HTTPCreated, + {}, None) + self.app.register( + 'DELETE', '/v1/a/ver_cont/001o/2', swob.HTTPOk, + {}, None) + + cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}}) + req = Request.blank( + '/v1/a/c/o', + headers={'X-If-Delete-At': 1}, + environ={'REQUEST_METHOD': 'DELETE', 'swift.cache': cache, + 'CONTENT_LENGTH': '0'}) + status, headers, body = self.call_vw(req) + self.assertEquals(status, '200 OK') + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + + # check that X-If-Delete-At was removed from DELETE request + calls = self.app.calls_with_headers + method, path, req_headers = calls.pop() + self.assertEquals('DELETE', method) + self.assertTrue(path.startswith('/v1/a/ver_cont/001o/2')) + self.assertFalse('x-if-delete-at' in req_headers or + 'X-If-Delete-At' in req_headers) + + def test_DELETE_on_expired_versioned_object(self): + self.app.register( + 'GET', '/v1/a/ver_cont?format=json&prefix=001o/&marker=', + swob.HTTPOk, {}, + '[{"hash": "x", ' + '"last_modified": "2014-11-21T14:14:27.409100", ' + '"bytes": 3, ' + '"name": "001o/1", ' + '"content_type": "text/plain"}, ' + '{"hash": "y", ' + '"last_modified": "2014-11-21T14:23:02.206740", ' + '"bytes": 3, ' + '"name": "001o/2", ' + '"content_type": "text/plain"}]') + self.app.register( + 'GET', '/v1/a/ver_cont?format=json&prefix=001o/' + '&marker=001o/2', + swob.HTTPNotFound, {}, None) + + # expired object + self.app.register( + 'COPY', '/v1/a/ver_cont/001o/2', swob.HTTPNotFound, + {}, None) + self.app.register( + 'COPY', '/v1/a/ver_cont/001o/1', swob.HTTPCreated, + {}, None) + self.app.register( + 'DELETE', '/v1/a/ver_cont/001o/1', swob.HTTPOk, + {}, None) + + cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}}) + req = Request.blank( + '/v1/a/c/o', + environ={'REQUEST_METHOD': 'DELETE', 'swift.cache': cache, + 'CONTENT_LENGTH': '0'}) + status, headers, body = self.call_vw(req) + self.assertEquals(status, '200 OK') + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + + def test_denied_DELETE_of_versioned_object(self): + authorize_call = [] + self.app.register( + 'DELETE', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') + self.app.register( + 'GET', '/v1/a/ver_cont?format=json&prefix=001o/&marker=', + swob.HTTPOk, {}, + '[{"hash": "x", ' + '"last_modified": "2014-11-21T14:14:27.409100", ' + '"bytes": 3, ' + '"name": "001o/1", ' + '"content_type": "text/plain"}, ' + '{"hash": "y", ' + '"last_modified": "2014-11-21T14:23:02.206740", ' + '"bytes": 3, ' + '"name": "001o/2", ' + '"content_type": "text/plain"}]') + self.app.register( + 'GET', '/v1/a/ver_cont?format=json&prefix=001o/' + '&marker=001o/2', + swob.HTTPNotFound, {}, None) + self.app.register( + 'DELETE', '/v1/a/c/o', swob.HTTPForbidden, + {}, None) + + def fake_authorize(req): + authorize_call.append(req) + return swob.HTTPForbidden() + + cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}}) + req = Request.blank( + '/v1/a/c/o', + environ={'REQUEST_METHOD': 'DELETE', 'swift.cache': cache, + 'swift.authorize': fake_authorize, + 'CONTENT_LENGTH': '0'}) + status, headers, body = self.call_vw(req) + self.assertEquals(status, '403 Forbidden') + self.assertEqual(len(authorize_call), 1) + self.assertRequestEqual(req, authorize_call[0]) diff --git a/test/unit/common/test_constraints.py b/test/unit/common/test_constraints.py index 7808511425..1fd3411ad2 100644 --- a/test/unit/common/test_constraints.py +++ b/test/unit/common/test_constraints.py @@ -515,6 +515,24 @@ class TestConstraints(unittest.TestCase): constraints.check_account_format, req, req.headers['X-Copy-From-Account']) + def test_check_container_format(self): + invalid_versions_locations = ( + 'container/with/slashes', + '', # empty + ) + for versions_location in invalid_versions_locations: + req = Request.blank( + '/v/a/c/o', headers={ + 'X-Versions-Location': versions_location}) + try: + constraints.check_container_format( + req, req.headers['X-Versions-Location']) + except HTTPException as e: + self.assertTrue(e.body.startswith('Container name cannot')) + else: + self.fail('check_container_format did not raise error for %r' % + req.headers['X-Versions-Location']) + class TestConstraintsConfig(unittest.TestCase): diff --git a/test/unit/common/test_wsgi.py b/test/unit/common/test_wsgi.py index cf92edeb76..a165ecb5f2 100644 --- a/test/unit/common/test_wsgi.py +++ b/test/unit/common/test_wsgi.py @@ -141,6 +141,11 @@ class TestWSGI(unittest.TestCase): expected = swift.common.middleware.dlo.DynamicLargeObject self.assertTrue(isinstance(app, expected)) + app = app.app + expected = \ + swift.common.middleware.versioned_writes.VersionedWritesMiddleware + self.assert_(isinstance(app, expected)) + app = app.app expected = swift.proxy.server.Application self.assertTrue(isinstance(app, expected)) @@ -1414,6 +1419,7 @@ class TestPipelineModification(unittest.TestCase): ['swift.common.middleware.catch_errors', 'swift.common.middleware.gatekeeper', 'swift.common.middleware.dlo', + 'swift.common.middleware.versioned_writes', 'swift.proxy.server']) def test_proxy_modify_wsgi_pipeline(self): @@ -1444,6 +1450,7 @@ class TestPipelineModification(unittest.TestCase): ['swift.common.middleware.catch_errors', 'swift.common.middleware.gatekeeper', 'swift.common.middleware.dlo', + 'swift.common.middleware.versioned_writes', 'swift.common.middleware.healthcheck', 'swift.proxy.server']) @@ -1541,6 +1548,7 @@ class TestPipelineModification(unittest.TestCase): 'swift.common.middleware.catch_errors', 'swift.common.middleware.gatekeeper', 'swift.common.middleware.dlo', + 'swift.common.middleware.versioned_writes', 'swift.common.middleware.healthcheck', 'swift.proxy.server']) @@ -1554,6 +1562,7 @@ class TestPipelineModification(unittest.TestCase): 'swift.common.middleware.healthcheck', 'swift.common.middleware.catch_errors', 'swift.common.middleware.dlo', + 'swift.common.middleware.versioned_writes', 'swift.proxy.server']) def test_catch_errors_gatekeeper_configured_not_at_start(self): @@ -1566,6 +1575,7 @@ class TestPipelineModification(unittest.TestCase): 'swift.common.middleware.catch_errors', 'swift.common.middleware.gatekeeper', 'swift.common.middleware.dlo', + 'swift.common.middleware.versioned_writes', 'swift.proxy.server']) @with_tempdir @@ -1598,7 +1608,7 @@ class TestPipelineModification(unittest.TestCase): tempdir, policy.ring_name + '.ring.gz') app = wsgi.loadapp(conf_path) - proxy_app = app.app.app.app.app + proxy_app = app.app.app.app.app.app self.assertEqual(proxy_app.account_ring.serialized_path, account_ring_path) self.assertEqual(proxy_app.container_ring.serialized_path, diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 2a7cb04328..d113a70afe 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -56,7 +56,7 @@ from swift.proxy.controllers.obj import ReplicatedObjectController from swift.account import server as account_server from swift.container import server as container_server from swift.obj import server as object_server -from swift.common.middleware import proxy_logging +from swift.common.middleware import proxy_logging, versioned_writes from swift.common.middleware.acl import parse_acl, format_acl from swift.common.exceptions import ChunkReadTimeout, DiskFileNotExist, \ APIVersionError @@ -70,7 +70,7 @@ from swift.proxy.controllers.base import get_container_memcache_key, \ import swift.proxy.controllers import swift.proxy.controllers.obj from swift.common.swob import Request, Response, HTTPUnauthorized, \ - HTTPException, HTTPForbidden, HeaderKeyDict + HTTPException, HeaderKeyDict from swift.common import storage_policy from swift.common.storage_policy import StoragePolicy, ECStoragePolicy, \ StoragePolicyCollection, POLICIES @@ -107,7 +107,7 @@ def do_setup(the_object_server): conf = {'devices': _testdir, 'swift_dir': _testdir, 'mount_check': 'false', 'allowed_headers': 'content-encoding, x-object-manifest, content-disposition, foo', - 'allow_versions': 'True'} + 'allow_versions': 't'} prolis = listen(('localhost', 0)) acc1lis = listen(('localhost', 0)) acc2lis = listen(('localhost', 0)) @@ -2710,162 +2710,6 @@ class TestObjectController(unittest.TestCase): exp = 'HTTP/1.1 200' self.assertEqual(headers[:len(exp)], exp) - def test_expirer_DELETE_on_versioned_object(self): - test_errors = [] - - def test_connect(ipaddr, port, device, partition, method, path, - headers=None, query_string=None): - if method == 'DELETE': - if 'x-if-delete-at' in headers or 'X-If-Delete-At' in headers: - test_errors.append('X-If-Delete-At in headers') - - body = json.dumps( - [{"name": "001o/1", - "hash": "x", - "bytes": 0, - "content_type": "text/plain", - "last_modified": "1970-01-01T00:00:01.000000"}]) - body_iter = ('', '', body, '', '', '', '', '', '', '', '', '', '', '') - with save_globals(): - controller = ReplicatedObjectController( - self.app, 'a', 'c', 'o') - # HEAD HEAD GET GET HEAD GET GET GET PUT PUT - # PUT DEL DEL DEL - set_http_connect(200, 200, 200, 200, 200, 200, 200, 200, 201, 201, - 201, 204, 204, 204, - give_connect=test_connect, - body_iter=body_iter, - headers={'x-versions-location': 'foo'}) - self.app.memcache.store = {} - req = Request.blank('/v1/a/c/o', - headers={'X-If-Delete-At': 1}, - environ={'REQUEST_METHOD': 'DELETE'}) - self.app.update_request(req) - controller.DELETE(req) - self.assertEqual(test_errors, []) - - @patch_policies([ - StoragePolicy(0, 'zero', False, object_ring=FakeRing()), - StoragePolicy(1, 'one', True, object_ring=FakeRing()) - ]) - def test_DELETE_on_expired_versioned_object(self): - # reset the router post patch_policies - self.app.obj_controller_router = proxy_server.ObjectControllerRouter() - methods = set() - authorize_call_count = [0] - - def test_connect(ipaddr, port, device, partition, method, path, - headers=None, query_string=None): - methods.add((method, path)) - - def fake_container_info(account, container, req): - return {'status': 200, 'sync_key': None, - 'meta': {}, 'cors': {'allow_origin': None, - 'expose_headers': None, - 'max_age': None}, - 'sysmeta': {}, 'read_acl': None, 'object_count': None, - 'write_acl': None, 'versions': 'foo', - 'partition': 1, 'bytes': None, 'storage_policy': '1', - 'nodes': [{'zone': 0, 'ip': '10.0.0.0', 'region': 0, - 'id': 0, 'device': 'sda', 'port': 1000}, - {'zone': 1, 'ip': '10.0.0.1', 'region': 1, - 'id': 1, 'device': 'sdb', 'port': 1001}, - {'zone': 2, 'ip': '10.0.0.2', 'region': 0, - 'id': 2, 'device': 'sdc', 'port': 1002}]} - - def fake_list_iter(container, prefix, env): - object_list = [{'name': '1'}, {'name': '2'}, {'name': '3'}] - for obj in object_list: - yield obj - - def fake_authorize(req): - authorize_call_count[0] += 1 - return None # allow the request - - with save_globals(): - controller = ReplicatedObjectController( - self.app, 'a', 'c', 'o') - controller.container_info = fake_container_info - controller._listing_iter = fake_list_iter - set_http_connect(404, 404, 404, # get for the previous version - 200, 200, 200, # get for the pre-previous - 201, 201, 201, # put move the pre-previous - 204, 204, 204, # delete for the pre-previous - give_connect=test_connect) - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'DELETE', - 'swift.authorize': fake_authorize}) - - self.app.memcache.store = {} - self.app.update_request(req) - controller.DELETE(req) - exp_methods = [('GET', '/a/foo/3'), - ('GET', '/a/foo/2'), - ('PUT', '/a/c/o'), - ('DELETE', '/a/foo/2')] - self.assertEqual(set(exp_methods), (methods)) - self.assertEqual(authorize_call_count[0], 2) - - @patch_policies([ - StoragePolicy(0, 'zero', False, object_ring=FakeRing()), - StoragePolicy(1, 'one', True, object_ring=FakeRing()) - ]) - def test_denied_DELETE_of_versioned_object(self): - # Verify that a request with read access to a versions container - # is unable to cause any write operations on the versioned container. - - # reset the router post patch_policies - self.app.obj_controller_router = proxy_server.ObjectControllerRouter() - methods = set() - authorize_call_count = [0] - - def test_connect(ipaddr, port, device, partition, method, path, - headers=None, query_string=None): - methods.add((method, path)) - - def fake_container_info(account, container, req): - return {'status': 200, 'sync_key': None, - 'meta': {}, 'cors': {'allow_origin': None, - 'expose_headers': None, - 'max_age': None}, - 'sysmeta': {}, 'read_acl': None, 'object_count': None, - 'write_acl': None, 'versions': 'foo', - 'partition': 1, 'bytes': None, 'storage_policy': '1', - 'nodes': [{'zone': 0, 'ip': '10.0.0.0', 'region': 0, - 'id': 0, 'device': 'sda', 'port': 1000}, - {'zone': 1, 'ip': '10.0.0.1', 'region': 1, - 'id': 1, 'device': 'sdb', 'port': 1001}, - {'zone': 2, 'ip': '10.0.0.2', 'region': 0, - 'id': 2, 'device': 'sdc', 'port': 1002}]} - - def fake_list_iter(container, prefix, env): - object_list = [{'name': '1'}, {'name': '2'}, {'name': '3'}] - for obj in object_list: - yield obj - - def fake_authorize(req): - # deny write access - authorize_call_count[0] += 1 - return HTTPForbidden(req) # allow the request - - with save_globals(): - controller = ReplicatedObjectController(self.app, 'a', 'c', 'o') - controller.container_info = fake_container_info - # patching _listing_iter simulates request being authorized - # to list versions container - controller._listing_iter = fake_list_iter - set_http_connect(give_connect=test_connect) - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'DELETE', - 'swift.authorize': fake_authorize}) - - self.app.memcache.store = {} - self.app.update_request(req) - resp = controller.DELETE(req) - self.assertEqual(403, resp.status_int) - self.assertFalse(methods, methods) - self.assertEqual(authorize_call_count[0], 1) - def test_PUT_auto_content_type(self): with save_globals(): controller = ReplicatedObjectController( @@ -5309,394 +5153,6 @@ class TestObjectController(unittest.TestCase): body = fd.read() self.assertEqual(body, 'oh hai123456789abcdef') - @unpatch_policies - def test_version_manifest(self, oc='versions', vc='vers', o='name'): - versions_to_create = 3 - # Create a container for our versioned object testing - (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, - obj2lis, obj3lis) = _test_sockets - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - pre = quote('%03x' % len(o)) - osub = '%s/sub' % o - presub = quote('%03x' % len(osub)) - osub = quote(osub) - presub = quote(presub) - oc = quote(oc) - vc = quote(vc) - fd.write('PUT /v1/a/%s HTTP/1.1\r\nHost: localhost\r\n' - 'Connection: close\r\nX-Storage-Token: t\r\n' - 'Content-Length: 0\r\nX-Versions-Location: %s\r\n\r\n' - % (oc, vc)) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 201' - self.assertEqual(headers[:len(exp)], exp) - # check that the header was set - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('GET /v1/a/%s HTTP/1.1\r\nHost: localhost\r\n' - 'Connection: close\r\nX-Storage-Token: t\r\n\r\n\r\n' % oc) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 2' # 2xx series response - self.assertEqual(headers[:len(exp)], exp) - self.assertTrue('X-Versions-Location: %s' % vc in headers) - # make the container for the object versions - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('PUT /v1/a/%s HTTP/1.1\r\nHost: localhost\r\n' - 'Connection: close\r\nX-Storage-Token: t\r\n' - 'Content-Length: 0\r\n\r\n' % vc) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 201' - self.assertEqual(headers[:len(exp)], exp) - # Create the versioned file - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('PUT /v1/a/%s/%s HTTP/1.1\r\nHost: ' - 'localhost\r\nConnection: close\r\nX-Storage-Token: ' - 't\r\nContent-Length: 5\r\nContent-Type: text/jibberish0\r\n' - 'X-Object-Meta-Foo: barbaz\r\n\r\n00000\r\n' % (oc, o)) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 201' - self.assertEqual(headers[:len(exp)], exp) - # Create the object versions - for segment in range(1, versions_to_create): - sleep(.01) # guarantee that the timestamp changes - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('PUT /v1/a/%s/%s HTTP/1.1\r\nHost: ' - 'localhost\r\nConnection: close\r\nX-Storage-Token: ' - 't\r\nContent-Length: 5\r\nContent-Type: text/jibberish%s' - '\r\n\r\n%05d\r\n' % (oc, o, segment, segment)) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 201' - self.assertEqual(headers[:len(exp)], exp) - # Ensure retrieving the manifest file gets the latest version - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('GET /v1/a/%s/%s HTTP/1.1\r\nHost: ' - 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n' - '\r\n' % (oc, o)) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 200' - self.assertEqual(headers[:len(exp)], exp) - self.assertTrue( - 'Content-Type: text/jibberish%s' % segment in headers) - self.assertTrue('X-Object-Meta-Foo: barbaz' not in headers) - body = fd.read() - self.assertEqual(body, '%05d' % segment) - # Ensure we have the right number of versions saved - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('GET /v1/a/%s?prefix=%s%s/ HTTP/1.1\r\nHost: ' - 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n' - % (vc, pre, o)) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 200' - self.assertEqual(headers[:len(exp)], exp) - body = fd.read() - versions = [x for x in body.split('\n') if x] - self.assertEqual(len(versions), versions_to_create - 1) - # copy a version and make sure the version info is stripped - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('COPY /v1/a/%s/%s HTTP/1.1\r\nHost: ' - 'localhost\r\nConnection: close\r\nX-Auth-Token: ' - 't\r\nDestination: %s/copied_name\r\n' - 'Content-Length: 0\r\n\r\n' % (oc, o, oc)) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 2' # 2xx series response to the COPY - self.assertEqual(headers[:len(exp)], exp) - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('GET /v1/a/%s/copied_name HTTP/1.1\r\nHost: ' - 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n' - % oc) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 200' - self.assertEqual(headers[:len(exp)], exp) - body = fd.read() - self.assertEqual(body, '%05d' % segment) - # post and make sure it's updated - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('POST /v1/a/%s/%s HTTP/1.1\r\nHost: ' - 'localhost\r\nConnection: close\r\nX-Auth-Token: ' - 't\r\nContent-Type: foo/bar\r\nContent-Length: 0\r\n' - 'X-Object-Meta-Bar: foo\r\n\r\n' % (oc, o)) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 2' # 2xx series response to the POST - self.assertEqual(headers[:len(exp)], exp) - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('GET /v1/a/%s/%s HTTP/1.1\r\nHost: ' - 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n' - % (oc, o)) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 200' - self.assertEqual(headers[:len(exp)], exp) - self.assertTrue('Content-Type: foo/bar' in headers) - self.assertTrue('X-Object-Meta-Bar: foo' in headers) - body = fd.read() - self.assertEqual(body, '%05d' % segment) - # Delete the object versions - for segment in range(versions_to_create - 1, 0, -1): - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('DELETE /v1/a/%s/%s HTTP/1.1\r\nHost: localhost\r' - '\nConnection: close\r\nX-Storage-Token: t\r\n\r\n' - % (oc, o)) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 2' # 2xx series response - self.assertEqual(headers[:len(exp)], exp) - # Ensure retrieving the manifest file gets the latest version - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('GET /v1/a/%s/%s HTTP/1.1\r\nHost: localhost\r\n' - 'Connection: close\r\nX-Auth-Token: t\r\n\r\n' - % (oc, o)) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 200' - self.assertEqual(headers[:len(exp)], exp) - self.assertTrue('Content-Type: text/jibberish%s' % (segment - 1) - in headers) - body = fd.read() - self.assertEqual(body, '%05d' % (segment - 1)) - # Ensure we have the right number of versions saved - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('GET /v1/a/%s?prefix=%s%s/ HTTP/1.1\r\nHost: ' - 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r' - '\n' % (vc, pre, o)) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 2' # 2xx series response - self.assertEqual(headers[:len(exp)], exp) - body = fd.read() - versions = [x for x in body.split('\n') if x] - self.assertEqual(len(versions), segment - 1) - # there is now one segment left (in the manifest) - # Ensure we have no saved versions - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('GET /v1/a/%s?prefix=%s%s/ HTTP/1.1\r\nHost: ' - 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n' - % (vc, pre, o)) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 204 No Content' - self.assertEqual(headers[:len(exp)], exp) - # delete the last verision - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('DELETE /v1/a/%s/%s HTTP/1.1\r\nHost: localhost\r\n' - 'Connection: close\r\nX-Storage-Token: t\r\n\r\n' % (oc, o)) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 2' # 2xx series response - self.assertEqual(headers[:len(exp)], exp) - # Ensure it's all gone - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('GET /v1/a/%s/%s HTTP/1.1\r\nHost: ' - 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n' - % (oc, o)) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 404' - self.assertEqual(headers[:len(exp)], exp) - - # make sure dlo manifest files don't get versioned - for _junk in range(1, versions_to_create): - sleep(.01) # guarantee that the timestamp changes - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('PUT /v1/a/%s/%s HTTP/1.1\r\nHost: ' - 'localhost\r\nConnection: close\r\nX-Storage-Token: ' - 't\r\nContent-Length: 0\r\n' - 'Content-Type: text/jibberish0\r\n' - 'Foo: barbaz\r\nX-Object-Manifest: %s/%s/\r\n\r\n' - % (oc, o, oc, o)) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 201' - self.assertEqual(headers[:len(exp)], exp) - - # Ensure we have no saved versions - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('GET /v1/a/%s?prefix=%s%s/ HTTP/1.1\r\nHost: ' - 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n' - % (vc, pre, o)) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 204 No Content' - self.assertEqual(headers[:len(exp)], exp) - - # DELETE v1/a/c/obj shouldn't delete v1/a/c/obj/sub versions - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('PUT /v1/a/%s/%s HTTP/1.1\r\nHost: ' - 'localhost\r\nConnection: close\r\nX-Storage-Token: ' - 't\r\nContent-Length: 5\r\nContent-Type: text/jibberish0\r\n' - 'Foo: barbaz\r\n\r\n00000\r\n' % (oc, o)) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 201' - self.assertEqual(headers[:len(exp)], exp) - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('PUT /v1/a/%s/%s HTTP/1.1\r\nHost: ' - 'localhost\r\nConnection: close\r\nX-Storage-Token: ' - 't\r\nContent-Length: 5\r\nContent-Type: text/jibberish0\r\n' - 'Foo: barbaz\r\n\r\n00001\r\n' % (oc, o)) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 201' - self.assertEqual(headers[:len(exp)], exp) - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('PUT /v1/a/%s/%s HTTP/1.1\r\nHost: ' - 'localhost\r\nConnection: close\r\nX-Storage-Token: ' - 't\r\nContent-Length: 4\r\nContent-Type: text/jibberish0\r\n' - 'Foo: barbaz\r\n\r\nsub1\r\n' % (oc, osub)) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 201' - self.assertEqual(headers[:len(exp)], exp) - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('PUT /v1/a/%s/%s HTTP/1.1\r\nHost: ' - 'localhost\r\nConnection: close\r\nX-Storage-Token: ' - 't\r\nContent-Length: 4\r\nContent-Type: text/jibberish0\r\n' - 'Foo: barbaz\r\n\r\nsub2\r\n' % (oc, osub)) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 201' - self.assertEqual(headers[:len(exp)], exp) - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('DELETE /v1/a/%s/%s HTTP/1.1\r\nHost: localhost\r\n' - 'Connection: close\r\nX-Storage-Token: t\r\n\r\n' % (oc, o)) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 2' # 2xx series response - self.assertEqual(headers[:len(exp)], exp) - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('GET /v1/a/%s?prefix=%s%s/ HTTP/1.1\r\nHost: ' - 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n' - % (vc, presub, osub)) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 2' # 2xx series response - self.assertEqual(headers[:len(exp)], exp) - body = fd.read() - versions = [x for x in body.split('\n') if x] - self.assertEqual(len(versions), 1) - - # Check for when the versions target container doesn't exist - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('PUT /v1/a/%swhoops HTTP/1.1\r\nHost: localhost\r\n' - 'Connection: close\r\nX-Storage-Token: t\r\n' - 'Content-Length: 0\r\nX-Versions-Location: none\r\n\r\n' % oc) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 201' - self.assertEqual(headers[:len(exp)], exp) - # Create the versioned file - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('PUT /v1/a/%swhoops/foo HTTP/1.1\r\nHost: ' - 'localhost\r\nConnection: close\r\nX-Storage-Token: ' - 't\r\nContent-Length: 5\r\n\r\n00000\r\n' % oc) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 201' - self.assertEqual(headers[:len(exp)], exp) - # Create another version - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('PUT /v1/a/%swhoops/foo HTTP/1.1\r\nHost: ' - 'localhost\r\nConnection: close\r\nX-Storage-Token: ' - 't\r\nContent-Length: 5\r\n\r\n00001\r\n' % oc) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 412' - self.assertEqual(headers[:len(exp)], exp) - # Delete the object - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('DELETE /v1/a/%swhoops/foo HTTP/1.1\r\nHost: localhost\r\n' - 'Connection: close\r\nX-Storage-Token: t\r\n\r\n' % oc) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 2' # 2xx response - self.assertEqual(headers[:len(exp)], exp) - - @unpatch_policies - def test_version_manifest_utf8(self): - oc = '0_oc_non_ascii\xc2\xa3' - vc = '0_vc_non_ascii\xc2\xa3' - o = '0_o_non_ascii\xc2\xa3' - self.test_version_manifest(oc, vc, o) - - @unpatch_policies - def test_version_manifest_utf8_container(self): - oc = '1_oc_non_ascii\xc2\xa3' - vc = '1_vc_ascii' - o = '1_o_ascii' - self.test_version_manifest(oc, vc, o) - - @unpatch_policies - def test_version_manifest_utf8_version_container(self): - oc = '2_oc_ascii' - vc = '2_vc_non_ascii\xc2\xa3' - o = '2_o_ascii' - self.test_version_manifest(oc, vc, o) - - @unpatch_policies - def test_version_manifest_utf8_containers(self): - oc = '3_oc_non_ascii\xc2\xa3' - vc = '3_vc_non_ascii\xc2\xa3' - o = '3_o_ascii' - self.test_version_manifest(oc, vc, o) - - @unpatch_policies - def test_version_manifest_utf8_object(self): - oc = '4_oc_ascii' - vc = '4_vc_ascii' - o = '4_o_non_ascii\xc2\xa3' - self.test_version_manifest(oc, vc, o) - - @unpatch_policies - def test_version_manifest_utf8_version_container_utf_object(self): - oc = '5_oc_ascii' - vc = '5_vc_non_ascii\xc2\xa3' - o = '5_o_non_ascii\xc2\xa3' - self.test_version_manifest(oc, vc, o) - - @unpatch_policies - def test_version_manifest_utf8_container_utf_object(self): - oc = '6_oc_non_ascii\xc2\xa3' - vc = '6_vc_ascii' - o = '6_o_non_ascii\xc2\xa3' - self.test_version_manifest(oc, vc, o) - @unpatch_policies def test_conditional_range_get(self): (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis, @@ -5924,129 +5380,6 @@ class TestObjectController(unittest.TestCase): finally: time.time = orig_time - @patch_policies([ - StoragePolicy(0, 'zero', False, object_ring=FakeRing()), - StoragePolicy(1, 'one', True, object_ring=FakeRing()) - ]) - def test_PUT_versioning_with_nonzero_default_policy(self): - # reset the router post patch_policies - self.app.obj_controller_router = proxy_server.ObjectControllerRouter() - - def test_connect(ipaddr, port, device, partition, method, path, - headers=None, query_string=None): - if method == "HEAD": - self.assertEqual(path, '/a/c/o.jpg') - self.assertNotEquals(None, - headers['X-Backend-Storage-Policy-Index']) - self.assertEqual(1, int(headers - ['X-Backend-Storage-Policy-Index'])) - - def fake_container_info(account, container, req): - return {'status': 200, 'sync_key': None, 'storage_policy': '1', - 'meta': {}, 'cors': {'allow_origin': None, - 'expose_headers': None, - 'max_age': None}, - 'sysmeta': {}, 'read_acl': None, 'object_count': None, - 'write_acl': None, 'versions': 'c-versions', - 'partition': 1, 'bytes': None, - 'nodes': [{'zone': 0, 'ip': '10.0.0.0', 'region': 0, - 'id': 0, 'device': 'sda', 'port': 1000}, - {'zone': 1, 'ip': '10.0.0.1', 'region': 1, - 'id': 1, 'device': 'sdb', 'port': 1001}, - {'zone': 2, 'ip': '10.0.0.2', 'region': 0, - 'id': 2, 'device': 'sdc', 'port': 1002}]} - with save_globals(): - controller = ReplicatedObjectController( - self.app, 'a', 'c', 'o.jpg') - - controller.container_info = fake_container_info - set_http_connect(200, 200, 200, # head: for the last version - 200, 200, 200, # get: for the last version - 201, 201, 201, # put: move the current version - 201, 201, 201, # put: save the new version - give_connect=test_connect) - req = Request.blank('/v1/a/c/o.jpg', - environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0'}) - self.app.update_request(req) - self.app.memcache.store = {} - res = controller.PUT(req) - self.assertEqual(201, res.status_int) - - @patch_policies([ - StoragePolicy(0, 'zero', False, object_ring=FakeRing()), - StoragePolicy(1, 'one', True, object_ring=FakeRing()) - ]) - def test_cross_policy_DELETE_versioning(self): - # reset the router post patch_policies - self.app.obj_controller_router = proxy_server.ObjectControllerRouter() - requests = [] - - def capture_requests(ipaddr, port, device, partition, method, path, - headers=None, query_string=None): - requests.append((method, path, headers)) - - def fake_container_info(app, env, account, container, **kwargs): - info = {'status': 200, 'sync_key': None, 'storage_policy': None, - 'meta': {}, 'cors': {'allow_origin': None, - 'expose_headers': None, - 'max_age': None}, - 'sysmeta': {}, 'read_acl': None, 'object_count': None, - 'write_acl': None, 'versions': None, - 'partition': 1, 'bytes': None, - 'nodes': [{'zone': 0, 'ip': '10.0.0.0', 'region': 0, - 'id': 0, 'device': 'sda', 'port': 1000}, - {'zone': 1, 'ip': '10.0.0.1', 'region': 1, - 'id': 1, 'device': 'sdb', 'port': 1001}, - {'zone': 2, 'ip': '10.0.0.2', 'region': 0, - 'id': 2, 'device': 'sdc', 'port': 1002}]} - if container == 'c': - info['storage_policy'] = '1' - info['versions'] = 'c-versions' - elif container == 'c-versions': - info['storage_policy'] = '0' - else: - self.fail('Unexpected call to get_info for %r' % container) - return info - container_listing = json.dumps([{'name': 'old_version'}]) - with save_globals(): - resp_status = ( - 200, 200, # listings for versions container - 200, 200, 200, # get: for the last version - 201, 201, 201, # put: move the last version - 200, 200, 200, # delete: for the last version - ) - body_iter = iter([container_listing] + [ - '' for x in range(len(resp_status) - 1)]) - set_http_connect(*resp_status, body_iter=body_iter, - give_connect=capture_requests) - req = Request.blank('/v1/a/c/current_version', method='DELETE') - self.app.update_request(req) - self.app.memcache.store = {} - with mock.patch('swift.proxy.controllers.base.get_info', - fake_container_info): - resp = self.app.handle_request(req) - self.assertEqual(200, resp.status_int) - expected = [('GET', '/a/c-versions')] * 2 + \ - [('GET', '/a/c-versions/old_version')] * 3 + \ - [('PUT', '/a/c/current_version')] * 3 + \ - [('DELETE', '/a/c-versions/old_version')] * 3 - self.assertEqual(expected, [(m, p) for m, p, h in requests]) - for method, path, headers in requests: - if 'current_version' in path: - expected_storage_policy = 1 - elif 'old_version' in path: - expected_storage_policy = 0 - else: - continue - storage_policy_index = \ - int(headers['X-Backend-Storage-Policy-Index']) - self.assertEqual( - expected_storage_policy, storage_policy_index, - 'Unexpected %s request for %s ' - 'with storage policy index %s' % ( - method, path, storage_policy_index)) - @unpatch_policies def test_leak_1(self): _request_instances = weakref.WeakKeyDictionary() @@ -9186,6 +8519,465 @@ class TestSwiftInfo(unittest.TestCase): self.assertEqual(sorted_pols[2]['name'], 'migrated') +class TestSocketObjectVersions(unittest.TestCase): + + def setUp(self): + self.prolis = prolis = listen(('localhost', 0)) + self._orig_prolis = _test_sockets[0] + allowed_headers = ', '.join([ + 'content-encoding', + 'x-object-manifest', + 'content-disposition', + 'foo' + ]) + conf = {'devices': _testdir, 'swift_dir': _testdir, + 'mount_check': 'false', 'allowed_headers': allowed_headers} + prosrv = versioned_writes.VersionedWritesMiddleware( + proxy_logging.ProxyLoggingMiddleware( + _test_servers[0], conf, + logger=_test_servers[0].logger), + {}) + self.coro = spawn(wsgi.server, prolis, prosrv, NullLogger()) + # replace global prosrv with one that's filtered with version + # middleware + global _test_sockets + self.sockets = list(_test_sockets) + self.sockets[0] = prolis + _test_sockets = tuple(self.sockets) + + def tearDown(self): + self.coro.kill() + # put the global state back + global _test_sockets + self.sockets[0] = self._orig_prolis + _test_sockets = tuple(self.sockets) + + def test_version_manifest(self, oc='versions', vc='vers', o='name'): + versions_to_create = 3 + # Create a container for our versioned object testing + (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, + obj2lis, obj3lis) = _test_sockets + pre = quote('%03x' % len(o)) + osub = '%s/sub' % o + presub = quote('%03x' % len(osub)) + osub = quote(osub) + presub = quote(presub) + oc = quote(oc) + vc = quote(vc) + + def put_container(): + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/%s HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Storage-Token: t\r\n' + 'Content-Length: 0\r\nX-Versions-Location: %s\r\n\r\n' + % (oc, vc)) + fd.flush() + headers = readuntil2crlfs(fd) + fd.read() + return headers + + headers = put_container() + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + + def get_container(): + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a/%s HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\n' + 'X-Storage-Token: t\r\n\r\n\r\n' % oc) + fd.flush() + headers = readuntil2crlfs(fd) + body = fd.read() + return headers, body + + # check that the header was set + headers, body = get_container() + exp = 'HTTP/1.1 2' # 2xx series response + self.assertEqual(headers[:len(exp)], exp) + self.assert_('X-Versions-Location: %s' % vc in headers) + + def put_version_container(): + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/%s HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Storage-Token: t\r\n' + 'Content-Length: 0\r\n\r\n' % vc) + fd.flush() + headers = readuntil2crlfs(fd) + fd.read() + return headers + + # make the container for the object versions + headers = put_version_container() + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + + def put(version): + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/%s/%s HTTP/1.1\r\nHost: ' + 'localhost\r\nConnection: close\r\nX-Storage-Token: ' + 't\r\nContent-Length: 5\r\nContent-Type: text/jibberish%s' + '\r\n\r\n%05d\r\n' % (oc, o, version, version)) + fd.flush() + headers = readuntil2crlfs(fd) + fd.read() + return headers + + def get(container=oc, obj=o): + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a/%s/%s HTTP/1.1\r\nHost: ' + 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n' + '\r\n' % (container, obj)) + fd.flush() + headers = readuntil2crlfs(fd) + body = fd.read() + return headers, body + + # Create the versioned file + headers = put(0) + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + + # Create the object versions + for version in range(1, versions_to_create): + sleep(.01) # guarantee that the timestamp changes + headers = put(version) + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + + # Ensure retrieving the manifest file gets the latest version + headers, body = get() + exp = 'HTTP/1.1 200' + self.assertEqual(headers[:len(exp)], exp) + self.assert_('Content-Type: text/jibberish%s' % version in headers) + self.assert_('X-Object-Meta-Foo: barbaz' not in headers) + self.assertEqual(body, '%05d' % version) + + def get_version_container(): + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a/%s HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\n' + 'X-Storage-Token: t\r\n\r\n' % vc) + fd.flush() + headers = readuntil2crlfs(fd) + body = fd.read() + return headers, body + + # Ensure we have the right number of versions saved + headers, body = get_version_container() + exp = 'HTTP/1.1 200' + self.assertEqual(headers[:len(exp)], exp) + versions = [x for x in body.split('\n') if x] + self.assertEqual(len(versions), versions_to_create - 1) + + def delete(): + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('DELETE /v1/a/%s/%s HTTP/1.1\r\nHost: localhost\r' + '\nConnection: close\r\nX-Storage-Token: t\r\n\r\n' + % (oc, o)) + fd.flush() + headers = readuntil2crlfs(fd) + fd.read() + return headers + + def copy(): + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('COPY /v1/a/%s/%s HTTP/1.1\r\nHost: ' + 'localhost\r\nConnection: close\r\nX-Auth-Token: ' + 't\r\nDestination: %s/copied_name\r\n' + 'Content-Length: 0\r\n\r\n' % (oc, o, oc)) + fd.flush() + headers = readuntil2crlfs(fd) + fd.read() + return headers + + # copy a version and make sure the version info is stripped + headers = copy() + exp = 'HTTP/1.1 2' # 2xx series response to the COPY + self.assertEqual(headers[:len(exp)], exp) + + def get_copy(): + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a/%s/copied_name HTTP/1.1\r\nHost: ' + 'localhost\r\nConnection: close\r\n' + 'X-Auth-Token: t\r\n\r\n' % oc) + fd.flush() + headers = readuntil2crlfs(fd) + body = fd.read() + return headers, body + + headers, body = get_copy() + exp = 'HTTP/1.1 200' + self.assertEqual(headers[:len(exp)], exp) + self.assertEqual(body, '%05d' % version) + + def post(): + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('POST /v1/a/%s/%s HTTP/1.1\r\nHost: ' + 'localhost\r\nConnection: close\r\nX-Auth-Token: ' + 't\r\nContent-Type: foo/bar\r\nContent-Length: 0\r\n' + 'X-Object-Meta-Bar: foo\r\n\r\n' % (oc, o)) + fd.flush() + headers = readuntil2crlfs(fd) + fd.read() + return headers + + # post and make sure it's updated + headers = post() + exp = 'HTTP/1.1 2' # 2xx series response to the POST + self.assertEqual(headers[:len(exp)], exp) + + headers, body = get() + self.assert_('Content-Type: foo/bar' in headers) + self.assert_('X-Object-Meta-Bar: foo' in headers) + self.assertEqual(body, '%05d' % version) + + # check container listing + headers, body = get_container() + exp = 'HTTP/1.1 200' + self.assertEqual(headers[:len(exp)], exp) + + # Delete the object versions + for segment in range(versions_to_create - 1, 0, -1): + + headers = delete() + exp = 'HTTP/1.1 2' # 2xx series response + self.assertEqual(headers[:len(exp)], exp) + + # Ensure retrieving the manifest file gets the latest version + headers, body = get() + exp = 'HTTP/1.1 200' + self.assertEqual(headers[:len(exp)], exp) + self.assert_('Content-Type: text/jibberish%s' % (segment - 1) + in headers) + self.assertEqual(body, '%05d' % (segment - 1)) + # Ensure we have the right number of versions saved + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a/%s?prefix=%s%s/ HTTP/1.1\r\nHost: ' + 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r' + '\n' % (vc, pre, o)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 2' # 2xx series response + self.assertEqual(headers[:len(exp)], exp) + body = fd.read() + versions = [x for x in body.split('\n') if x] + self.assertEqual(len(versions), segment - 1) + + # there is now one version left (in the manifest) + # Ensure we have no saved versions + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a/%s?prefix=%s%s/ HTTP/1.1\r\nHost: ' + 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n' + % (vc, pre, o)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 204 No Content' + self.assertEqual(headers[:len(exp)], exp) + + # delete the last version + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('DELETE /v1/a/%s/%s HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Storage-Token: t\r\n\r\n' % (oc, o)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 2' # 2xx series response + self.assertEqual(headers[:len(exp)], exp) + + # Ensure it's all gone + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a/%s/%s HTTP/1.1\r\nHost: ' + 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n' + % (oc, o)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 404' + self.assertEqual(headers[:len(exp)], exp) + + # make sure manifest files will be ignored + for _junk in range(1, versions_to_create): + sleep(.01) # guarantee that the timestamp changes + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/%s/%s HTTP/1.1\r\nHost: ' + 'localhost\r\nConnection: close\r\nX-Storage-Token: ' + 't\r\nContent-Length: 0\r\n' + 'Content-Type: text/jibberish0\r\n' + 'Foo: barbaz\r\nX-Object-Manifest: %s/%s/\r\n\r\n' + % (oc, o, oc, o)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a/%s?prefix=%s%s/ HTTP/1.1\r\nhost: ' + 'localhost\r\nconnection: close\r\nx-auth-token: t\r\n\r\n' + % (vc, pre, o)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 204 No Content' + self.assertEqual(headers[:len(exp)], exp) + + # DELETE v1/a/c/obj shouldn't delete v1/a/c/obj/sub versions + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/%s/%s HTTP/1.1\r\nHost: ' + 'localhost\r\nConnection: close\r\nX-Storage-Token: ' + 't\r\nContent-Length: 5\r\nContent-Type: text/jibberish0\r\n' + 'Foo: barbaz\r\n\r\n00000\r\n' % (oc, o)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/%s/%s HTTP/1.1\r\nHost: ' + 'localhost\r\nConnection: close\r\nX-Storage-Token: ' + 't\r\nContent-Length: 5\r\nContent-Type: text/jibberish0\r\n' + 'Foo: barbaz\r\n\r\n00001\r\n' % (oc, o)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/%s/%s HTTP/1.1\r\nHost: ' + 'localhost\r\nConnection: close\r\nX-Storage-Token: ' + 't\r\nContent-Length: 4\r\nContent-Type: text/jibberish0\r\n' + 'Foo: barbaz\r\n\r\nsub1\r\n' % (oc, osub)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/%s/%s HTTP/1.1\r\nHost: ' + 'localhost\r\nConnection: close\r\nX-Storage-Token: ' + 't\r\nContent-Length: 4\r\nContent-Type: text/jibberish0\r\n' + 'Foo: barbaz\r\n\r\nsub2\r\n' % (oc, osub)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('DELETE /v1/a/%s/%s HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Storage-Token: t\r\n\r\n' % (oc, o)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 2' # 2xx series response + self.assertEqual(headers[:len(exp)], exp) + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a/%s?prefix=%s%s/ HTTP/1.1\r\nHost: ' + 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n' + % (vc, presub, osub)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 2' # 2xx series response + self.assertEqual(headers[:len(exp)], exp) + body = fd.read() + versions = [x for x in body.split('\n') if x] + self.assertEqual(len(versions), 1) + + # Check for when the versions target container doesn't exist + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/%swhoops HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Storage-Token: t\r\n' + 'Content-Length: 0\r\nX-Versions-Location: none\r\n\r\n' % oc) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + # Create the versioned file + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/%swhoops/foo HTTP/1.1\r\nHost: ' + 'localhost\r\nConnection: close\r\nX-Storage-Token: ' + 't\r\nContent-Length: 5\r\n\r\n00000\r\n' % oc) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + # Create another version + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/%swhoops/foo HTTP/1.1\r\nHost: ' + 'localhost\r\nConnection: close\r\nX-Storage-Token: ' + 't\r\nContent-Length: 5\r\n\r\n00001\r\n' % oc) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 412' + self.assertEqual(headers[:len(exp)], exp) + # Delete the object + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('DELETE /v1/a/%swhoops/foo HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Storage-Token: t\r\n\r\n' % oc) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 2' # 2xx response + self.assertEqual(headers[:len(exp)], exp) + + def test_version_manifest_utf8(self): + oc = '0_oc_non_ascii\xc2\xa3' + vc = '0_vc_non_ascii\xc2\xa3' + o = '0_o_non_ascii\xc2\xa3' + self.test_version_manifest(oc, vc, o) + + def test_version_manifest_utf8_container(self): + oc = '1_oc_non_ascii\xc2\xa3' + vc = '1_vc_ascii' + o = '1_o_ascii' + self.test_version_manifest(oc, vc, o) + + def test_version_manifest_utf8_version_container(self): + oc = '2_oc_ascii' + vc = '2_vc_non_ascii\xc2\xa3' + o = '2_o_ascii' + self.test_version_manifest(oc, vc, o) + + def test_version_manifest_utf8_containers(self): + oc = '3_oc_non_ascii\xc2\xa3' + vc = '3_vc_non_ascii\xc2\xa3' + o = '3_o_ascii' + self.test_version_manifest(oc, vc, o) + + def test_version_manifest_utf8_object(self): + oc = '4_oc_ascii' + vc = '4_vc_ascii' + o = '4_o_non_ascii\xc2\xa3' + self.test_version_manifest(oc, vc, o) + + def test_version_manifest_utf8_version_container_utf_object(self): + oc = '5_oc_ascii' + vc = '5_vc_non_ascii\xc2\xa3' + o = '5_o_non_ascii\xc2\xa3' + self.test_version_manifest(oc, vc, o) + + def test_version_manifest_utf8_container_utf_object(self): + oc = '6_oc_non_ascii\xc2\xa3' + vc = '6_vc_ascii' + o = '6_o_non_ascii\xc2\xa3' + self.test_version_manifest(oc, vc, o) + + if __name__ == '__main__': setup() try: