diff --git a/doc/source/misc.rst b/doc/source/misc.rst index b4e7de093e..5db87d7075 100644 --- a/doc/source/misc.rst +++ b/doc/source/misc.rst @@ -179,3 +179,10 @@ Bulk Operations (Delete and Archive Auto Extraction) :members: :show-inheritance: +Container Quotas +============= + +.. automodule:: swift.common.middleware.container_quotas + :members: + :show-inheritance: + diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index 6086a87995..f7a7a32aff 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -33,7 +33,7 @@ # eventlet_debug = false [pipeline:main] -pipeline = catch_errors healthcheck cache ratelimit tempauth proxy-logging proxy-server +pipeline = catch_errors healthcheck cache ratelimit tempauth container-quotas proxy-logging proxy-server [app:proxy-server] use = egg:swift#proxy @@ -338,3 +338,7 @@ use = egg:swift#bulk # max_containers_per_extraction = 10000 # max_failed_files = 1000 # max_deletes_per_request = 1000 + +# Note: Put after auth in the pipeline. +[filter:container-quotas] +use = egg:swift#container_quotas diff --git a/setup.py b/setup.py index 35ddb0e08e..9914ea7b8e 100644 --- a/setup.py +++ b/setup.py @@ -99,9 +99,11 @@ setup( 'tempurl=swift.common.middleware.tempurl:filter_factory', 'formpost=swift.common.middleware.formpost:filter_factory', 'name_check=swift.common.middleware.name_check:filter_factory', - 'proxy_logging=' - 'swift.common.middleware.proxy_logging:filter_factory', 'bulk=swift.common.middleware.bulk:filter_factory', + 'container_quotas=swift.common.middleware.container_quotas:' + 'filter_factory', + 'proxy_logging=swift.common.middleware.proxy_logging:' + 'filter_factory', ], }, ) diff --git a/swift/common/middleware/container_quotas.py b/swift/common/middleware/container_quotas.py new file mode 100644 index 0000000000..a9504f9516 --- /dev/null +++ b/swift/common/middleware/container_quotas.py @@ -0,0 +1,107 @@ +# Copyright (c) 2010-2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +The ``container_quotas`` middleware implements simple quotas that can be +imposed on swift containers by a user with the ability to set container +metadata, most likely the account administrator. This can be useful for +limiting the scope of containers that are delegated to non-admin users, exposed +to ``formpost`` uploads, or just as a self-imposed sanity check. + +Any object PUT operations that exceed these quotas return a 413 response +(request entity too large) with a descriptive body. + +Quotas are subject to several limitations: eventual consistency, the timeliness +of the cached container_info (60 second ttl by default), and it's unable to +reject chunked transfer uploads that exceed the quota (though once the quota +is exceeded, new chunked transfers will be refused). + +Quotas are set by adding meta values to the container, and are validated when +set: + ++---------------------------------------------+-------------------------------+ +|Metadata | Use | ++=============================================+===============================+ +| X-Container-Meta-Quota-Bytes | Maximum size of the | +| | container, in bytes. | ++---------------------------------------------+-------------------------------+ +| X-Container-Meta-Quota-Count | Maximum object count of the | +| | container. | ++---------------------------------------------+-------------------------------+ +""" + +from swift.common.utils import split_path +from swift.common.http import is_success +from swift.proxy.controllers.base import get_container_info +from swift.common.swob import Response, HTTPBadRequest, wsgify + + +class ContainerQuotaMiddleware(object): + def __init__(self, app, *args, **kwargs): + self.app = app + + def bad_response(self, req, container_info): + # 401 if the user couldn't have PUT this object in the first place. + # This prevents leaking the container's existence to unauthed users. + if 'swift.authorize' in req.environ: + req.acl = container_info['write_acl'] + aresp = req.environ['swift.authorize'](req) + if aresp: + return aresp + return Response(status=413, body='Upload exceeds quota.') + + @wsgify + def __call__(self, req): + try: + (version, account, container, obj) = req.split_path(2, 4, True) + except ValueError: + return self.app + + # verify new quota headers are properly formatted + if container and not obj and req.method in ('PUT', 'POST'): + val = req.headers.get('X-Container-Meta-Quota-Bytes') + if val and not val.isdigit(): + return HTTPBadRequest(body='Invalid bytes quota.') + val = req.headers.get('X-Container-Meta-Quota-Count') + if val and not val.isdigit(): + return HTTPBadRequest(body='Invalid count quota.') + + # check user uploads against quotas + elif obj and req.method == 'PUT': + container_info = get_container_info(req.environ, self.app) + if not container_info or not is_success(container_info['status']): + # this will hopefully 404 later + return self.app + if 'quota-bytes' in container_info.get('meta', {}) and \ + 'bytes' in container_info and \ + container_info['meta']['quota-bytes'].isdigit(): + new_size = int(container_info['bytes']) + (req.content_length + or 0) + if int(container_info['meta']['quota-bytes']) < new_size: + return self.bad_response(req, container_info) + if 'quota-count' in container_info.get('meta', {}) and \ + 'count' in container_info and \ + container_info['meta']['quota-count'].isdigit(): + new_count = int(container_info['count']) + 1 + if int(container_info['meta']['quota-count']) < new_count: + return self.bad_response(req, container_info) + + return self.app + + +def filter_factory(global_conf, **local_conf): + def container_quota_filter(app): + return ContainerQuotaMiddleware(app) + return container_quota_filter diff --git a/swift/proxy/controllers/base.py b/swift/proxy/controllers/base.py index 7e0dd82bac..dce9e4fdda 100644 --- a/swift/proxy/controllers/base.py +++ b/swift/proxy/controllers/base.py @@ -32,7 +32,9 @@ from eventlet import spawn_n, GreenPile from eventlet.queue import Queue, Empty, Full from eventlet.timeout import Timeout -from swift.common.utils import normalize_timestamp, config_true_value, public +from swift.common.wsgi import make_pre_authed_request +from swift.common.utils import normalize_timestamp, config_true_value, \ + public, split_path, cache_from_env from swift.common.bufferedhttp import http_connect from swift.common.constraints import MAX_ACCOUNT_NAME_LENGTH from swift.common.exceptions import ChunkReadTimeout, ConnectionTimeout @@ -188,6 +190,32 @@ def cors_validation(func): return wrapped +def get_container_info(env, app): + """ + Get the info structure for a container, based on env and app. + This is useful to middlewares. + """ + cache = cache_from_env(env) + if not cache: + return None + (version, account, container, obj) = \ + split_path(env['PATH_INFO'], 2, 4, True) + cache_key = get_container_memcache_key(account, container) + # Use a unique environment cache key per container. If you copy this env + # to make a new request, it won't accidentally reuse the old container info + env_key = 'swift.%s' % cache_key + if env_key not in env: + container_info = cache.get(cache_key) + if not container_info: + resp = make_pre_authed_request( + env, 'HEAD', '/%s/%s/%s' % (version, account, container) + ).get_response(app) + container_info = headers_to_container_info( + resp.headers, resp.status_int) + env[env_key] = container_info + return env[env_key] + + class Controller(object): """Base WSGI controller class for the proxy""" server_type = 'Base' diff --git a/test/unit/common/middleware/test_quotas.py b/test/unit/common/middleware/test_quotas.py new file mode 100644 index 0000000000..ccb40110fd --- /dev/null +++ b/test/unit/common/middleware/test_quotas.py @@ -0,0 +1,165 @@ +# Copyright (c) 2010-2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from swift.common.swob import Request, Response, HTTPUnauthorized +from swift.common.middleware import container_quotas + +class FakeCache(object): + def __init__(self, val): + if 'status' not in val: + val['status'] = 200 + self.val = val + + def get(self, *args): + return self.val + +class FakeApp(object): + def __init__(self): + pass + + def __call__(self, env, start_response): + start_response('200 OK', []) + return [] + +class FakeMissingApp(object): + def __init__(self): + pass + + def __call__(self, env, start_response): + start_response('404 Not Found', []) + return [] + +def start_response(*args): + pass + +class TestContainerQuotas(unittest.TestCase): + + def test_not_handled(self): + app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) + req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'PUT'}) + res = req.get_response(app) + self.assertEquals(res.status_int, 200) + + app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) + res = req.get_response(app) + self.assertEquals(res.status_int, 200) + + def test_no_quotas(self): + app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', 'swift.cache': FakeCache({}), + 'CONTENT_LENGTH': '100'}) + res = req.get_response(app) + self.assertEquals(res.status_int, 200) + + def test_exceed_bytes_quota(self): + app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) + cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '2'}}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache, + 'CONTENT_LENGTH': '100'}) + res = req.get_response(app) + self.assertEquals(res.status_int, 413) + + def test_not_exceed_bytes_quota(self): + app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) + cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache, + 'CONTENT_LENGTH': '100'}) + res = req.get_response(app) + self.assertEquals(res.status_int, 200) + + def test_exceed_counts_quota(self): + app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) + cache = FakeCache({'count': 1, 'meta': {'quota-count': '1'}}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache, + 'CONTENT_LENGTH': '100'}) + res = req.get_response(app) + self.assertEquals(res.status_int, 413) + + def test_not_exceed_counts_quota(self): + app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) + cache = FakeCache({'count': 1, 'meta': {'quota-count': '2'}}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache, + 'CONTENT_LENGTH': '100'}) + res = req.get_response(app) + self.assertEquals(res.status_int, 200) + + def test_invalid_quotas(self): + req = Request.blank('/v1/a/c', + environ={'REQUEST_METHOD': 'POST', + 'HTTP_X_CONTAINER_META_QUOTA_BYTES': 'abc'}) + res = req.get_response( + container_quotas.ContainerQuotaMiddleware(FakeApp(), {})) + self.assertEquals(res.status_int, 400) + + req = Request.blank('/v1/a/c', + environ={'REQUEST_METHOD': 'POST', + 'HTTP_X_CONTAINER_META_QUOTA_COUNT': 'abc'}) + res = req.get_response( + container_quotas.ContainerQuotaMiddleware(FakeApp(), {})) + self.assertEquals(res.status_int, 400) + + def test_valid_quotas(self): + req = Request.blank('/v1/a/c', + environ={'REQUEST_METHOD': 'POST', + 'HTTP_X_CONTAINER_META_QUOTA_BYTES': '123'}) + res = req.get_response( + container_quotas.ContainerQuotaMiddleware(FakeApp(), {})) + self.assertEquals(res.status_int, 200) + + req = Request.blank('/v1/a/c', + environ={'REQUEST_METHOD': 'POST', + 'HTTP_X_CONTAINER_META_QUOTA_COUNT': '123'}) + res = req.get_response( + container_quotas.ContainerQuotaMiddleware(FakeApp(), {})) + self.assertEquals(res.status_int, 200) + + def test_delete_quotas(self): + req = Request.blank('/v1/a/c', + environ={'REQUEST_METHOD': 'POST', + 'HTTP_X_CONTAINER_META_QUOTA_BYTES': None}) + res = req.get_response( + container_quotas.ContainerQuotaMiddleware(FakeApp(), {})) + self.assertEquals(res.status_int, 200) + + def test_missing_container(self): + app = container_quotas.ContainerQuotaMiddleware(FakeMissingApp(), {}) + cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache, + 'CONTENT_LENGTH': '100'}) + res = req.get_response(app) + self.assertEquals(res.status_int, 404) + + def test_auth_fail(self): + app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) + cache = FakeCache({'count': 1, 'meta': {'quota-count': '1'}, + 'write_acl': None}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache, + 'CONTENT_LENGTH': '100', + 'swift.authorize': lambda *args: HTTPUnauthorized()}) + res = req.get_response(app) + self.assertEquals(res.status_int, 401) + +if __name__ == '__main__': + unittest.main()