Basic container quotas
Add a new middleware implementing some basic container quotas. Quotas are subject to several limitations: eventual consistency, the timeliness of the cached container_info (60 second ttl by default), and it’s unable to reject chunked transfer uploads that exceed the quota (though once the quota is exceeded, new chunked transfers will be refused). However, they get most of the way to container quotas fairly inexpensively. Quotas are set by adding meta values to the container, and are validated when set: X-Container-Meta-Quota-Bytes: Maximum size of the container, in bytes. X-Container-Meta-Quota-Count: Maximum object count of the container. DocImpact Change-Id: I77cfbf6dc231a2e522bd67328e4c082424a93eee
This commit is contained in:
Michael Barton
parent
65baec39d2
commit
24ef12027c
@ -179,3 +179,10 @@ Bulk Operations (Delete and Archive Auto Extraction)
|
||||
:members:
|
||||
:show-inheritance:
|
||||
|
||||
Container Quotas
|
||||
=============
|
||||
|
||||
.. automodule:: swift.common.middleware.container_quotas
|
||||
:members:
|
||||
:show-inheritance:
|
||||
|
||||
|
||||
@ -33,7 +33,7 @@
|
||||
# eventlet_debug = false
|
||||
|
||||
[pipeline:main]
|
||||
pipeline = catch_errors healthcheck cache ratelimit tempauth proxy-logging proxy-server
|
||||
pipeline = catch_errors healthcheck cache ratelimit tempauth container-quotas proxy-logging proxy-server
|
||||
|
||||
[app:proxy-server]
|
||||
use = egg:swift#proxy
|
||||
@ -338,3 +338,7 @@ use = egg:swift#bulk
|
||||
# max_containers_per_extraction = 10000
|
||||
# max_failed_files = 1000
|
||||
# max_deletes_per_request = 1000
|
||||
|
||||
# Note: Put after auth in the pipeline.
|
||||
[filter:container-quotas]
|
||||
use = egg:swift#container_quotas
|
||||
|
||||
6
setup.py
6
setup.py
@ -99,9 +99,11 @@ setup(
|
||||
'tempurl=swift.common.middleware.tempurl:filter_factory',
|
||||
'formpost=swift.common.middleware.formpost:filter_factory',
|
||||
'name_check=swift.common.middleware.name_check:filter_factory',
|
||||
'proxy_logging='
|
||||
'swift.common.middleware.proxy_logging:filter_factory',
|
||||
'bulk=swift.common.middleware.bulk:filter_factory',
|
||||
'container_quotas=swift.common.middleware.container_quotas:'
|
||||
'filter_factory',
|
||||
'proxy_logging=swift.common.middleware.proxy_logging:'
|
||||
'filter_factory',
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
107
swift/common/middleware/container_quotas.py
Normal file
107
swift/common/middleware/container_quotas.py
Normal file
@ -0,0 +1,107 @@
|
||||
# Copyright (c) 2010-2012 OpenStack, LLC.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
The ``container_quotas`` middleware implements simple quotas that can be
|
||||
imposed on swift containers by a user with the ability to set container
|
||||
metadata, most likely the account administrator. This can be useful for
|
||||
limiting the scope of containers that are delegated to non-admin users, exposed
|
||||
to ``formpost`` uploads, or just as a self-imposed sanity check.
|
||||
|
||||
Any object PUT operations that exceed these quotas return a 413 response
|
||||
(request entity too large) with a descriptive body.
|
||||
|
||||
Quotas are subject to several limitations: eventual consistency, the timeliness
|
||||
of the cached container_info (60 second ttl by default), and it's unable to
|
||||
reject chunked transfer uploads that exceed the quota (though once the quota
|
||||
is exceeded, new chunked transfers will be refused).
|
||||
|
||||
Quotas are set by adding meta values to the container, and are validated when
|
||||
set:
|
||||
|
||||
+---------------------------------------------+-------------------------------+
|
||||
|Metadata | Use |
|
||||
+=============================================+===============================+
|
||||
| X-Container-Meta-Quota-Bytes | Maximum size of the |
|
||||
| | container, in bytes. |
|
||||
+---------------------------------------------+-------------------------------+
|
||||
| X-Container-Meta-Quota-Count | Maximum object count of the |
|
||||
| | container. |
|
||||
+---------------------------------------------+-------------------------------+
|
||||
"""
|
||||
|
||||
from swift.common.utils import split_path
|
||||
from swift.common.http import is_success
|
||||
from swift.proxy.controllers.base import get_container_info
|
||||
from swift.common.swob import Response, HTTPBadRequest, wsgify
|
||||
|
||||
|
||||
class ContainerQuotaMiddleware(object):
|
||||
def __init__(self, app, *args, **kwargs):
|
||||
self.app = app
|
||||
|
||||
def bad_response(self, req, container_info):
|
||||
# 401 if the user couldn't have PUT this object in the first place.
|
||||
# This prevents leaking the container's existence to unauthed users.
|
||||
if 'swift.authorize' in req.environ:
|
||||
req.acl = container_info['write_acl']
|
||||
aresp = req.environ['swift.authorize'](req)
|
||||
if aresp:
|
||||
return aresp
|
||||
return Response(status=413, body='Upload exceeds quota.')
|
||||
|
||||
@wsgify
|
||||
def __call__(self, req):
|
||||
try:
|
||||
(version, account, container, obj) = req.split_path(2, 4, True)
|
||||
except ValueError:
|
||||
return self.app
|
||||
|
||||
# verify new quota headers are properly formatted
|
||||
if container and not obj and req.method in ('PUT', 'POST'):
|
||||
val = req.headers.get('X-Container-Meta-Quota-Bytes')
|
||||
if val and not val.isdigit():
|
||||
return HTTPBadRequest(body='Invalid bytes quota.')
|
||||
val = req.headers.get('X-Container-Meta-Quota-Count')
|
||||
if val and not val.isdigit():
|
||||
return HTTPBadRequest(body='Invalid count quota.')
|
||||
|
||||
# check user uploads against quotas
|
||||
elif obj and req.method == 'PUT':
|
||||
container_info = get_container_info(req.environ, self.app)
|
||||
if not container_info or not is_success(container_info['status']):
|
||||
# this will hopefully 404 later
|
||||
return self.app
|
||||
if 'quota-bytes' in container_info.get('meta', {}) and \
|
||||
'bytes' in container_info and \
|
||||
container_info['meta']['quota-bytes'].isdigit():
|
||||
new_size = int(container_info['bytes']) + (req.content_length
|
||||
or 0)
|
||||
if int(container_info['meta']['quota-bytes']) < new_size:
|
||||
return self.bad_response(req, container_info)
|
||||
if 'quota-count' in container_info.get('meta', {}) and \
|
||||
'count' in container_info and \
|
||||
container_info['meta']['quota-count'].isdigit():
|
||||
new_count = int(container_info['count']) + 1
|
||||
if int(container_info['meta']['quota-count']) < new_count:
|
||||
return self.bad_response(req, container_info)
|
||||
|
||||
return self.app
|
||||
|
||||
|
||||
def filter_factory(global_conf, **local_conf):
|
||||
def container_quota_filter(app):
|
||||
return ContainerQuotaMiddleware(app)
|
||||
return container_quota_filter
|
||||
@ -32,7 +32,9 @@ from eventlet import spawn_n, GreenPile
|
||||
from eventlet.queue import Queue, Empty, Full
|
||||
from eventlet.timeout import Timeout
|
||||
|
||||
from swift.common.utils import normalize_timestamp, config_true_value, public
|
||||
from swift.common.wsgi import make_pre_authed_request
|
||||
from swift.common.utils import normalize_timestamp, config_true_value, \
|
||||
public, split_path, cache_from_env
|
||||
from swift.common.bufferedhttp import http_connect
|
||||
from swift.common.constraints import MAX_ACCOUNT_NAME_LENGTH
|
||||
from swift.common.exceptions import ChunkReadTimeout, ConnectionTimeout
|
||||
@ -188,6 +190,32 @@ def cors_validation(func):
|
||||
return wrapped
|
||||
|
||||
|
||||
def get_container_info(env, app):
|
||||
"""
|
||||
Get the info structure for a container, based on env and app.
|
||||
This is useful to middlewares.
|
||||
"""
|
||||
cache = cache_from_env(env)
|
||||
if not cache:
|
||||
return None
|
||||
(version, account, container, obj) = \
|
||||
split_path(env['PATH_INFO'], 2, 4, True)
|
||||
cache_key = get_container_memcache_key(account, container)
|
||||
# Use a unique environment cache key per container. If you copy this env
|
||||
# to make a new request, it won't accidentally reuse the old container info
|
||||
env_key = 'swift.%s' % cache_key
|
||||
if env_key not in env:
|
||||
container_info = cache.get(cache_key)
|
||||
if not container_info:
|
||||
resp = make_pre_authed_request(
|
||||
env, 'HEAD', '/%s/%s/%s' % (version, account, container)
|
||||
).get_response(app)
|
||||
container_info = headers_to_container_info(
|
||||
resp.headers, resp.status_int)
|
||||
env[env_key] = container_info
|
||||
return env[env_key]
|
||||
|
||||
|
||||
class Controller(object):
|
||||
"""Base WSGI controller class for the proxy"""
|
||||
server_type = 'Base'
|
||||
|
||||
165
test/unit/common/middleware/test_quotas.py
Normal file
165
test/unit/common/middleware/test_quotas.py
Normal file
@ -0,0 +1,165 @@
|
||||
# Copyright (c) 2010-2012 OpenStack, LLC.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import unittest
|
||||
|
||||
from swift.common.swob import Request, Response, HTTPUnauthorized
|
||||
from swift.common.middleware import container_quotas
|
||||
|
||||
class FakeCache(object):
|
||||
def __init__(self, val):
|
||||
if 'status' not in val:
|
||||
val['status'] = 200
|
||||
self.val = val
|
||||
|
||||
def get(self, *args):
|
||||
return self.val
|
||||
|
||||
class FakeApp(object):
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def __call__(self, env, start_response):
|
||||
start_response('200 OK', [])
|
||||
return []
|
||||
|
||||
class FakeMissingApp(object):
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def __call__(self, env, start_response):
|
||||
start_response('404 Not Found', [])
|
||||
return []
|
||||
|
||||
def start_response(*args):
|
||||
pass
|
||||
|
||||
class TestContainerQuotas(unittest.TestCase):
|
||||
|
||||
def test_not_handled(self):
|
||||
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
|
||||
req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'PUT'})
|
||||
res = req.get_response(app)
|
||||
self.assertEquals(res.status_int, 200)
|
||||
|
||||
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
|
||||
req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
|
||||
res = req.get_response(app)
|
||||
self.assertEquals(res.status_int, 200)
|
||||
|
||||
def test_no_quotas(self):
|
||||
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
|
||||
req = Request.blank('/v1/a/c/o',
|
||||
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': FakeCache({}),
|
||||
'CONTENT_LENGTH': '100'})
|
||||
res = req.get_response(app)
|
||||
self.assertEquals(res.status_int, 200)
|
||||
|
||||
def test_exceed_bytes_quota(self):
|
||||
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
|
||||
cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '2'}})
|
||||
req = Request.blank('/v1/a/c/o',
|
||||
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
|
||||
'CONTENT_LENGTH': '100'})
|
||||
res = req.get_response(app)
|
||||
self.assertEquals(res.status_int, 413)
|
||||
|
||||
def test_not_exceed_bytes_quota(self):
|
||||
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
|
||||
cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}})
|
||||
req = Request.blank('/v1/a/c/o',
|
||||
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
|
||||
'CONTENT_LENGTH': '100'})
|
||||
res = req.get_response(app)
|
||||
self.assertEquals(res.status_int, 200)
|
||||
|
||||
def test_exceed_counts_quota(self):
|
||||
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
|
||||
cache = FakeCache({'count': 1, 'meta': {'quota-count': '1'}})
|
||||
req = Request.blank('/v1/a/c/o',
|
||||
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
|
||||
'CONTENT_LENGTH': '100'})
|
||||
res = req.get_response(app)
|
||||
self.assertEquals(res.status_int, 413)
|
||||
|
||||
def test_not_exceed_counts_quota(self):
|
||||
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
|
||||
cache = FakeCache({'count': 1, 'meta': {'quota-count': '2'}})
|
||||
req = Request.blank('/v1/a/c/o',
|
||||
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
|
||||
'CONTENT_LENGTH': '100'})
|
||||
res = req.get_response(app)
|
||||
self.assertEquals(res.status_int, 200)
|
||||
|
||||
def test_invalid_quotas(self):
|
||||
req = Request.blank('/v1/a/c',
|
||||
environ={'REQUEST_METHOD': 'POST',
|
||||
'HTTP_X_CONTAINER_META_QUOTA_BYTES': 'abc'})
|
||||
res = req.get_response(
|
||||
container_quotas.ContainerQuotaMiddleware(FakeApp(), {}))
|
||||
self.assertEquals(res.status_int, 400)
|
||||
|
||||
req = Request.blank('/v1/a/c',
|
||||
environ={'REQUEST_METHOD': 'POST',
|
||||
'HTTP_X_CONTAINER_META_QUOTA_COUNT': 'abc'})
|
||||
res = req.get_response(
|
||||
container_quotas.ContainerQuotaMiddleware(FakeApp(), {}))
|
||||
self.assertEquals(res.status_int, 400)
|
||||
|
||||
def test_valid_quotas(self):
|
||||
req = Request.blank('/v1/a/c',
|
||||
environ={'REQUEST_METHOD': 'POST',
|
||||
'HTTP_X_CONTAINER_META_QUOTA_BYTES': '123'})
|
||||
res = req.get_response(
|
||||
container_quotas.ContainerQuotaMiddleware(FakeApp(), {}))
|
||||
self.assertEquals(res.status_int, 200)
|
||||
|
||||
req = Request.blank('/v1/a/c',
|
||||
environ={'REQUEST_METHOD': 'POST',
|
||||
'HTTP_X_CONTAINER_META_QUOTA_COUNT': '123'})
|
||||
res = req.get_response(
|
||||
container_quotas.ContainerQuotaMiddleware(FakeApp(), {}))
|
||||
self.assertEquals(res.status_int, 200)
|
||||
|
||||
def test_delete_quotas(self):
|
||||
req = Request.blank('/v1/a/c',
|
||||
environ={'REQUEST_METHOD': 'POST',
|
||||
'HTTP_X_CONTAINER_META_QUOTA_BYTES': None})
|
||||
res = req.get_response(
|
||||
container_quotas.ContainerQuotaMiddleware(FakeApp(), {}))
|
||||
self.assertEquals(res.status_int, 200)
|
||||
|
||||
def test_missing_container(self):
|
||||
app = container_quotas.ContainerQuotaMiddleware(FakeMissingApp(), {})
|
||||
cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}})
|
||||
req = Request.blank('/v1/a/c/o',
|
||||
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
|
||||
'CONTENT_LENGTH': '100'})
|
||||
res = req.get_response(app)
|
||||
self.assertEquals(res.status_int, 404)
|
||||
|
||||
def test_auth_fail(self):
|
||||
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
|
||||
cache = FakeCache({'count': 1, 'meta': {'quota-count': '1'},
|
||||
'write_acl': None})
|
||||
req = Request.blank('/v1/a/c/o',
|
||||
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
|
||||
'CONTENT_LENGTH': '100',
|
||||
'swift.authorize': lambda *args: HTTPUnauthorized()})
|
||||
res = req.get_response(app)
|
||||
self.assertEquals(res.status_int, 401)
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Loading…
Reference in New Issue
Block a user