From b62299376a5a6ead5d547e6c00a74faf011442e5 Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Fri, 8 Feb 2013 11:48:26 +0100 Subject: [PATCH] Account and container info fixes and improvement. - Fixes bug 1119282. - Allow middleware accessing metadata of an account without having to store it separately in a new memcache namespace. - Add tests for get_container_info that was previously missed. - Add get_account_info method based on get_container_info, a function for other middleware to query accounts. - Rename container_info['count'] as container_info['object_count']. Change-Id: I43787916c7a812cb08d278edf45370521f12c912 --- .mailmap | 1 + swift/common/middleware/container_quotas.py | 5 +- swift/proxy/controllers/base.py | 71 ++++++++++-- test/unit/common/middleware/test_quotas.py | 6 +- test/unit/proxy/controllers/test_base.py | 121 +++++++++++++++++++- test/unit/proxy/test_server.py | 14 ++- 6 files changed, 197 insertions(+), 21 deletions(-) diff --git a/.mailmap b/.mailmap index 651622fa95..727e4772dc 100644 --- a/.mailmap +++ b/.mailmap @@ -17,6 +17,7 @@ Anne Gentle annegentle Fujita Tomonori Greg Lange Greg Lange +Chmouel Boudjnah Gaurav B. Gangalwar gaurav@gluster.com <> Joe Arnold Kapil Thangavelu kapil.foss@gmail.com <> diff --git a/swift/common/middleware/container_quotas.py b/swift/common/middleware/container_quotas.py index 0c0d539d00..34643a7bbc 100644 --- a/swift/common/middleware/container_quotas.py +++ b/swift/common/middleware/container_quotas.py @@ -42,7 +42,6 @@ set: +---------------------------------------------+-------------------------------+ """ -from swift.common.utils import split_path from swift.common.http import is_success from swift.proxy.controllers.base import get_container_info from swift.common.swob import Response, HTTPBadRequest, wsgify @@ -93,9 +92,9 @@ class ContainerQuotaMiddleware(object): if int(container_info['meta']['quota-bytes']) < new_size: return self.bad_response(req, container_info) if 'quota-count' in container_info.get('meta', {}) and \ - 'count' in container_info and \ + 'object_count' in container_info and \ container_info['meta']['quota-count'].isdigit(): - new_count = int(container_info['count']) + 1 + new_count = int(container_info['object_count']) + 1 if int(container_info['meta']['quota-count']) < new_count: return self.bad_response(req, container_info) diff --git a/swift/proxy/controllers/base.py b/swift/proxy/controllers/base.py index f4ed3f5af6..a867182ede 100644 --- a/swift/proxy/controllers/base.py +++ b/swift/proxy/controllers/base.py @@ -97,17 +97,33 @@ def get_container_memcache_key(account, container): return 'container/%s/%s' % (account, container) +def headers_to_account_info(headers, status_int=HTTP_OK): + """ + Construct a cacheable dict of account info based on response headers. + """ + headers = dict((k.lower(), v) for k, v in dict(headers).iteritems()) + return { + 'status': status_int, + 'container_count': headers.get('x-account-container-count'), + 'total_object_count': headers.get('x-account-object-count'), + 'bytes': headers.get('x-account-bytes-used'), + 'meta': dict((key[15:], value) + for key, value in headers.iteritems() + if key.startswith('x-account-meta-')) + } + + def headers_to_container_info(headers, status_int=HTTP_OK): """ Construct a cacheable dict of container info based on response headers. """ - headers = dict(headers) + headers = dict((k.lower(), v) for k, v in dict(headers).iteritems()) return { 'status': status_int, 'read_acl': headers.get('x-container-read'), 'write_acl': headers.get('x-container-write'), 'sync_key': headers.get('x-container-sync-key'), - 'count': headers.get('x-container-object-count'), + 'object_count': headers.get('x-container-object-count'), 'bytes': headers.get('x-container-bytes-used'), 'versions': headers.get('x-versions-location'), 'cors': { @@ -120,9 +136,9 @@ def headers_to_container_info(headers, status_int=HTTP_OK): 'max_age': headers.get( 'x-container-meta-access-control-max-age') }, - 'meta': dict((key.lower()[17:], value) + 'meta': dict((key[17:], value) for key, value in headers.iteritems() - if key.lower().startswith('x-container-meta-')) + if key.startswith('x-container-meta-')) } @@ -198,8 +214,8 @@ def get_container_info(env, app, swift_source=None): cache = cache_from_env(env) if not cache: return None - (version, account, container, obj) = \ - split_path(env['PATH_INFO'], 2, 4, True) + (version, account, container, _) = \ + split_path(env['PATH_INFO'], 3, 4, True) cache_key = get_container_memcache_key(account, container) # Use a unique environment cache key per container. If you copy this env # to make a new request, it won't accidentally reuse the old container info @@ -217,6 +233,33 @@ def get_container_info(env, app, swift_source=None): return env[env_key] +def get_account_info(env, app, swift_source=None): + """ + Get the info structure for an account, based on env and app. + This is useful to middlewares. + """ + cache = cache_from_env(env) + if not cache: + return None + (version, account, container, _) = \ + split_path(env['PATH_INFO'], 2, 4, True) + cache_key = get_account_memcache_key(account) + # Use a unique environment cache key per account. If you copy this env + # to make a new request, it won't accidentally reuse the old account info + env_key = 'swift.%s' % cache_key + if env_key not in env: + account_info = cache.get(cache_key) + if not account_info: + resp = make_pre_authed_request( + env, 'HEAD', '/%s/%s' % (version, account), + swift_source=swift_source, + ).get_response(app) + account_info = headers_to_account_info( + resp.headers, resp.status_int) + env[env_key] = account_info + return env[env_key] + + class Controller(object): """Base WSGI controller class for the proxy""" server_type = 'Base' @@ -326,6 +369,11 @@ class Controller(object): or (None, None, None) if it does not exist """ partition, nodes = self.app.account_ring.get_nodes(account) + account_info = {'status': 0, + 'container_count': 0, + 'total_object_count': None, + 'bytes': None, + 'meta': {}} # 0 = no responses, 200 = found, 404 = not found, -1 = mixed responses if self.app.memcache: cache_key = get_account_memcache_key(account) @@ -341,7 +389,6 @@ class Controller(object): elif result_code == HTTP_NOT_FOUND and not autocreate: return None, None, None result_code = 0 - container_count = 0 attempts_left = len(nodes) path = '/%s' % account headers = {'x-trans-id': self.trans_id, 'Connection': 'close'} @@ -362,8 +409,8 @@ class Controller(object): resp.read() if is_success(resp.status): result_code = HTTP_OK - container_count = int( - resp.getheader('x-account-container-count') or 0) + account_info.update( + headers_to_account_info(resp.getheaders())) break elif resp.status == HTTP_NOT_FOUND: if result_code == 0: @@ -398,12 +445,12 @@ class Controller(object): cache_timeout = self.app.recheck_account_existence else: cache_timeout = self.app.recheck_account_existence * 0.1 + account_info.update(status=result_code) self.app.memcache.set(cache_key, - {'status': result_code, - 'container_count': container_count}, + account_info, time=cache_timeout) if result_code == HTTP_OK: - return partition, nodes, container_count + return partition, nodes, account_info['container_count'] return None, None, None def container_info(self, account, container, account_autocreate=False): diff --git a/test/unit/common/middleware/test_quotas.py b/test/unit/common/middleware/test_quotas.py index ccb40110fd..81572b1f7f 100644 --- a/test/unit/common/middleware/test_quotas.py +++ b/test/unit/common/middleware/test_quotas.py @@ -87,7 +87,7 @@ class TestContainerQuotas(unittest.TestCase): def test_exceed_counts_quota(self): app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) - cache = FakeCache({'count': 1, 'meta': {'quota-count': '1'}}) + cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '1'}}) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache, 'CONTENT_LENGTH': '100'}) @@ -96,7 +96,7 @@ class TestContainerQuotas(unittest.TestCase): def test_not_exceed_counts_quota(self): app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) - cache = FakeCache({'count': 1, 'meta': {'quota-count': '2'}}) + cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '2'}}) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache, 'CONTENT_LENGTH': '100'}) @@ -152,7 +152,7 @@ class TestContainerQuotas(unittest.TestCase): def test_auth_fail(self): app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) - cache = FakeCache({'count': 1, 'meta': {'quota-count': '1'}, + cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '1'}, 'write_acl': None}) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache, diff --git a/test/unit/proxy/controllers/test_base.py b/test/unit/proxy/controllers/test_base.py index 13bb11e3c1..1ed742372a 100644 --- a/test/unit/proxy/controllers/test_base.py +++ b/test/unit/proxy/controllers/test_base.py @@ -14,10 +14,101 @@ # limitations under the License. import unittest -from swift.proxy.controllers.base import headers_to_container_info + +import swift.proxy.controllers.base +from swift.proxy.controllers.base import headers_to_container_info, \ + headers_to_account_info, get_container_info, get_container_memcache_key, \ + get_account_info, get_account_memcache_key +from swift.common.swob import Request +from swift.common.utils import split_path + + +class FakeResponse(object): + def __init__(self, headers): + self.headers = headers + self.status_int = 201 + + +class FakeRequest(object): + def __init__(self, env, method, path, swift_source=None): + (version, account, + container, obj) = split_path(env['PATH_INFO'], 2, 4, True) + stype = container and 'container' or 'account' + self.headers = {'x-%s-object-count' % (stype): 1000, + 'x-%s-bytes-used' % (stype): 6666} + + def get_response(self, app): + return FakeResponse(self.headers) + + +class FakeCache(object): + def __init__(self, val): + self.val = val + + def get(self, *args): + return self.val class TestFuncs(unittest.TestCase): + def test_get_container_info_no_cache(self): + swift.proxy.controllers.base.make_pre_authed_request = FakeRequest + req = Request.blank("/v1/AUTH_account/cont", + environ={'swift.cache': FakeCache({})}) + resp = get_container_info(req.environ, 'xxx') + self.assertEquals(resp['bytes'], 6666) + self.assertEquals(resp['object_count'], 1000) + + def test_get_container_info_cache(self): + swift.proxy.controllers.base.make_pre_authed_request = FakeRequest + cached = {'status': 404, + 'bytes': 3333, + 'object_count': 10} + req = Request.blank("/v1/account/cont", + environ={'swift.cache': FakeCache(cached)}) + resp = get_container_info(req.environ, 'xxx') + self.assertEquals(resp['bytes'], 3333) + self.assertEquals(resp['object_count'], 10) + self.assertEquals(resp['status'], 404) + + def test_get_container_info_env(self): + cache_key = get_container_memcache_key("account", "cont") + env_key = 'swift.%s' % cache_key + req = Request.blank("/v1/account/cont", + environ={ env_key: {'bytes': 3867}, + 'swift.cache': FakeCache({})}) + resp = get_container_info(req.environ, 'xxx') + self.assertEquals(resp['bytes'], 3867) + + def test_get_account_info_no_cache(self): + swift.proxy.controllers.base.make_pre_authed_request = FakeRequest + req = Request.blank("/v1/AUTH_account", + environ={'swift.cache': FakeCache({})}) + resp = get_account_info(req.environ, 'xxx') + print resp + self.assertEquals(resp['bytes'], 6666) + self.assertEquals(resp['total_object_count'], 1000) + + def test_get_account_info_cache(self): + swift.proxy.controllers.base.make_pre_authed_request = FakeRequest + cached = {'status': 404, + 'bytes': 3333, + 'total_object_count': 10} + req = Request.blank("/v1/account/cont", + environ={'swift.cache': FakeCache(cached)}) + resp = get_account_info(req.environ, 'xxx') + self.assertEquals(resp['bytes'], 3333) + self.assertEquals(resp['total_object_count'], 10) + self.assertEquals(resp['status'], 404) + + def test_get_account_info_env(self): + cache_key = get_account_memcache_key("account") + env_key = 'swift.%s' % cache_key + req = Request.blank("/v1/account", + environ={ env_key: {'bytes': 3867}, + 'swift.cache': FakeCache({})}) + resp = get_account_info(req.environ, 'xxx') + self.assertEquals(resp['bytes'], 3867) + def test_headers_to_container_info_missing(self): resp = headers_to_container_info({}, 404) self.assertEquals(resp['status'], 404) @@ -48,3 +139,31 @@ class TestFuncs(unittest.TestCase): self.assertEquals( resp, headers_to_container_info(headers.items(), 200)) + + def test_headers_to_account_info_missing(self): + resp = headers_to_account_info({}, 404) + self.assertEquals(resp['status'], 404) + self.assertEquals(resp['bytes'], None) + self.assertEquals(resp['container_count'], None) + + def test_headers_to_account_info_meta(self): + headers = {'X-Account-Meta-Whatevs': 14, + 'x-account-meta-somethingelse': 0} + resp = headers_to_account_info(headers.items(), 200) + self.assertEquals(len(resp['meta']), 2) + self.assertEquals(resp['meta']['whatevs'], 14) + self.assertEquals(resp['meta']['somethingelse'], 0) + + def test_headers_to_account_info_values(self): + headers = { + 'x-account-object-count': '10', + 'x-account-container-count': '20', + } + resp = headers_to_account_info(headers.items(), 200) + self.assertEquals(resp['total_object_count'], '10') + self.assertEquals(resp['container_count'], '20') + + headers['x-unused-header'] = 'blahblahblah' + self.assertEquals( + resp, + headers_to_account_info(headers.items(), 200)) diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index f720c4ae74..fcbffedaa1 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -466,7 +466,12 @@ class TestController(unittest.TestCase): self.assertEquals(count, 12345) cache_key = get_account_memcache_key(self.account) - self.assertEquals({'status': 200, 'container_count': 12345}, + container_info = {'status': 200, + 'container_count': 12345, + 'total_object_count': None, + 'bytes': None, + 'meta': {}} + self.assertEquals(container_info, self.memcache.get(cache_key)) set_http_connect() @@ -485,7 +490,12 @@ class TestController(unittest.TestCase): self.assertEquals(count, None) cache_key = get_account_memcache_key(self.account) - self.assertEquals({'status': 404, 'container_count': 0}, + container_info = {'status': 404, + 'container_count': 0, + 'total_object_count': None, + 'bytes': None, + 'meta': {}} + self.assertEquals(container_info, self.memcache.get(cache_key)) set_http_connect()