openstack/swift
Code Issues Proposed changes

Account and container info fixes and improvement.

Browse Source 
- Fixes bug 1119282.
- Allow middleware accessing metadata of an account without having to
  store it separately in a new memcache namespace.
- Add tests for get_container_info that was previously missed.
- Add get_account_info method based on get_container_info, a function
  for other middleware to query accounts.
- Rename container_info['count'] as container_info['object_count'].

Change-Id: I43787916c7a812cb08d278edf45370521f12c912
This commit is contained in:
Chmouel Boudjnah 2013-02-08 11:48:26 +01:00
parent e88ff34685
commit b62299376a
6 changed files with 197 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.mailmap
View File

@ -17,6 +17,7 @@ Anne Gentle <anne@openstack.org> annegentle
Fujita Tomonori <fujita.tomonori@lab.ntt.co.jp>
Greg Lange <greglange@gmail.com> <glange@rackspace.com>
Greg Lange <greglange@gmail.com> <greglange+launchpad@gmail.com>
Chmouel Boudjnah <chmouel@chmouel.com> <chmouel@enovance.com>
Gaurav B. Gangalwar <gaurav@gluster.com> gaurav@gluster.com <>
Joe Arnold <joe@swiftstack.com> <joe@cloudscaling.com>
Kapil Thangavelu <kapil.foss@gmail.com> kapil.foss@gmail.com <>

5
swift/common/middleware/container_quotas.py
View File

@ -42,7 +42,6 @@ set:
+---------------------------------------------+-------------------------------+
"""
from swift.common.utils import split_path
from swift.common.http import is_success
from swift.proxy.controllers.base import get_container_info
from swift.common.swob import Response, HTTPBadRequest, wsgify
@ -93,9 +92,9 @@ class ContainerQuotaMiddleware(object):
if int(container_info['meta']['quota-bytes']) < new_size:
return self.bad_response(req, container_info)
if 'quota-count' in container_info.get('meta', {}) and \
'count' in container_info and \
'object_count' in container_info and \
container_info['meta']['quota-count'].isdigit():
new_count = int(container_info['count']) + 1
new_count = int(container_info['object_count']) + 1
if int(container_info['meta']['quota-count']) < new_count:
return self.bad_response(req, container_info)

71
swift/proxy/controllers/base.py
View File

@ -97,17 +97,33 @@ def get_container_memcache_key(account, container):
return 'container/%s/%s' % (account, container)
def headers_to_account_info(headers, status_int=HTTP_OK):
"""
Construct a cacheable dict of account info based on response headers.
"""
headers = dict((k.lower(), v) for k, v in dict(headers).iteritems())
return {
'status': status_int,
'container_count': headers.get('x-account-container-count'),
'total_object_count': headers.get('x-account-object-count'),
'bytes': headers.get('x-account-bytes-used'),
'meta': dict((key[15:], value)
for key, value in headers.iteritems()
if key.startswith('x-account-meta-'))
}
def headers_to_container_info(headers, status_int=HTTP_OK):
"""
Construct a cacheable dict of container info based on response headers.
"""
headers = dict(headers)
headers = dict((k.lower(), v) for k, v in dict(headers).iteritems())
return {
'status': status_int,
'read_acl': headers.get('x-container-read'),
'write_acl': headers.get('x-container-write'),
'sync_key': headers.get('x-container-sync-key'),
'count': headers.get('x-container-object-count'),
'object_count': headers.get('x-container-object-count'),
'bytes': headers.get('x-container-bytes-used'),
'versions': headers.get('x-versions-location'),
'cors': {
@ -120,9 +136,9 @@ def headers_to_container_info(headers, status_int=HTTP_OK):
'max_age': headers.get(
'x-container-meta-access-control-max-age')
},
'meta': dict((key.lower()[17:], value)
'meta': dict((key[17:], value)
for key, value in headers.iteritems()
if key.lower().startswith('x-container-meta-'))
if key.startswith('x-container-meta-'))
}
@ -198,8 +214,8 @@ def get_container_info(env, app, swift_source=None):
cache = cache_from_env(env)
if not cache:
return None
(version, account, container, obj) = \
split_path(env['PATH_INFO'], 2, 4, True)
(version, account, container, _) = \
split_path(env['PATH_INFO'], 3, 4, True)
cache_key = get_container_memcache_key(account, container)
# Use a unique environment cache key per container. If you copy this env
# to make a new request, it won't accidentally reuse the old container info
@ -217,6 +233,33 @@ def get_container_info(env, app, swift_source=None):
return env[env_key]
def get_account_info(env, app, swift_source=None):
"""
Get the info structure for an account, based on env and app.
This is useful to middlewares.
"""
cache = cache_from_env(env)
if not cache:
return None
(version, account, container, _) = \
split_path(env['PATH_INFO'], 2, 4, True)
cache_key = get_account_memcache_key(account)
# Use a unique environment cache key per account. If you copy this env
# to make a new request, it won't accidentally reuse the old account info
env_key = 'swift.%s' % cache_key
if env_key not in env:
account_info = cache.get(cache_key)
if not account_info:
resp = make_pre_authed_request(
env, 'HEAD', '/%s/%s' % (version, account),
swift_source=swift_source,
).get_response(app)
account_info = headers_to_account_info(
resp.headers, resp.status_int)
env[env_key] = account_info
return env[env_key]
class Controller(object):
"""Base WSGI controller class for the proxy"""
server_type = 'Base'
@ -326,6 +369,11 @@ class Controller(object):
or (None, None, None) if it does not exist
"""
partition, nodes = self.app.account_ring.get_nodes(account)
account_info = {'status': 0,
'container_count': 0,
'total_object_count': None,
'bytes': None,
'meta': {}}
# 0 = no responses, 200 = found, 404 = not found, -1 = mixed responses
if self.app.memcache:
cache_key = get_account_memcache_key(account)
@ -341,7 +389,6 @@ class Controller(object):
elif result_code == HTTP_NOT_FOUND and not autocreate:
return None, None, None
result_code = 0
container_count = 0
attempts_left = len(nodes)
path = '/%s' % account
headers = {'x-trans-id': self.trans_id, 'Connection': 'close'}
@ -362,8 +409,8 @@ class Controller(object):
resp.read()
if is_success(resp.status):
result_code = HTTP_OK
container_count = int(
resp.getheader('x-account-container-count') or 0)
account_info.update(
headers_to_account_info(resp.getheaders()))
break
elif resp.status == HTTP_NOT_FOUND:
if result_code == 0:
@ -398,12 +445,12 @@ class Controller(object):
cache_timeout = self.app.recheck_account_existence
else:
cache_timeout = self.app.recheck_account_existence * 0.1
account_info.update(status=result_code)
self.app.memcache.set(cache_key,
{'status': result_code,
'container_count': container_count},
account_info,
time=cache_timeout)
if result_code == HTTP_OK:
return partition, nodes, container_count
return partition, nodes, account_info['container_count']
return None, None, None
def container_info(self, account, container, account_autocreate=False):

6
test/unit/common/middleware/test_quotas.py
View File

@ -87,7 +87,7 @@ class TestContainerQuotas(unittest.TestCase):
def test_exceed_counts_quota(self):
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
cache = FakeCache({'count': 1, 'meta': {'quota-count': '1'}})
cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '1'}})
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
'CONTENT_LENGTH': '100'})
@ -96,7 +96,7 @@ class TestContainerQuotas(unittest.TestCase):
def test_not_exceed_counts_quota(self):
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
cache = FakeCache({'count': 1, 'meta': {'quota-count': '2'}})
cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '2'}})
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
'CONTENT_LENGTH': '100'})
@ -152,7 +152,7 @@ class TestContainerQuotas(unittest.TestCase):
def test_auth_fail(self):
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
cache = FakeCache({'count': 1, 'meta': {'quota-count': '1'},
cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '1'},
'write_acl': None})
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,

121
test/unit/proxy/controllers/test_base.py
View File

@ -14,10 +14,101 @@
# limitations under the License.
import unittest
from swift.proxy.controllers.base import headers_to_container_info
import swift.proxy.controllers.base
from swift.proxy.controllers.base import headers_to_container_info, \
headers_to_account_info, get_container_info, get_container_memcache_key, \
get_account_info, get_account_memcache_key
from swift.common.swob import Request
from swift.common.utils import split_path
class FakeResponse(object):
def __init__(self, headers):
self.headers = headers
self.status_int = 201
class FakeRequest(object):
def __init__(self, env, method, path, swift_source=None):
(version, account,
container, obj) = split_path(env['PATH_INFO'], 2, 4, True)
stype = container and 'container' or 'account'
self.headers = {'x-%s-object-count' % (stype): 1000,
'x-%s-bytes-used' % (stype): 6666}
def get_response(self, app):
return FakeResponse(self.headers)
class FakeCache(object):
def __init__(self, val):
self.val = val
def get(self, *args):
return self.val
class TestFuncs(unittest.TestCase):
def test_get_container_info_no_cache(self):
swift.proxy.controllers.base.make_pre_authed_request = FakeRequest
req = Request.blank("/v1/AUTH_account/cont",
environ={'swift.cache': FakeCache({})})
resp = get_container_info(req.environ, 'xxx')
self.assertEquals(resp['bytes'], 6666)
self.assertEquals(resp['object_count'], 1000)
def test_get_container_info_cache(self):
swift.proxy.controllers.base.make_pre_authed_request = FakeRequest
cached = {'status': 404,
'bytes': 3333,
'object_count': 10}
req = Request.blank("/v1/account/cont",
environ={'swift.cache': FakeCache(cached)})
resp = get_container_info(req.environ, 'xxx')
self.assertEquals(resp['bytes'], 3333)
self.assertEquals(resp['object_count'], 10)
self.assertEquals(resp['status'], 404)
def test_get_container_info_env(self):
cache_key = get_container_memcache_key("account", "cont")
env_key = 'swift.%s' % cache_key
req = Request.blank("/v1/account/cont",
environ={ env_key: {'bytes': 3867},
'swift.cache': FakeCache({})})
resp = get_container_info(req.environ, 'xxx')
self.assertEquals(resp['bytes'], 3867)
def test_get_account_info_no_cache(self):
swift.proxy.controllers.base.make_pre_authed_request = FakeRequest
req = Request.blank("/v1/AUTH_account",
environ={'swift.cache': FakeCache({})})
resp = get_account_info(req.environ, 'xxx')
print resp
self.assertEquals(resp['bytes'], 6666)
self.assertEquals(resp['total_object_count'], 1000)
def test_get_account_info_cache(self):
swift.proxy.controllers.base.make_pre_authed_request = FakeRequest
cached = {'status': 404,
'bytes': 3333,
'total_object_count': 10}
req = Request.blank("/v1/account/cont",
environ={'swift.cache': FakeCache(cached)})
resp = get_account_info(req.environ, 'xxx')
self.assertEquals(resp['bytes'], 3333)
self.assertEquals(resp['total_object_count'], 10)
self.assertEquals(resp['status'], 404)
def test_get_account_info_env(self):
cache_key = get_account_memcache_key("account")
env_key = 'swift.%s' % cache_key
req = Request.blank("/v1/account",
environ={ env_key: {'bytes': 3867},
'swift.cache': FakeCache({})})
resp = get_account_info(req.environ, 'xxx')
self.assertEquals(resp['bytes'], 3867)
def test_headers_to_container_info_missing(self):
resp = headers_to_container_info({}, 404)
self.assertEquals(resp['status'], 404)
@ -48,3 +139,31 @@ class TestFuncs(unittest.TestCase):
self.assertEquals(
resp,
headers_to_container_info(headers.items(), 200))
def test_headers_to_account_info_missing(self):
resp = headers_to_account_info({}, 404)
self.assertEquals(resp['status'], 404)
self.assertEquals(resp['bytes'], None)
self.assertEquals(resp['container_count'], None)
def test_headers_to_account_info_meta(self):
headers = {'X-Account-Meta-Whatevs': 14,
'x-account-meta-somethingelse': 0}
resp = headers_to_account_info(headers.items(), 200)
self.assertEquals(len(resp['meta']), 2)
self.assertEquals(resp['meta']['whatevs'], 14)
self.assertEquals(resp['meta']['somethingelse'], 0)
def test_headers_to_account_info_values(self):
headers = {
'x-account-object-count': '10',
'x-account-container-count': '20',
}
resp = headers_to_account_info(headers.items(), 200)
self.assertEquals(resp['total_object_count'], '10')
self.assertEquals(resp['container_count'], '20')
headers['x-unused-header'] = 'blahblahblah'
self.assertEquals(
resp,
headers_to_account_info(headers.items(), 200))

14
test/unit/proxy/test_server.py
View File

@ -466,7 +466,12 @@ class TestController(unittest.TestCase):
self.assertEquals(count, 12345)
cache_key = get_account_memcache_key(self.account)
self.assertEquals({'status': 200, 'container_count': 12345},
container_info = {'status': 200,
'container_count': 12345,
'total_object_count': None,
'bytes': None,
'meta': {}}
self.assertEquals(container_info,
self.memcache.get(cache_key))
set_http_connect()
@ -485,7 +490,12 @@ class TestController(unittest.TestCase):
self.assertEquals(count, None)
cache_key = get_account_memcache_key(self.account)
self.assertEquals({'status': 404, 'container_count': 0},
container_info = {'status': 404,
'container_count': 0,
'total_object_count': None,
'bytes': None,
'meta': {}}
self.assertEquals(container_info,
self.memcache.get(cache_key))
set_http_connect()