Account and container info fixes and improvement.

- Fixes bug 1119282.
- Allow middleware accessing metadata of an account without having to
  store it separately in a new memcache namespace.
- Add tests for get_container_info that was previously missed.
- Add get_account_info method based on get_container_info, a function
  for other middleware to query accounts.
- Rename container_info['count'] as container_info['object_count'].

Change-Id: I43787916c7a812cb08d278edf45370521f12c912

.mailmap

@@ -17,6 +17,7 @@ Anne Gentle <anne@openstack.org> annegentle
 Fujita Tomonori <fujita.tomonori@lab.ntt.co.jp>
 Greg Lange <greglange@gmail.com> <glange@rackspace.com>
 Greg Lange <greglange@gmail.com> <greglange+launchpad@gmail.com>
+Chmouel Boudjnah <chmouel@chmouel.com> <chmouel@enovance.com>
 Gaurav B. Gangalwar <gaurav@gluster.com> gaurav@gluster.com <>
 Joe Arnold <joe@swiftstack.com> <joe@cloudscaling.com>
 Kapil Thangavelu <kapil.foss@gmail.com> kapil.foss@gmail.com <>

swift/common/middleware/container_quotas.py

@@ -42,7 +42,6 @@ set:
 +---------------------------------------------+-------------------------------+
 """
 
-from swift.common.utils import split_path
 from swift.common.http import is_success
 from swift.proxy.controllers.base import get_container_info
 from swift.common.swob import Response, HTTPBadRequest, wsgify
@@ -93,9 +92,9 @@ class ContainerQuotaMiddleware(object):
                 if int(container_info['meta']['quota-bytes']) < new_size:
                     return self.bad_response(req, container_info)
             if 'quota-count' in container_info.get('meta', {}) and \
-                    'count' in container_info and \
+                    'object_count' in container_info and \
                     container_info['meta']['quota-count'].isdigit():
-                new_count = int(container_info['count']) + 1
+                new_count = int(container_info['object_count']) + 1
                 if int(container_info['meta']['quota-count']) < new_count:
                     return self.bad_response(req, container_info)
 

swift/proxy/controllers/base.py

@@ -97,17 +97,33 @@ def get_container_memcache_key(account, container):
     return 'container/%s/%s' % (account, container)
 
 
+def headers_to_account_info(headers, status_int=HTTP_OK):
+    """
+    Construct a cacheable dict of account info based on response headers.
+    """
+    headers = dict((k.lower(), v) for k, v in dict(headers).iteritems())
+    return {
+        'status': status_int,
+        'container_count': headers.get('x-account-container-count'),
+        'total_object_count': headers.get('x-account-object-count'),
+        'bytes': headers.get('x-account-bytes-used'),
+        'meta': dict((key[15:], value)
+                     for key, value in headers.iteritems()
+                     if key.startswith('x-account-meta-'))
+    }
+
+
 def headers_to_container_info(headers, status_int=HTTP_OK):
     """
     Construct a cacheable dict of container info based on response headers.
     """
-    headers = dict(headers)
+    headers = dict((k.lower(), v) for k, v in dict(headers).iteritems())
     return {
         'status': status_int,
         'read_acl': headers.get('x-container-read'),
         'write_acl': headers.get('x-container-write'),
         'sync_key': headers.get('x-container-sync-key'),
-        'count': headers.get('x-container-object-count'),
+        'object_count': headers.get('x-container-object-count'),
         'bytes': headers.get('x-container-bytes-used'),
         'versions': headers.get('x-versions-location'),
         'cors': {
@@ -120,9 +136,9 @@ def headers_to_container_info(headers, status_int=HTTP_OK):
             'max_age': headers.get(
                 'x-container-meta-access-control-max-age')
         },
-        'meta': dict((key.lower()[17:], value)
+        'meta': dict((key[17:], value)
                      for key, value in headers.iteritems()
-                     if key.lower().startswith('x-container-meta-'))
+                     if key.startswith('x-container-meta-'))
     }
 
 
@@ -198,8 +214,8 @@ def get_container_info(env, app, swift_source=None):
     cache = cache_from_env(env)
     if not cache:
         return None
-    (version, account, container, obj) = \
+    (version, account, container, _) = \
-        split_path(env['PATH_INFO'], 2, 4, True)
+        split_path(env['PATH_INFO'], 3, 4, True)
     cache_key = get_container_memcache_key(account, container)
     # Use a unique environment cache key per container.  If you copy this env
     # to make a new request, it won't accidentally reuse the old container info
@@ -217,6 +233,33 @@ def get_container_info(env, app, swift_source=None):
     return env[env_key]
 
 
+def get_account_info(env, app, swift_source=None):
+    """
+    Get the info structure for an account, based on env and app.
+    This is useful to middlewares.
+    """
+    cache = cache_from_env(env)
+    if not cache:
+        return None
+    (version, account, container, _) = \
+        split_path(env['PATH_INFO'], 2, 4, True)
+    cache_key = get_account_memcache_key(account)
+    # Use a unique environment cache key per account.  If you copy this env
+    # to make a new request, it won't accidentally reuse the old account info
+    env_key = 'swift.%s' % cache_key
+    if env_key not in env:
+        account_info = cache.get(cache_key)
+        if not account_info:
+            resp = make_pre_authed_request(
+                env, 'HEAD', '/%s/%s' % (version, account),
+                swift_source=swift_source,
+            ).get_response(app)
+            account_info = headers_to_account_info(
+                resp.headers, resp.status_int)
+        env[env_key] = account_info
+    return env[env_key]
+
+
 class Controller(object):
     """Base WSGI controller class for the proxy"""
     server_type = 'Base'
@@ -326,6 +369,11 @@ class Controller(object):
                   or (None, None, None) if it does not exist
         """
         partition, nodes = self.app.account_ring.get_nodes(account)
+        account_info = {'status': 0,
+                        'container_count': 0,
+                        'total_object_count': None,
+                        'bytes': None,
+                        'meta': {}}
         # 0 = no responses, 200 = found, 404 = not found, -1 = mixed responses
         if self.app.memcache:
             cache_key = get_account_memcache_key(account)
@@ -341,7 +389,6 @@ class Controller(object):
             elif result_code == HTTP_NOT_FOUND and not autocreate:
                 return None, None, None
         result_code = 0
-        container_count = 0
         attempts_left = len(nodes)
         path = '/%s' % account
         headers = {'x-trans-id': self.trans_id, 'Connection': 'close'}
@@ -362,8 +409,8 @@ class Controller(object):
                     resp.read()
                     if is_success(resp.status):
                         result_code = HTTP_OK
-                        container_count = int(
+                        account_info.update(
-                            resp.getheader('x-account-container-count') or 0)
+                            headers_to_account_info(resp.getheaders()))
                         break
                     elif resp.status == HTTP_NOT_FOUND:
                         if result_code == 0:
@@ -398,12 +445,12 @@ class Controller(object):
                 cache_timeout = self.app.recheck_account_existence
             else:
                 cache_timeout = self.app.recheck_account_existence * 0.1
+            account_info.update(status=result_code)
             self.app.memcache.set(cache_key,
-                                  {'status': result_code,
+                                  account_info,
-                                  'container_count': container_count},
                                   time=cache_timeout)
         if result_code == HTTP_OK:
-            return partition, nodes, container_count
+            return partition, nodes, account_info['container_count']
         return None, None, None
 
     def container_info(self, account, container, account_autocreate=False):

test/unit/common/middleware/test_quotas.py

@@ -87,7 +87,7 @@ class TestContainerQuotas(unittest.TestCase):
 
     def test_exceed_counts_quota(self):
         app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
-        cache = FakeCache({'count': 1, 'meta': {'quota-count': '1'}})
+        cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '1'}})
         req = Request.blank('/v1/a/c/o',
             environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
                      'CONTENT_LENGTH': '100'})
@@ -96,7 +96,7 @@ class TestContainerQuotas(unittest.TestCase):
 
     def test_not_exceed_counts_quota(self):
         app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
-        cache = FakeCache({'count': 1, 'meta': {'quota-count': '2'}})
+        cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '2'}})
         req = Request.blank('/v1/a/c/o',
             environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
                      'CONTENT_LENGTH': '100'})
@@ -152,7 +152,7 @@ class TestContainerQuotas(unittest.TestCase):
 
     def test_auth_fail(self):
         app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
-        cache = FakeCache({'count': 1, 'meta': {'quota-count': '1'},
+        cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '1'},
                            'write_acl': None})
         req = Request.blank('/v1/a/c/o',
             environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,

test/unit/proxy/controllers/test_base.py

@@ -14,10 +14,101 @@
 # limitations under the License.
 
 import unittest
-from swift.proxy.controllers.base import headers_to_container_info
+
+import swift.proxy.controllers.base
+from swift.proxy.controllers.base import headers_to_container_info, \
+    headers_to_account_info, get_container_info, get_container_memcache_key, \
+    get_account_info, get_account_memcache_key
+from swift.common.swob import Request
+from swift.common.utils import split_path
+
+
+class FakeResponse(object):
+    def __init__(self, headers):
+        self.headers = headers
+        self.status_int = 201
+
+
+class FakeRequest(object):
+    def __init__(self, env, method, path, swift_source=None):
+        (version, account,
+         container, obj) = split_path(env['PATH_INFO'], 2, 4, True)
+        stype = container and 'container' or 'account'
+        self.headers = {'x-%s-object-count' % (stype): 1000,
+                        'x-%s-bytes-used' % (stype): 6666}
+
+    def get_response(self, app):
+        return FakeResponse(self.headers)
+
+
+class FakeCache(object):
+    def __init__(self, val):
+        self.val = val
+
+    def get(self, *args):
+        return self.val
 
 
 class TestFuncs(unittest.TestCase):
+    def test_get_container_info_no_cache(self):
+        swift.proxy.controllers.base.make_pre_authed_request = FakeRequest
+        req = Request.blank("/v1/AUTH_account/cont",
+                            environ={'swift.cache': FakeCache({})})
+        resp = get_container_info(req.environ, 'xxx')
+        self.assertEquals(resp['bytes'], 6666)
+        self.assertEquals(resp['object_count'], 1000)
+
+    def test_get_container_info_cache(self):
+        swift.proxy.controllers.base.make_pre_authed_request = FakeRequest
+        cached = {'status': 404,
+                  'bytes': 3333,
+                  'object_count': 10}
+        req = Request.blank("/v1/account/cont",
+                            environ={'swift.cache': FakeCache(cached)})
+        resp = get_container_info(req.environ, 'xxx')
+        self.assertEquals(resp['bytes'], 3333)
+        self.assertEquals(resp['object_count'], 10)
+        self.assertEquals(resp['status'], 404)
+
+    def test_get_container_info_env(self):
+        cache_key = get_container_memcache_key("account", "cont")
+        env_key = 'swift.%s' % cache_key
+        req = Request.blank("/v1/account/cont",
+                            environ={ env_key: {'bytes': 3867},
+                                      'swift.cache': FakeCache({})})
+        resp = get_container_info(req.environ, 'xxx')
+        self.assertEquals(resp['bytes'], 3867)
+
+    def test_get_account_info_no_cache(self):
+        swift.proxy.controllers.base.make_pre_authed_request = FakeRequest
+        req = Request.blank("/v1/AUTH_account",
+                            environ={'swift.cache': FakeCache({})})
+        resp = get_account_info(req.environ, 'xxx')
+        print resp
+        self.assertEquals(resp['bytes'], 6666)
+        self.assertEquals(resp['total_object_count'], 1000)
+
+    def test_get_account_info_cache(self):
+        swift.proxy.controllers.base.make_pre_authed_request = FakeRequest
+        cached = {'status': 404,
+                  'bytes': 3333,
+                  'total_object_count': 10}
+        req = Request.blank("/v1/account/cont",
+                            environ={'swift.cache': FakeCache(cached)})
+        resp = get_account_info(req.environ, 'xxx')
+        self.assertEquals(resp['bytes'], 3333)
+        self.assertEquals(resp['total_object_count'], 10)
+        self.assertEquals(resp['status'], 404)
+
+    def test_get_account_info_env(self):
+        cache_key = get_account_memcache_key("account")
+        env_key = 'swift.%s' % cache_key
+        req = Request.blank("/v1/account",
+                            environ={ env_key: {'bytes': 3867},
+                                      'swift.cache': FakeCache({})})
+        resp = get_account_info(req.environ, 'xxx')
+        self.assertEquals(resp['bytes'], 3867)
+
     def test_headers_to_container_info_missing(self):
         resp = headers_to_container_info({}, 404)
         self.assertEquals(resp['status'], 404)
@@ -48,3 +139,31 @@ class TestFuncs(unittest.TestCase):
         self.assertEquals(
             resp,
             headers_to_container_info(headers.items(), 200))
+
+    def test_headers_to_account_info_missing(self):
+        resp = headers_to_account_info({}, 404)
+        self.assertEquals(resp['status'], 404)
+        self.assertEquals(resp['bytes'], None)
+        self.assertEquals(resp['container_count'], None)
+
+    def test_headers_to_account_info_meta(self):
+        headers = {'X-Account-Meta-Whatevs': 14,
+                   'x-account-meta-somethingelse': 0}
+        resp = headers_to_account_info(headers.items(), 200)
+        self.assertEquals(len(resp['meta']), 2)
+        self.assertEquals(resp['meta']['whatevs'], 14)
+        self.assertEquals(resp['meta']['somethingelse'], 0)
+
+    def test_headers_to_account_info_values(self):
+        headers = {
+            'x-account-object-count': '10',
+            'x-account-container-count': '20',
+        }
+        resp = headers_to_account_info(headers.items(), 200)
+        self.assertEquals(resp['total_object_count'], '10')
+        self.assertEquals(resp['container_count'], '20')
+
+        headers['x-unused-header'] = 'blahblahblah'
+        self.assertEquals(
+            resp,
+            headers_to_account_info(headers.items(), 200))

test/unit/proxy/test_server.py

@@ -466,7 +466,12 @@ class TestController(unittest.TestCase):
             self.assertEquals(count, 12345)
 
             cache_key = get_account_memcache_key(self.account)
-            self.assertEquals({'status': 200, 'container_count': 12345},
+            container_info = {'status': 200,
+                              'container_count': 12345,
+                              'total_object_count': None,
+                              'bytes': None,
+                              'meta': {}}
+            self.assertEquals(container_info,
                               self.memcache.get(cache_key))
 
             set_http_connect()
@@ -485,7 +490,12 @@ class TestController(unittest.TestCase):
             self.assertEquals(count, None)
 
             cache_key = get_account_memcache_key(self.account)
-            self.assertEquals({'status': 404, 'container_count': 0},
+            container_info = {'status': 404,
+                              'container_count': 0,
+                              'total_object_count': None,
+                              'bytes': None,
+                              'meta': {}}
+            self.assertEquals(container_info,
                               self.memcache.get(cache_key))
 
             set_http_connect()