Merge "Added optional max_containers_per_account restr..."

bin/swift

@@ -1806,8 +1806,17 @@ def st_upload(options, args, print_queue, error_queue):
         conn.put_container(args[0])
         if options.segment_size is not None:
             conn.put_container(args[0] + '_segments')
-    except Exception:
+    except ClientException, err:
-        pass
+        msg = ' '.join(str(x) for x in (err.http_status, err.http_reason))
+        if err.http_response_content:
+            if msg:
+                msg += ': '
+            msg += err.http_response_content[:60]
+        error_queue.put(
+            'Error trying to create container %r: %s' % (args[0], msg))
+    except Exception, err:
+        error_queue.put(
+            'Error trying to create container %r: %s' % (args[0], err))
     try:
         for arg in args[1:]:
             if isdir(arg):

doc/source/deployment_guide.rst

@@ -561,6 +561,20 @@ account_autocreate            false            If set to 'true' authorized
                                                accounts that do not yet exist
                                                within the Swift cluster will
                                                be automatically created.
+max_containers_per_account    0                If set to a positive value,
+                                               trying to create a container
+                                               when the account already has at
+                                               least this maximum containers
+                                               will result in a 403 Forbidden.
+                                               Note: This is a soft limit,
+                                               meaning a user might exceed the
+                                               cap for
+                                               recheck_account_existence before
+                                               the 403s kick in.
+max_containers_whitelist                       This is a comma separated list
+                                               of account hashes that ignore
+                                               the max_containers_per_account
+                                               cap.
 ============================  ===============  =============================
 
 [tempauth]

etc/proxy-server.conf-sample

@@ -49,6 +49,14 @@ use = egg:swift#proxy
 # If set to 'true' authorized accounts that do not yet exist within the Swift
 # cluster will be automatically created.
 # account_autocreate = false
+# If set to a positive value, trying to create a container when the account
+# already has at least this maximum containers will result in a 403 Forbidden.
+# Note: This is a soft limit, meaning a user might exceed the cap for
+# recheck_account_existence before the 403s kick in.
+# max_containers_per_account = 0
+# This is a comma separated list of account hashes that ignore the
+# max_containers_per_account cap.
+# max_containers_whitelist =
 
 [filter:tempauth]
 use = egg:swift#tempauth

swift/proxy/server.py

@@ -45,11 +45,10 @@ from random import shuffle
 from eventlet import sleep, spawn_n, GreenPile, Timeout
 from eventlet.queue import Queue, Empty, Full
 from eventlet.timeout import Timeout
-from webob.exc import HTTPAccepted, HTTPBadRequest, HTTPMethodNotAllowed, \
+from webob.exc import HTTPAccepted, HTTPBadRequest, HTTPForbidden, \
-    HTTPNotFound, HTTPPreconditionFailed, \
+    HTTPMethodNotAllowed, HTTPNotFound, HTTPPreconditionFailed, \
-    HTTPRequestTimeout, HTTPServiceUnavailable, \
+    HTTPRequestEntityTooLarge, HTTPRequestTimeout, HTTPServerError, \
-    HTTPUnprocessableEntity, HTTPRequestEntityTooLarge, HTTPServerError, \
+    HTTPServiceUnavailable, HTTPUnprocessableEntity, status_map
-    status_map
 from webob import Request, Response
 
 from swift.common.ring import Ring
@@ -389,19 +388,26 @@ class Controller(object):
         Get account information, and also verify that the account exists.
 
         :param account: name of the account to get the info for
-        :returns: tuple of (account partition, account nodes) or (None, None)
+        :returns: tuple of (account partition, account nodes, container_count)
-                  if it does not exist
+                  or (None, None, None) if it does not exist
         """
         partition, nodes = self.app.account_ring.get_nodes(account)
         # 0 = no responses, 200 = found, 404 = not found, -1 = mixed responses
         if self.app.memcache:
             cache_key = get_account_memcache_key(account)
-            result_code = self.app.memcache.get(cache_key)
+            cache_value = self.app.memcache.get(cache_key)
+            if not isinstance(cache_value, dict):
+                result_code = cache_value
+                container_count = 0
+            else:
+                result_code = cache_value['status']
+                container_count = cache_value['container_count']
             if result_code == 200:
-                return partition, nodes
+                return partition, nodes, container_count
             elif result_code == 404 and not autocreate:
-                return None, None
+                return None, None, None
         result_code = 0
+        container_count = 0
         attempts_left = self.app.account_ring.replica_count
         path = '/%s' % account
         headers = {'x-trans-id': self.trans_id, 'Connection': 'close'}
@@ -415,6 +421,8 @@ class Controller(object):
                     body = resp.read()
                     if 200 <= resp.status <= 299:
                         result_code = 200
+                        container_count = int(
+                            resp.getheader('x-account-container-count') or 0)
                         break
                     elif resp.status == 404:
                         if result_code == 0:
@@ -434,7 +442,7 @@ class Controller(object):
                     _('Trying to get account info for %s') % path)
         if result_code == 404 and autocreate:
             if len(account) > MAX_ACCOUNT_NAME_LENGTH:
-                return None, None
+                return None, None, None
             headers = {'X-Timestamp': normalize_timestamp(time.time()),
                        'X-Trans-Id': self.trans_id,
                        'Connection': 'close'}
@@ -449,11 +457,12 @@ class Controller(object):
                 cache_timeout = self.app.recheck_account_existence
             else:
                 cache_timeout = self.app.recheck_account_existence * 0.1
-            self.app.memcache.set(cache_key, result_code,
+            self.app.memcache.set(cache_key,
+                {'status': result_code, 'container_count': container_count},
                 timeout=cache_timeout)
         if result_code == 200:
-            return partition, nodes
+            return partition, nodes, container_count
-        return None, None
+        return None, None, None
 
     def container_info(self, account, container, account_autocreate=False):
         """
@@ -1503,8 +1512,16 @@ class ContainerController(Controller):
             resp.body = 'Container name length of %d longer than %d' % \
                         (len(self.container_name), MAX_CONTAINER_NAME_LENGTH)
             return resp
-        account_partition, accounts = self.account_info(self.account_name,
+        account_partition, accounts, container_count = \
+            self.account_info(self.account_name,
                               autocreate=self.app.account_autocreate)
+        if self.app.max_containers_per_account > 0 and \
+                container_count >= self.app.max_containers_per_account and \
+                self.account_name not in self.app.max_containers_whitelist:
+            resp = HTTPForbidden(request=req)
+            resp.body = 'Reached container limit of %s' % \
+                        self.app.max_containers_per_account
+            return resp
         if not accounts:
             return HTTPNotFound(request=req)
         container_partition, containers = self.app.container_ring.get_nodes(
@@ -1533,7 +1550,8 @@ class ContainerController(Controller):
             self.clean_acls(req) or check_metadata(req, 'container')
         if error_response:
             return error_response
-        account_partition, accounts = self.account_info(self.account_name,
+        account_partition, accounts, container_count = \
+            self.account_info(self.account_name,
                               autocreate=self.app.account_autocreate)
         if not accounts:
             return HTTPNotFound(request=req)
@@ -1554,7 +1572,8 @@ class ContainerController(Controller):
     @public
     def DELETE(self, req):
         """HTTP DELETE request handler."""
-        account_partition, accounts = self.account_info(self.account_name)
+        account_partition, accounts, container_count = \
+            self.account_info(self.account_name)
         if not accounts:
             return HTTPNotFound(request=req)
         container_partition, containers = self.app.container_ring.get_nodes(
@@ -1742,6 +1761,11 @@ class BaseApplication(object):
             'expiring_objects'
         self.expiring_objects_container_divisor = \
             int(conf.get('expiring_objects_container_divisor') or 86400)
+        self.max_containers_per_account = \
+            int(conf.get('max_containers_per_account') or 0)
+        self.max_containers_whitelist = [a.strip()
+            for a in conf.get('max_containers_whitelist', '').split(',')
+            if a.strip()]
 
     def get_controller(self, path):
         """

test/unit/proxy/test_server.py

@@ -180,7 +180,7 @@ def fake_http_connect(*code_iter, **kwargs):
                        'etag':
                             self.etag or '"68b329da9893e34099c7d8ad5cb9c940"',
                        'x-works': 'yes',
-                       }
+                       'x-account-container-count': 12345}
             if not self.timestamp:
                 del headers['x-timestamp']
             try:
@@ -353,7 +353,8 @@ class TestController(unittest.TestCase):
     def test_make_requests(self):
         with save_globals():
             proxy_server.http_connect = fake_http_connect(200)
-            partition, nodes = self.controller.account_info(self.account)
+            partition, nodes, count = \
+                self.controller.account_info(self.account)
             proxy_server.http_connect = fake_http_connect(201,
                                             raise_timeout_exc=True)
             self.controller._make_request(nodes, partition, 'POST',
@@ -363,37 +364,49 @@ class TestController(unittest.TestCase):
     def test_account_info_200(self):
         with save_globals():
             proxy_server.http_connect = fake_http_connect(200)
-            partition, nodes = self.controller.account_info(self.account)
+            partition, nodes, count = \
+                self.controller.account_info(self.account)
             self.check_account_info_return(partition, nodes)
+            self.assertEquals(count, 12345)
 
             cache_key = proxy_server.get_account_memcache_key(self.account)
-            self.assertEquals(200, self.memcache.get(cache_key))
+            self.assertEquals({'status': 200, 'container_count': 12345},
+                              self.memcache.get(cache_key))
 
             proxy_server.http_connect = fake_http_connect()
-            partition, nodes = self.controller.account_info(self.account)
+            partition, nodes, count = \
+                self.controller.account_info(self.account)
             self.check_account_info_return(partition, nodes)
+            self.assertEquals(count, 12345)
 
     # tests if 404 is cached and used
     def test_account_info_404(self):
         with save_globals():
             proxy_server.http_connect = fake_http_connect(404, 404, 404)
-            partition, nodes = self.controller.account_info(self.account)
+            partition, nodes, count = \
+                self.controller.account_info(self.account)
             self.check_account_info_return(partition, nodes, True)
+            self.assertEquals(count, None)
 
             cache_key = proxy_server.get_account_memcache_key(self.account)
-            self.assertEquals(404, self.memcache.get(cache_key))
+            self.assertEquals({'status': 404, 'container_count': 0},
+                              self.memcache.get(cache_key))
 
             proxy_server.http_connect = fake_http_connect()
-            partition, nodes = self.controller.account_info(self.account)
+            partition, nodes, count = \
+                self.controller.account_info(self.account)
             self.check_account_info_return(partition, nodes, True)
+            self.assertEquals(count, None)
 
     # tests if some http status codes are not cached
     def test_account_info_no_cache(self):
         def test(*status_list):
             proxy_server.http_connect = fake_http_connect(*status_list)
-            partition, nodes = self.controller.account_info(self.account)
+            partition, nodes, count = \
+                self.controller.account_info(self.account)
             self.assertEqual(len(self.memcache.keys()), 0)
             self.check_account_info_return(partition, nodes, True)
+            self.assertEquals(count, None)
 
         with save_globals():
             test(503, 404, 404)
@@ -406,37 +419,41 @@ class TestController(unittest.TestCase):
             self.memcache.store = {}
             proxy_server.http_connect = \
                 fake_http_connect(404, 404, 404, 201, 201, 201)
-            partition, nodes = \
+            partition, nodes, count = \
                 self.controller.account_info(self.account, autocreate=False)
             self.check_account_info_return(partition, nodes, is_none=True)
+            self.assertEquals(count, None)
 
             self.memcache.store = {}
             proxy_server.http_connect = \
                 fake_http_connect(404, 404, 404, 201, 201, 201)
-            partition, nodes = \
+            partition, nodes, count = \
                 self.controller.account_info(self.account)
             self.check_account_info_return(partition, nodes, is_none=True)
+            self.assertEquals(count, None)
 
             self.memcache.store = {}
             proxy_server.http_connect = \
                 fake_http_connect(404, 404, 404, 201, 201, 201)
-            partition, nodes = \
+            partition, nodes, count = \
                 self.controller.account_info(self.account, autocreate=True)
             self.check_account_info_return(partition, nodes)
+            self.assertEquals(count, 0)
 
             self.memcache.store = {}
             proxy_server.http_connect = \
                 fake_http_connect(404, 404, 404, 503, 201, 201)
-            partition, nodes = \
+            partition, nodes, count = \
                 self.controller.account_info(self.account, autocreate=True)
             self.check_account_info_return(partition, nodes)
+            self.assertEquals(count, 0)
 
             self.memcache.store = {}
             proxy_server.http_connect = \
                 fake_http_connect(404, 404, 404, 503, 201, 503)
             exc = None
             try:
-                partition, nodes = \
+                partition, nodes, count = \
                     self.controller.account_info(self.account, autocreate=True)
             except Exception, err:
                 exc = err
@@ -468,7 +485,7 @@ class TestController(unittest.TestCase):
     # tests if 200 is cached and used
     def test_container_info_200(self):
         def account_info(self, account, autocreate=False):
-            return True, True
+            return True, True, 0
 
         with save_globals():
             headers = {'x-container-read': self.read_acl,
@@ -494,7 +511,7 @@ class TestController(unittest.TestCase):
     # tests if 404 is cached and used
     def test_container_info_404(self):
         def account_info(self, account, autocreate=False):
-            return True, True
+            return True, True, 0
 
         with save_globals():
             proxy_server.Controller.account_info = account_info
@@ -3164,6 +3181,29 @@ class TestContainerController(unittest.TestCase):
             test_status_map((200, 204, 404, 404), 404, missing_container=True)
             test_status_map((200, 204, 500, 404), 503, missing_container=True)
 
+    def test_PUT_max_containers_per_account(self):
+        with save_globals():
+            self.app.max_containers_per_account = 12346
+            controller = proxy_server.ContainerController(self.app, 'account',
+                                                          'container')
+            self.assert_status_map(controller.PUT,
+                                   (200, 200, 200, 201, 201, 201), 201,
+                                   missing_container=True)
+
+            self.app.max_containers_per_account = 12345
+            controller = proxy_server.ContainerController(self.app, 'account',
+                                                          'container')
+            self.assert_status_map(controller.PUT, (201, 201, 201), 403,
+                                   missing_container=True)
+
+            self.app.max_containers_per_account = 12345
+            self.app.max_containers_whitelist = ['account']
+            controller = proxy_server.ContainerController(self.app, 'account',
+                                                          'container')
+            self.assert_status_map(controller.PUT,
+                                   (200, 200, 200, 201, 201, 201), 201,
+                                   missing_container=True)
+
     def test_PUT_max_container_name_length(self):
         with save_globals():
             controller = proxy_server.ContainerController(self.app, 'account',