Merge "backend ratelimit: support per-method rate limits"

etc/backend-ratelimit.conf-sample

@@ -1,10 +1,30 @@
 [backend_ratelimit]
-# Set the maximum rate of requests per second per device per worker. Beyond
+# The rate of requests to each device is limited by an overall per-device rate
-# this rate the server will return 529 responses and emit a 'backend.ratelimit'
+# limit that is applied to all requests to the device and/or a
-# statsd metric without logging. The default value of zero causes no
+# per-method-per-device rate limit that is applied to requests of that method
-# rate-limiting to be applied.
+# to the device. If either of these rates would be exceeded the server will
+# return 529 responses and emit a 'backend.ratelimit' statsd metric without
+# logging.
+
+# Set the maximum overall rate of requests per device per second per worker for
+# all request methods. The default value of zero causes no per-device
+# rate-limiting to be applied other than that configured for specific request
+# methods.
 # requests_per_device_per_second = 0.0
-#
+
+# Set maximum rate of requests per device per second per worker for individual
+# request methods. The default value of zero causes no per-method
+# rate-limiting to be applied. Note: the aggregate rate of requests for all
+# methods is still limited by requests_per_device_per_second even if a higher
+# per method rate is configured.
+# delete_requests_per_device_per_second = 0.0
+# get_requests_per_device_per_second = 0.0
+# head_requests_per_device_per_second = 0.0
+# post_requests_per_device_per_second = 0.0
+# put_requests_per_device_per_second = 0.0
+# replicate_requests_per_device_per_second = 0.0
+# update_requests_per_device_per_second = 0.0
+
 # Set the number of seconds of unused rate-limiting allowance that can
 # accumulate and be used to allow a subsequent burst of requests.
 # requests_per_device_rate_buffer = 1.0

swift/common/middleware/backend_ratelimit.py

@@ -14,7 +14,6 @@
 # limitations under the License.
 import os
 import time
-from collections import defaultdict
 
 from swift.common.request_helpers import split_and_validate_path
 from swift.common.swob import Request, HTTPTooManyBackendRequests, \
@@ -35,31 +34,26 @@ class BackendRateLimitMiddleware(object):
     """
     Backend rate-limiting middleware.
 
-    Rate-limits requests to backend storage node devices. Each device is
+    Rate-limits requests to backend storage node devices. Each (device, request
-    independently rate-limited. All requests with a 'GET', 'HEAD', 'PUT',
+    method) combination is independently rate-limited. All requests with a
-    'POST', 'DELETE', 'UPDATE' or 'REPLICATE' method are included in a device's
+    'GET', 'HEAD', 'PUT', 'POST', 'DELETE', 'UPDATE' or 'REPLICATE' method are
-    rate limit.
+    rate limited on a per-device basis by both a method-specific rate and an
+    overall device rate limit.
 
-    If a request would cause the rate-limit to be exceeded then a response with
+    If a request would cause the rate-limit to be exceeded for the method
-    a 529 status code is returned.
+    and/or device then a response with a 529 status code is returned.
     """
     def __init__(self, app, filter_conf, logger=None):
         self.app = app
         self.filter_conf = filter_conf
-        self.current_conf = {}
         self.logger = logger or get_logger(self.filter_conf,
                                            log_route='backend_ratelimit')
-        self.requests_per_device_per_second = \
-            DEFAULT_REQUESTS_PER_DEVICE_PER_SECOND
         self.requests_per_device_rate_buffer = \
             DEFAULT_REQUESTS_PER_DEVICE_RATE_BUFFER
-        # map device -> RateLimiter
+        # map (device, method) -> rate
-        self.rate_limiters = defaultdict(
+        self.requests_per_device_per_second = {}
-            lambda: EventletRateLimiter(
+        # map (device, method) -> RateLimiter, populated on-demand
-                max_rate=self.requests_per_device_per_second,
+        self.rate_limiters = {}
-                rate_buffer=self.requests_per_device_rate_buffer,
-                running_time=time.time(),
-                burst_after_idle=True))
 
         # some config options are *only* read from filter conf at startup...
         default_conf_path = os.path.join(
@@ -82,24 +76,35 @@ class BackendRateLimitMiddleware(object):
         self._load_config_file()
 
     def _refresh_ratelimiters(self):
-        for dev, rl in self.rate_limiters.items():
+        # note: if we ever wanted to prune the ratelimiters (in case devices
-            rl.set_max_rate(self.requests_per_device_per_second)
+        # have been removed) we could inspect each ratelimiter's running_time
+        # and remove those with very old running_time
+        for (dev, method), rl in self.rate_limiters.items():
+            rl.set_max_rate(self.requests_per_device_per_second[method])
             rl.set_rate_buffer(self.requests_per_device_rate_buffer)
 
     def _apply_config(self, conf):
-        self.current_conf = conf
         modified = False
-        new_value = non_negative_float(
+        reqs_per_device_rate_buffer = non_negative_float(
-            conf.get('requests_per_device_per_second',
-                     DEFAULT_REQUESTS_PER_DEVICE_PER_SECOND))
-        if new_value != self.requests_per_device_per_second:
-            self.requests_per_device_per_second = new_value
-            modified = True
-        new_value = non_negative_float(
             conf.get('requests_per_device_rate_buffer',
                      DEFAULT_REQUESTS_PER_DEVICE_RATE_BUFFER))
-        if new_value != self.requests_per_device_rate_buffer:
+
-            self.requests_per_device_rate_buffer = new_value
+        # note: 'None' key holds the aggregate per-device limit for all methods
+        reqs_per_device_per_second = {None: non_negative_float(
+            conf.get('requests_per_device_per_second', 0.0))}
+        for method in RATE_LIMITED_METHODS:
+            val = non_negative_float(
+                conf.get('%s_requests_per_device_per_second'
+                         % method.lower(), 0.0))
+            reqs_per_device_per_second[method] = val
+
+        if reqs_per_device_rate_buffer != self.requests_per_device_rate_buffer:
+            self.requests_per_device_rate_buffer = reqs_per_device_rate_buffer
+            modified = True
+        if reqs_per_device_per_second != self.requests_per_device_per_second:
+            self.requests_per_device_per_second = reqs_per_device_per_second
+            self.is_any_rate_limit_configured = any(
+                self.requests_per_device_per_second.values())
             modified = True
         if modified:
             self._refresh_ratelimiters()
@@ -112,7 +117,7 @@ class BackendRateLimitMiddleware(object):
         # filter conf value or default value. If the conf file cannot be read
         # or is invalid, then the current config is left unchanged.
         try:
-            new_conf = dict(self.filter_conf)  # filter_conf not current_conf
+            new_conf = dict(self.filter_conf)  # filter_conf not current conf
             new_conf.update(
                 readconf(self.conf_path, BACKEND_RATELIMIT_CONFIG_SECTION))
             modified = self._apply_config(new_conf)
@@ -150,6 +155,46 @@ class BackendRateLimitMiddleware(object):
                     # always reset last loaded time to avoid re-try storm
                     self._last_config_reload_attempt = now
 
+    def _get_ratelimiter(self, device, method=None):
+        """
+        Get a rate limiter for the (device, method) combination. If a rate
+        limiter does not yet exist for the given (device, method) combination
+        then it is created and added to the map of rate limiters.
+
+        :param: the device.
+        :method: the request method; if None then the aggregate rate limiter
+            for all requests to the device is returned.
+        :returns: an instance of ``EventletRateLimiter``.
+        """
+        try:
+            rl = self.rate_limiters[(device, method)]
+        except KeyError:
+            rl = EventletRateLimiter(
+                max_rate=self.requests_per_device_per_second[method],
+                rate_buffer=self.requests_per_device_rate_buffer,
+                running_time=time.time(),
+                burst_after_idle=True)
+            self.rate_limiters[(device, method)] = rl
+        return rl
+
+    def _is_allowed(self, device, method):
+        """
+        Evaluate backend rate-limiting policies for the incoming request.
+
+        A request is allowed when neither the per-(device, method) rate-limit
+        nor the per-device rate-limit has been reached.
+
+        Note: a request will be disallowed if the aggregate per-device
+        rate-limit has been reached, even if the per-(device, method)
+        rate-limit has not been reached for the request's method.
+
+        :param: the device.
+        :method: the request method.
+        :returns: boolean, is_allowed.
+        """
+        return (self._get_ratelimiter(device, None).is_allowed()
+                and self._get_ratelimiter(device, method).is_allowed())
+
     def __call__(self, env, start_response):
         """
         WSGI entry point.
@@ -160,7 +205,7 @@ class BackendRateLimitMiddleware(object):
         self._maybe_reload_config()
         req = Request(env)
         handler = self.app
-        if (self.requests_per_device_per_second
+        if (self.is_any_rate_limit_configured
                 and req.method in RATE_LIMITED_METHODS):
             try:
                 device, partition, _ = split_and_validate_path(req, 1, 3, True)
@@ -169,8 +214,7 @@ class BackendRateLimitMiddleware(object):
                 # request may not have device/partition e.g. a healthcheck req
                 pass
             else:
-                rate_limiter = self.rate_limiters[device]
+                if not self._is_allowed(device, req.method):
-                if not rate_limiter.is_allowed():
                     self.logger.increment('backend.ratelimit')
                     handler = HTTPTooManyBackendRequests()
         return handler(env, start_response)

test/unit/common/middleware/test_backend_ratelimit.py

@@ -46,23 +46,35 @@ class TestBackendRatelimitMiddleware(unittest.TestCase):
         super(TestBackendRatelimitMiddleware, self).setUp()
         self.swift = FakeSwift()
         self.tempdir = mkdtemp()
+        self.default_req_per_dev_per_sec = dict(
+            (key, 0.0) for key in
+            (None, 'GET', 'HEAD', 'PUT', 'POST', 'DELETE', 'UPDATE',
+             'REPLICATE')
+        )
 
     def tearDown(self):
         shutil.rmtree(self.tempdir, ignore_errors=True)
 
     def test_init(self):
-        conf = {}
+        conf = {'swift_dir': self.tempdir}
         factory = backend_ratelimit.filter_factory(conf)
         rl = factory(self.swift)
-        self.assertEqual(0.0, rl.requests_per_device_per_second)
+        self.assertEqual(self.default_req_per_dev_per_sec,
+                         rl.requests_per_device_per_second)
         self.assertEqual(1.0, rl.requests_per_device_rate_buffer)
+        self.assertFalse(rl.is_any_rate_limit_configured)
 
-        conf = {'requests_per_device_per_second': 1.3,
+        conf = {'swift_dir': self.tempdir,
+                'requests_per_device_per_second': 1.3,
                 'requests_per_device_rate_buffer': 2.4}
         factory = backend_ratelimit.filter_factory(conf)
         rl = factory(self.swift)
-        self.assertEqual(1.3, rl.requests_per_device_per_second)
+        exp_req_per_dev_per_sec = dict(self.default_req_per_dev_per_sec)
+        exp_req_per_dev_per_sec[None] = 1.3
+        self.assertEqual(exp_req_per_dev_per_sec,
+                         rl.requests_per_device_per_second)
         self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
+        self.assertTrue(rl.is_any_rate_limit_configured)
 
         conf = {'requests_per_device_per_second': -1}
         factory = backend_ratelimit.filter_factory(conf)
@@ -128,7 +140,10 @@ class TestBackendRatelimitMiddleware(unittest.TestCase):
                     'swift.common.middleware.backend_ratelimit.get_logger',
                     return_value=debug_logger()):
                 rl = factory(self.swift)
-            self.assertEqual(1.3, rl.requests_per_device_per_second)
+            exp_req_per_dev_per_sec = dict(self.default_req_per_dev_per_sec)
+            exp_req_per_dev_per_sec.update({None: 1.3})
+            self.assertEqual(exp_req_per_dev_per_sec,
+                             rl.requests_per_device_per_second)
             self.assertEqual(1.0, rl.requests_per_device_rate_buffer)
             self.assertEqual([], rl.logger.get_lines_for_level('error'))
             self.assertEqual(
@@ -141,13 +156,17 @@ class TestBackendRatelimitMiddleware(unittest.TestCase):
 
     def test_init_config_file_unset_and_missing(self):
         # don't warn if missing conf file during init (conf_path not set)
-        conf = {'requests_per_device_per_second': "1.3"}
+        conf = {'swift_dir': self.tempdir,
+                'requests_per_device_per_second': "1.3"}
         factory = backend_ratelimit.filter_factory(conf)
         with mock.patch(
                 'swift.common.middleware.backend_ratelimit.get_logger',
                 return_value=debug_logger()):
             rl = factory(self.swift)
-        self.assertEqual(1.3, rl.requests_per_device_per_second)
+        exp_req_per_dev_per_sec = dict(self.default_req_per_dev_per_sec)
+        exp_req_per_dev_per_sec[None] = 1.3
+        self.assertEqual(exp_req_per_dev_per_sec,
+                         rl.requests_per_device_per_second)
         self.assertEqual(1.0, rl.requests_per_device_rate_buffer)
         self.assertEqual([], rl.logger.get_lines_for_level('error'))
         self.assertEqual([], rl.logger.get_lines_for_level('warning'))
@@ -164,7 +183,11 @@ class TestBackendRatelimitMiddleware(unittest.TestCase):
         with mock.patch('swift.common.middleware.backend_ratelimit.get_logger',
                         return_value=debug_logger()):
             rl = factory(self.swift)
-        self.assertEqual(1.3, rl.requests_per_device_per_second)
+        exp_req_per_dev_per_sec = dict(self.default_req_per_dev_per_sec)
+        exp_req_per_dev_per_sec[None] = 1.3
+        self.assertEqual(exp_req_per_dev_per_sec,
+                         rl.requests_per_device_per_second)
+        self.assertEqual(1.0, rl.requests_per_device_rate_buffer)
         lines = rl.logger.get_lines_for_level('warning')
         self.assertEqual(1, len(lines), lines)
         self.assertIn('Invalid config file', lines[0])
@@ -186,9 +209,13 @@ class TestBackendRatelimitMiddleware(unittest.TestCase):
                         return_value=debug_logger()):
             rl = factory(self.swift)
         # backend-ratelimit.conf overrides options
-        self.assertEqual(12.3, rl.requests_per_device_per_second)
+        exp_req_per_dev_per_sec = dict(self.default_req_per_dev_per_sec)
+        exp_req_per_dev_per_sec[None] = 12.3
+        self.assertEqual(exp_req_per_dev_per_sec,
+                         rl.requests_per_device_per_second)
         # but only the ones that are listed
         self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
+        self.assertTrue(rl.is_any_rate_limit_configured)
         lines = rl.logger.get_lines_for_level('info')
         self.assertEqual(['Loaded config file %s, config changed' % conf_path],
                          lines)
@@ -210,32 +237,151 @@ class TestBackendRatelimitMiddleware(unittest.TestCase):
         with mock.patch('swift.common.middleware.backend_ratelimit.get_logger',
                         return_value=debug_logger()):
             rl = factory(self.swift)
-        # we DO read rate limit options
+        exp_req_per_dev_per_sec = dict(self.default_req_per_dev_per_sec)
-        self.assertEqual(12.3, rl.requests_per_device_per_second)
+        exp_req_per_dev_per_sec[None] = 12.3
+        self.assertEqual(exp_req_per_dev_per_sec,
+                         rl.requests_per_device_per_second)
         self.assertEqual(12.4, rl.requests_per_device_rate_buffer)
-        # but we do NOT read config reload options
+        self.assertTrue(rl.is_any_rate_limit_configured)
+        # options related to conf file loading are not loaded from conf file...
         self.assertEqual(conf_path, rl.conf_path)
         self.assertEqual(15, rl.config_reload_interval)
         lines = rl.logger.logger.get_lines_for_level('info')
         self.assertEqual(['Loaded config file %s, config changed' % conf_path],
                          lines)
 
-    def _do_test_config_file_reload(self, filter_conf, exp_reload_time):
+    def _do_test_init_config_file_overrides_filter_conf(
+            self, path_to_actual_conf_file, configured_conf_path):
+        # verify that conf file options override filter conf options
+        # create the actual file, but no options
+        with open(path_to_actual_conf_file, 'w') as fd:
+            fd.write('[backend_ratelimit]')
+        conf = {'swift_dir': self.tempdir,
+                'requests_per_device_per_second': "1.3",
+                'requests_per_device_rate_buffer': "2.4",
+                'config_reload_interval': 15}
+        if configured_conf_path:
+            # only configure if given a conf_path
+            conf['backend_ratelimit_conf_path'] = configured_conf_path
+            exp_configured_conf_path = configured_conf_path
+        else:
+            # fall back to default
+            exp_configured_conf_path = os.path.join(self.tempdir,
+                                                    'backend-ratelimit.conf')
+        factory = backend_ratelimit.filter_factory(conf)
+        rl = factory(self.swift)
+        exp_req_per_dev_per_sec = dict(self.default_req_per_dev_per_sec)
+        exp_req_per_dev_per_sec[None] = 1.3
+        self.assertEqual(exp_req_per_dev_per_sec,
+                         rl.requests_per_device_per_second)
+        self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
+        self.assertTrue(rl.is_any_rate_limit_configured)
+        self.assertEqual(exp_configured_conf_path, rl.conf_path)
+        self.assertEqual(15, rl.config_reload_interval)
+
+        # create file with option
+        with open(path_to_actual_conf_file, 'w') as fd:
+            fd.write('[backend_ratelimit]\n'
+                     'requests_per_device_per_second = 12.3\n'
+                     'backend_ratelimit_conf_path = /etc/swift/ignored.conf\n'
+                     'config_reload_interval = 999999\n')
+        factory = backend_ratelimit.filter_factory(conf)
+        rl = factory(self.swift)
+        exp_req_per_dev_per_sec[None] = 12.3
+        self.assertEqual(exp_req_per_dev_per_sec,
+                         rl.requests_per_device_per_second)
+        self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
+        self.assertTrue(rl.is_any_rate_limit_configured)
+        # options related to conf file loading are not loaded from conf file...
+        self.assertEqual(exp_configured_conf_path, rl.conf_path)
+        self.assertEqual(15, rl.config_reload_interval)
+
+        with open(path_to_actual_conf_file, 'w') as fd:
+            fd.write(
+                '[backend_ratelimit]\n'
+                'requests_per_device_per_second = 5.3\n'
+                'requests_per_device_rate_buffer = 0.5\n'
+                'delete_requests_per_device_per_second = 1\n'
+                'get_requests_per_device_per_second = 2\n'
+                'head_requests_per_device_per_second = 3\n'
+                'post_requests_per_device_per_second = 4\n'
+                'put_requests_per_device_per_second = 5\n'
+                'replicate_requests_per_device_per_second = 6\n'
+                'update_requests_per_device_per_second = 7\n'
+                'backend_ratelimit_conf_path = /etc/swift/ignored.conf\n'
+                'config_reload_interval = 999999\n'
+            )
+        factory = backend_ratelimit.filter_factory(conf)
+        rl = factory(self.swift)
+        exp_req_per_dev_per_sec.update(
+            {
+                None: 5.3,
+                'DELETE': 1,
+                'GET': 2,
+                'HEAD': 3,
+                'POST': 4,
+                'PUT': 5,
+                'REPLICATE': 6,
+                'UPDATE': 7,
+            }
+        )
+        self.assertEqual(exp_req_per_dev_per_sec,
+                         rl.requests_per_device_per_second)
+        self.assertEqual(0.5, rl.requests_per_device_rate_buffer)
+        self.assertTrue(rl.is_any_rate_limit_configured)
+        # options related to conf file loading are not loaded from conf file...
+        self.assertEqual(exp_configured_conf_path, rl.conf_path)
+        self.assertEqual(15, rl.config_reload_interval)
+
+    def test_init_config_file_at_default_path_overrides_filter_conf(self):
+        # default conf path is loaded if it exists
+        default_conf_path = os.path.join(self.tempdir,
+                                         'backend-ratelimit.conf')
+        self._do_test_init_config_file_overrides_filter_conf(
+            path_to_actual_conf_file=default_conf_path,
+            configured_conf_path=None)
+
+        self._do_test_init_config_file_overrides_filter_conf(
+            path_to_actual_conf_file=default_conf_path,
+            configured_conf_path=default_conf_path)
+
+    def test_init_config_file_at_configured_path_overrides_filter_conf(self):
+        # explicitly configured conf path is loaded
+        custom_conf_path = os.path.join(self.tempdir, 'backend_rl.conf')
+        self._do_test_init_config_file_overrides_filter_conf(
+            path_to_actual_conf_file=custom_conf_path,
+            configured_conf_path=custom_conf_path)
+
+    def _do_test_config_file_reload(self, reload_interval):
         # verify that conf file options are periodically reloaded
+        filter_conf = {'swift_dir': self.tempdir,
+                       'requests_per_device_per_second': "1.3",
+                       'requests_per_device_rate_buffer': "2.4",
+                       'head_requests_per_device_per_second': '6.2'}
+        if reload_interval:
+            filter_conf['config_reload_interval'] = reload_interval
+
         now = time.time()
         # create the actual file
-        conf_path = os.path.join(self.tempdir, 'backend-ratelimit.conf')
+        conf_path = os.path.join(filter_conf['swift_dir'],
+                                 'backend-ratelimit.conf')
         with open(conf_path, 'w') as fd:
             fd.write('[backend_ratelimit]\n'
                      'requests_per_device_per_second = 12.3\n'
-                     'backend_ratelimit_conf_path = /etc/swift/rl.conf\n'
+                     # conf file cannot re-configure where the conf file is...
-                     'config_reload_interval = 999999\n')
+                     'backend_ratelimit_conf_path = /etc/ignored\n'
+                     'config_reload_interval = also_ignored\n')
         factory = backend_ratelimit.filter_factory(filter_conf)
         with mock.patch('swift.common.middleware.backend_ratelimit.time.time',
                         return_value=now):
             rl = factory(self.swift)
-        self.assertEqual(12.3, rl.requests_per_device_per_second)
+        exp_req_per_dev_per_sec = dict(self.default_req_per_dev_per_sec)
-        self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
+        exp_req_per_dev_per_sec.update({None: 12.3, 'HEAD': float(
+            filter_conf['head_requests_per_device_per_second'])})
+        self.assertEqual(exp_req_per_dev_per_sec,
+                         rl.requests_per_device_per_second)
+        self.assertEqual(float(filter_conf['requests_per_device_rate_buffer']),
+                         rl.requests_per_device_rate_buffer)
         self.assertEqual(conf_path, rl.conf_path)
 
         # modify the conf file
@@ -243,104 +389,198 @@ class TestBackendRatelimitMiddleware(unittest.TestCase):
             fd.write('[backend_ratelimit]\n'
                      'requests_per_device_per_second = 29.3\n'
                      'requests_per_device_rate_buffer = 12.4\n'
-                     'backend_ratelimit_conf_path = /etc/swift/rl.conf\n'
+                     'backend_ratelimit_conf_path = /etc/ignored\n'
-                     'config_reload_interval = 999999\n')
+                     'config_reload_interval = also_ignored\n'
+                     'head_requests_per_device_per_second = 5.1\n'
+                     'delete_requests_per_device_per_second = 7.3\n'
+                     'get_requests_per_device_per_second = 8.4\n')
 
         # send some requests, but too soon for config file to be reloaded
         req1 = Request.blank('/sda1/99/a/c/o')
-        req2 = Request.blank('/sda2/99/a/c/o')
+        req2 = Request.blank('/sda2/99/a/c/o',
+                             environ={'REQUEST_METHOD': 'DELETE'})
         self.swift.register(req1.method, req1.path, HTTPOk, {})
         self.swift.register(req2.method, req2.path, HTTPOk, {})
         with mock.patch('swift.common.middleware.backend_ratelimit.time.time',
-                        return_value=now + exp_reload_time - 1):
+                        return_value=now + rl.config_reload_interval - 1):
             resp1 = req1.get_response(rl)
             resp2 = req2.get_response(rl)
         self.assertEqual(200, resp1.status_int)
         self.assertEqual(200, resp2.status_int)
-        self.assertEqual(12.3, rl.requests_per_device_per_second)
+        self.assertEqual(exp_req_per_dev_per_sec,
-        self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
+                         rl.requests_per_device_per_second)
+        self.assertEqual(float(filter_conf['requests_per_device_rate_buffer']),
+                         rl.requests_per_device_rate_buffer)
         self.assertEqual(conf_path, rl.conf_path)
 
+        # verify the per dev ratelimiters
+        self.assertEqual({('sda1', 'GET'): 0.0,
+                          ('sda2', 'DELETE'): 0.0,
+                          ('sda1', None): 12.3,
+                          ('sda2', None): 12.3},
+                         dict((key, val.max_rate)
+                              for key, val in rl.rate_limiters.items()))
+        for (dev, method), limiter in rl.rate_limiters.items():
+            self.assertEqual(2.4 * limiter.clock_accuracy,
+                             limiter.rate_buffer_ms, (dev, method))
+
         # send some requests, time for config file to be reloaded
         with mock.patch('swift.common.middleware.backend_ratelimit.time.time',
-                        return_value=now + exp_reload_time):
+                        return_value=now + rl.config_reload_interval + 0.01):
             resp1 = req1.get_response(rl)
             resp2 = req2.get_response(rl)
         self.assertEqual(200, resp1.status_int)
         self.assertEqual(200, resp2.status_int)
-        self.assertEqual(29.3, rl.requests_per_device_per_second)
+        exp_req_per_dev_per_sec.update({
+            None: 29.3,
+            'HEAD': 5.1,
+            'DELETE': 7.3,
+            'GET': 8.4})
+        self.assertEqual(exp_req_per_dev_per_sec,
+                         rl.requests_per_device_per_second)
         self.assertEqual(12.4, rl.requests_per_device_rate_buffer)
         self.assertEqual(conf_path, rl.conf_path)
 
         # verify the per dev ratelimiters were updated
-        per_dev_rl_rates = [per_dev_rl.max_rate
+        self.assertEqual({('sda1', 'GET'): 8.4,
-                            for per_dev_rl in list(rl.rate_limiters.values())]
+                          ('sda2', 'DELETE'): 7.3,
-        self.assertEqual([29.3, 29.3], per_dev_rl_rates)
+                          ('sda1', None): 29.3,
-        per_dev_rl_buffer = [per_dev_rl.rate_buffer_ms
+                          ('sda2', None): 29.3},
-                             for per_dev_rl in list(rl.rate_limiters.values())]
+                         dict((key, val.max_rate)
-        self.assertEqual([12400, 12400], sorted(per_dev_rl_buffer))
+                              for key, val in rl.rate_limiters.items()))
+        for (dev, method), limiter in rl.rate_limiters.items():
+            self.assertEqual(12.4 * limiter.clock_accuracy,
+                             limiter.rate_buffer_ms, (dev, method))
 
         # modify the config file again
         # remove requests_per_device_per_second option
+        # remove [head|delete]_requests_per_device_per_second options
         with open(conf_path, 'w') as fd:
             fd.write('[backend_ratelimit]\n'
-                     'backend_ratelimit_conf_path = /etc/swift/rl.conf\n'
+                     'backend_ratelimit_conf_path = /etc/ignored\n'
-                     'config_reload_interval = 999999\n')
+                     'config_reload_interval = also_ignored\n'
+                     'requests_per_device_rate_buffer = 0.5\n'
+                     'get_requests_per_device_per_second = 9.5\n')
 
         # send some requests, not yet time for config file to be reloaded
         with mock.patch('swift.common.middleware.backend_ratelimit.time.time',
-                        return_value=now + 2 * exp_reload_time - 1):
+                        return_value=now + 2 * rl.config_reload_interval - 1):
             resp1 = req1.get_response(rl)
             resp2 = req2.get_response(rl)
         self.assertEqual(200, resp1.status_int)
         self.assertEqual(200, resp2.status_int)
-        self.assertEqual(29.3, rl.requests_per_device_per_second)
+        self.assertEqual(exp_req_per_dev_per_sec,
+                         rl.requests_per_device_per_second)
         self.assertEqual(12.4, rl.requests_per_device_rate_buffer)
         self.assertEqual(conf_path, rl.conf_path)
 
         # verify the per dev ratelimiters were not updated
-        per_dev_rl_rates = [per_dev_rl.max_rate
+        self.assertEqual({('sda1', 'GET'): 8.4,
-                            for per_dev_rl in list(rl.rate_limiters.values())]
+                          ('sda2', 'DELETE'): 7.3,
-        self.assertEqual([29.3, 29.3], sorted(per_dev_rl_rates))
+                          ('sda1', None): 29.3,
-        per_dev_rl_buffer = [per_dev_rl.rate_buffer_ms
+                          ('sda2', None): 29.3},
-                             for per_dev_rl in list(rl.rate_limiters.values())]
+                         dict((key, val.max_rate)
-        self.assertEqual([12400, 12400], sorted(per_dev_rl_buffer))
+                              for key, val in rl.rate_limiters.items()))
+        for (dev, method), limiter in rl.rate_limiters.items():
+            self.assertEqual(12.4 * limiter.clock_accuracy,
+                             limiter.rate_buffer_ms, (dev, method))
 
         # send some requests, time for config file to be reloaded
-        with mock.patch('swift.common.middleware.backend_ratelimit.time.time',
+        with mock.patch(
-                        return_value=now + 2 * exp_reload_time):
+                'swift.common.middleware.backend_ratelimit.time.time',
+                return_value=now + 2 * rl.config_reload_interval + 0.01):
             resp1 = req1.get_response(rl)
             resp2 = req2.get_response(rl)
         self.assertEqual(200, resp1.status_int)
         self.assertEqual(200, resp2.status_int)
         # requests_per_device_per_second option reverts to filter conf
-        self.assertEqual(1.3, rl.requests_per_device_per_second)
+        # delete_requests_per_device_per_second option reverts to default
-        self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
+        # head_requests_per_device_per_second option reverts to filter conf
+        exp_req_per_dev_per_sec.update({
+            None: 1.3,
+            'HEAD': 6.2,
+            'DELETE': 0.0,
+            'GET': 9.5})
+        self.assertEqual(exp_req_per_dev_per_sec,
+                         rl.requests_per_device_per_second)
+        self.assertEqual(0.5, rl.requests_per_device_rate_buffer)
         self.assertEqual(conf_path, rl.conf_path)
 
-        # verify the per dev ratelimiters were not updated
+        # verify the per dev ratelimiters were updated
-        per_dev_rl_rates = [per_dev_rl.max_rate
+        self.assertEqual({('sda1', 'GET'): 9.5,
-                            for per_dev_rl in list(rl.rate_limiters.values())]
+                          ('sda2', 'DELETE'): 0.0,
-        self.assertEqual([1.3, 1.3], sorted(per_dev_rl_rates))
+                          ('sda1', None): 1.3,
-        per_dev_rl_buffer = [per_dev_rl.rate_buffer_ms
+                          ('sda2', None): 1.3},
-                             for per_dev_rl in list(rl.rate_limiters.values())]
+                         dict((key, val.max_rate)
-        self.assertEqual([2400, 2400], sorted(per_dev_rl_buffer))
+                              for key, val in rl.rate_limiters.items()))
+        for (dev, method), limiter in rl.rate_limiters.items():
+            self.assertEqual(0.5 * limiter.clock_accuracy,
+                             limiter.rate_buffer_ms, (dev, method))
         return rl
 
     def test_config_file_reload_default_interval(self):
-        filter_conf = {'swift_dir': self.tempdir,
+        rl = self._do_test_config_file_reload(None)
-                       'requests_per_device_per_second': "1.3",
-                       'requests_per_device_rate_buffer': "2.4"}
-        rl = self._do_test_config_file_reload(filter_conf, 60)
         self.assertEqual(60, rl.config_reload_interval)
 
     def test_config_file_reload_custom_interval(self):
+        rl = self._do_test_config_file_reload(30.1)
+        self.assertEqual(30.1, rl.config_reload_interval)
+
+    def test_config_file_reload_clears_all_limits(self):
+        # verify that reloaded config file can disable all rate limits
+        now = time.time()
+        conf_path = os.path.join(self.tempdir, 'missing')
         filter_conf = {'swift_dir': self.tempdir,
-                       'config_reload_interval': "30",
+                       # path set so expect warning during init
+                       'backend_ratelimit_conf_path': conf_path,
                        'requests_per_device_per_second': "1.3",
+                       'head_requests_per_device_per_second = 1.1\n'
                        'requests_per_device_rate_buffer': "2.4"}
-        rl = self._do_test_config_file_reload(filter_conf, 30)
+        with open(conf_path, 'w') as fd:
-        self.assertEqual(30, rl.config_reload_interval)
+            fd.write('[backend_ratelimit]\n'
+                     'requests_per_device_per_second = 29.3\n'
+                     'head_requests_per_device_per_second = 5.1\n'
+                     'get_requests_per_device_per_second = 8.4\n')
+        factory = backend_ratelimit.filter_factory(filter_conf)
+
+        # expect warning during init
+        with mock.patch('swift.common.middleware.backend_ratelimit.time.time',
+                        return_value=now):
+            with mock.patch(
+                    'swift.common.middleware.backend_ratelimit.get_logger',
+                    return_value=debug_logger()):
+                rl = factory(self.swift)
+        # filter conf has been applied
+        exp_req_per_dev_per_sec = dict(self.default_req_per_dev_per_sec)
+        exp_req_per_dev_per_sec.update({
+            None: 29.3,
+            'HEAD': 5.1,
+            'GET': 8.4})
+        self.assertTrue(rl.is_any_rate_limit_configured)
+        self.assertEqual(exp_req_per_dev_per_sec,
+                         rl.requests_per_device_per_second)
+        self.assertEqual([], rl.logger.get_lines_for_level('warning'))
+        self.assertEqual([], rl.logger.get_lines_for_level('error'))
+
+        # write zero rate limits to conf file
+        # jump into future, send request, config reload attempted
+        with open(conf_path, 'w') as fd:
+            fd.write('[backend_ratelimit]\n'
+                     'requests_per_device_per_second = 0.0\n'
+                     'head_requests_per_device_per_second = 0.0\n'
+                     'get_requests_per_device_per_second = 0.0\n')
+        req1 = Request.blank('/sda1/99/a/c/o')
+        self.swift.register(req1.method, req1.path, HTTPOk, {})
+        with mock.patch('swift.common.middleware.backend_ratelimit.time.time',
+                        return_value=now + 10000):
+            resp1 = req1.get_response(rl)
+        self.assertEqual(200, resp1.status_int)
+        exp_req_per_dev_per_sec = dict(self.default_req_per_dev_per_sec)
+        self.assertFalse(rl.is_any_rate_limit_configured)
+        self.assertEqual(exp_req_per_dev_per_sec,
+                         rl.requests_per_device_per_second)
+        self.assertEqual([], rl.logger.get_lines_for_level('warning'))
+        self.assertEqual([], rl.logger.get_lines_for_level('error'))
 
     def test_config_file_reload_set_and_missing(self):
         now = time.time()
@@ -360,7 +600,11 @@ class TestBackendRatelimitMiddleware(unittest.TestCase):
                     return_value=debug_logger()):
                 rl = factory(self.swift)
         # filter conf has been applied
-        self.assertEqual(1.3, rl.requests_per_device_per_second)
+        exp_req_per_dev_per_sec = dict(self.default_req_per_dev_per_sec)
+        exp_req_per_dev_per_sec[None] = 1.3
+        self.assertTrue(rl.is_any_rate_limit_configured)
+        self.assertEqual(exp_req_per_dev_per_sec,
+                         rl.requests_per_device_per_second)
         self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
         self.assertEqual(
             ['Failed to load config file, config unchanged: Unable to read '
@@ -377,7 +621,9 @@ class TestBackendRatelimitMiddleware(unittest.TestCase):
                         return_value=now + 10000):
             resp1 = req1.get_response(rl)
         self.assertEqual(200, resp1.status_int)
-        self.assertEqual(1.3, rl.requests_per_device_per_second)
+        self.assertTrue(rl.is_any_rate_limit_configured)
+        self.assertEqual(exp_req_per_dev_per_sec,
+                         rl.requests_per_device_per_second)
         self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
         self.assertEqual([], rl.logger.get_lines_for_level('warning'))
         self.assertEqual([], rl.logger.get_lines_for_level('error'))
@@ -398,7 +644,10 @@ class TestBackendRatelimitMiddleware(unittest.TestCase):
                     return_value=debug_logger()):
                 rl = factory(self.swift)
         # filter conf has been applied
-        self.assertEqual(1.3, rl.requests_per_device_per_second)
+        exp_req_per_dev_per_sec = dict(self.default_req_per_dev_per_sec)
+        exp_req_per_dev_per_sec[None] = 1.3
+        self.assertEqual(exp_req_per_dev_per_sec,
+                         rl.requests_per_device_per_second)
         self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
         self.assertEqual([], rl.logger.get_lines_for_level('warning'))
         self.assertEqual([], rl.logger.get_lines_for_level('error'))
@@ -412,7 +661,8 @@ class TestBackendRatelimitMiddleware(unittest.TestCase):
             resp1 = req1.get_response(rl)
         self.assertEqual(200, resp1.status_int)
         # previous conf file value has been retained
-        self.assertEqual(1.3, rl.requests_per_device_per_second)
+        self.assertEqual(exp_req_per_dev_per_sec,
+                         rl.requests_per_device_per_second)
         self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
         self.assertEqual([], rl.logger.get_lines_for_level('warning'))
         self.assertEqual([], rl.logger.get_lines_for_level('error'))
@@ -435,7 +685,10 @@ class TestBackendRatelimitMiddleware(unittest.TestCase):
                     return_value=debug_logger()):
                 rl = factory(self.swift)
         # conf file value has been applied
-        self.assertEqual(1.3, rl.requests_per_device_per_second)
+        exp_req_per_dev_per_sec = dict(self.default_req_per_dev_per_sec)
+        exp_req_per_dev_per_sec[None] = 1.3
+        self.assertEqual(exp_req_per_dev_per_sec,
+                         rl.requests_per_device_per_second)
         self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
         self.assertEqual([], rl.logger.get_lines_for_level('warning'))
         self.assertEqual([], rl.logger.get_lines_for_level('error'))
@@ -459,7 +712,10 @@ class TestBackendRatelimitMiddleware(unittest.TestCase):
                     return_value=debug_logger()):
                 rl = factory(self.swift)
         # conf file value has been applied
-        self.assertEqual(12.3, rl.requests_per_device_per_second)
+        exp_req_per_dev_per_sec = dict(self.default_req_per_dev_per_sec)
+        exp_req_per_dev_per_sec[None] = 12.3
+        self.assertEqual(exp_req_per_dev_per_sec,
+                         rl.requests_per_device_per_second)
         self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
         self.assertEqual([], rl.logger.get_lines_for_level('warning'))
         self.assertEqual([], rl.logger.get_lines_for_level('error'))
@@ -480,7 +736,8 @@ class TestBackendRatelimitMiddleware(unittest.TestCase):
                 resp1 = req1.get_response(rl)
         self.assertEqual(200, resp1.status_int)
         # previous conf file value has been retained
-        self.assertEqual(12.3, rl.requests_per_device_per_second)
+        self.assertEqual(exp_req_per_dev_per_sec,
+                         rl.requests_per_device_per_second)
         self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
         mock_readconf.assert_called_once()
         self.assertEqual(
@@ -495,7 +752,8 @@ class TestBackendRatelimitMiddleware(unittest.TestCase):
             resp1 = req1.get_response(rl)
         self.assertEqual(200, resp1.status_int)
         # previous conf file value has been retained
-        self.assertEqual(12.3, rl.requests_per_device_per_second)
+        self.assertEqual(exp_req_per_dev_per_sec,
+                         rl.requests_per_device_per_second)
         self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
         self.assertEqual([], rl.logger.get_lines_for_level('warning'))
         self.assertEqual([], rl.logger.get_lines_for_level('error'))
@@ -506,8 +764,10 @@ class TestBackendRatelimitMiddleware(unittest.TestCase):
                         return_value=now + 10060):
             resp1 = req1.get_response(rl)
         self.assertEqual(200, resp1.status_int)
-        # updated conf file value is applied
+        # previous conf file value has been retained
-        self.assertEqual(29.3, rl.requests_per_device_per_second)
+        exp_req_per_dev_per_sec.update({None: 29.3})
+        self.assertEqual(exp_req_per_dev_per_sec,
+                         rl.requests_per_device_per_second)
         self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
         self.assertEqual([], rl.logger.get_lines_for_level('warning'))
         self.assertEqual([], rl.logger.get_lines_for_level('error'))
@@ -531,7 +791,10 @@ class TestBackendRatelimitMiddleware(unittest.TestCase):
                     return_value=debug_logger()):
                 rl = factory(self.swift)
         # conf file value has been applied
-        self.assertEqual(12.3, rl.requests_per_device_per_second)
+        exp_req_per_dev_per_sec = dict(self.default_req_per_dev_per_sec)
+        exp_req_per_dev_per_sec[None] = 12.3
+        self.assertEqual(exp_req_per_dev_per_sec,
+                         rl.requests_per_device_per_second)
         self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
         lines = rl.logger.get_lines_for_level('info')
         self.assertEqual(['Loaded config file %s, config changed' % conf_path],
@@ -545,7 +808,8 @@ class TestBackendRatelimitMiddleware(unittest.TestCase):
                         return_value=now + 10000):
             resp1 = req1.get_response(rl)
         self.assertEqual(200, resp1.status_int)
-        self.assertEqual(12.3, rl.requests_per_device_per_second)
+        self.assertEqual(exp_req_per_dev_per_sec,
+                         rl.requests_per_device_per_second)
         self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
         lines = rl.logger.get_lines_for_level('info')
         self.assertEqual([], lines)
@@ -560,7 +824,9 @@ class TestBackendRatelimitMiddleware(unittest.TestCase):
             resp1 = req1.get_response(rl)
         self.assertEqual(200, resp1.status_int)
         # previous conf file value has been retained
-        self.assertEqual(23.4, rl.requests_per_device_per_second)
+        exp_req_per_dev_per_sec.update({None: 23.4})
+        self.assertEqual(exp_req_per_dev_per_sec,
+                         rl.requests_per_device_per_second)
         self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
         lines = rl.logger.get_lines_for_level('info')
         self.assertEqual(['Loaded config file %s, config changed' % conf_path],
@@ -586,7 +852,10 @@ class TestBackendRatelimitMiddleware(unittest.TestCase):
                     return_value=debug_logger()):
                 rl = factory(self.swift)
         # conf file value has been applied
-        self.assertEqual(12.3, rl.requests_per_device_per_second)
+        exp_req_per_dev_per_sec = dict(self.default_req_per_dev_per_sec)
+        exp_req_per_dev_per_sec[None] = 12.3
+        self.assertEqual(exp_req_per_dev_per_sec,
+                         rl.requests_per_device_per_second)
         self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
         lines = rl.logger.get_lines_for_level('info')
         self.assertEqual(
@@ -656,7 +925,10 @@ class TestBackendRatelimitMiddleware(unittest.TestCase):
         with mock.patch('swift.common.middleware.backend_ratelimit.time.time',
                         return_value=now):
             rl = factory(self.swift)
-        self.assertEqual(12.3, rl.requests_per_device_per_second)
+        exp_req_per_dev_per_sec = dict(self.default_req_per_dev_per_sec)
+        exp_req_per_dev_per_sec[None] = 12.3
+        self.assertEqual(exp_req_per_dev_per_sec,
+                         rl.requests_per_device_per_second)
         self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
 
         with open(conf_path, 'w') as fd:
@@ -671,10 +943,14 @@ class TestBackendRatelimitMiddleware(unittest.TestCase):
             resp = req.get_response(rl)
         self.assertEqual(200, resp.status_int)
         # no change
-        self.assertEqual(12.3, rl.requests_per_device_per_second)
+        exp_req_per_dev_per_sec = dict(self.default_req_per_dev_per_sec)
+        exp_req_per_dev_per_sec[None] = 12.3
+        self.assertEqual(exp_req_per_dev_per_sec,
+                         rl.requests_per_device_per_second)
         self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
 
-    def _do_test_ratelimit(self, method, req_per_sec, rate_buffer):
+    def _do_test_ratelimit(self, method, req_per_sec, rate_buffer,
+                           extra_conf=None):
         # send 20 requests, time increments by 0.01 between each request
         start = time.time()
         fake_time = [start]
@@ -685,8 +961,11 @@ class TestBackendRatelimitMiddleware(unittest.TestCase):
         app = FakeSwift()
         logger = debug_logger()
         # apply a ratelimit
-        conf = {'requests_per_device_per_second': req_per_sec,
+        conf = {'swift_dir': self.tempdir,
+                'requests_per_device_per_second': req_per_sec,
                 'requests_per_device_rate_buffer': rate_buffer}
+        if extra_conf:
+            conf.update(extra_conf)
         rl = BackendRateLimitMiddleware(app, conf, logger)
         success = defaultdict(int)
         ratelimited = 0
@@ -712,13 +991,13 @@ class TestBackendRatelimitMiddleware(unittest.TestCase):
                 'backend.ratelimit', 0))
         return success
 
-    def test_ratelimited(self):
+    def test_method_ratelimited(self):
         def do_test_ratelimit(method):
             # no rate-limiting
             success_per_dev = self._do_test_ratelimit(method, 0, 0)
             self.assertEqual([20] * 3, list(success_per_dev.values()))
 
-            # rate-limited
+            # global rate-limited
             success_per_dev = self._do_test_ratelimit(method, 1, 0)
             self.assertEqual([1] * 3, list(success_per_dev.values()))
 
@@ -734,6 +1013,20 @@ class TestBackendRatelimitMiddleware(unittest.TestCase):
             success_per_dev = self._do_test_ratelimit(method, 10, 1)
             self.assertEqual([12] * 3, list(success_per_dev.values()))
 
+            # method rate-limited
+            extra_conf = {
+                '%s_requests_per_device_per_second' % method.lower(): 1
+            }
+            success_per_dev = self._do_test_ratelimit(method, 0, 0, extra_conf)
+            self.assertEqual([1] * 3, list(success_per_dev.values()))
+
+            # method not rate-limited, global rate limited
+            extra_conf = {
+                '%s_requests_per_device_per_second' % method.lower(): 100
+            }
+            success_per_dev = self._do_test_ratelimit(method, 1, 0, extra_conf)
+            self.assertEqual([1] * 3, list(success_per_dev.values()))
+
         do_test_ratelimit('GET')
         do_test_ratelimit('HEAD')
         do_test_ratelimit('PUT')
@@ -742,7 +1035,7 @@ class TestBackendRatelimitMiddleware(unittest.TestCase):
         do_test_ratelimit('UPDATE')
         do_test_ratelimit('REPLICATE')
 
-    def test_not_ratelimited(self):
+    def test_method_not_ratelimited(self):
         def do_test_no_ratelimit(method):
             # verify no rate-limiting
             success_per_dev = self._do_test_ratelimit(method, 1, 0)
@@ -751,6 +1044,15 @@ class TestBackendRatelimitMiddleware(unittest.TestCase):
         do_test_no_ratelimit('OPTIONS')
         do_test_no_ratelimit('SSYNC')
 
+    def test_no_ratelimiting_configured(self):
+        # verify shortcut path when no ratelimiting is configured
+        with mock.patch(
+                'swift.common.middleware.backend_ratelimit.'
+                'BackendRateLimitMiddleware._is_allowed') as mock_is_allowed:
+            success_per_dev = self._do_test_ratelimit('GET', 0, 0)
+        self.assertEqual([20] * 3, list(success_per_dev.values()))
+        mock_is_allowed.assert_not_called()
+
     def test_unhandled_request(self):
         app = FakeSwift()
         logger = debug_logger()