backend ratelimit: support reloadable config file

Add support for a backend_ratelimit_conf_path option in the
[filter:backend_ratelimit] config. If specified then the middleware
will give precedence to config options from that file over config
options from the [filter:backend_ratelimit] section.

The path defaults to /etc/swift/backend-ratelimit.conf.

The config file is periodically reloaded and any changed options are
applied. The middleware will log a warning the first time it fails to
load a config file that had previously been successfully loaded. The
middleware also logs at info level when it first successfully loads a
config file that had previously failed to be loaded. Otherwise, the
middleware will log when a config file is loaded that results in the
config being changed.

Change-Id: I6554e37c6ab5b0a260f99b54169cb90ab5718f81

etc/account-server.conf-sample

@@ -130,15 +130,17 @@ use = egg:swift#recon
 
 [filter:backend_ratelimit]
 use = egg:swift#backend_ratelimit
-# Set the maximum rate of requests per second per device per worker. Beyond
+# Config options can optionally be loaded from a separate config file. Config
-# this rate the server will return 529 responses and emit a 'backend.ratelimit'
+# options in this section will be used unless the same option is found in the
-# statsd metric without logging. The default value of zero causes no
+# config file, in which case the config file option will be used. See the
-# rate-limiting to be applied.
+# backend-ratelimit.conf-sample file for details of available config options.
-# requests_per_device_per_second = 0.0
+# backend_ratelimit_conf_path = /etc/swift/backend-ratelimit.conf
-#
+
-# Set the number of seconds of unused rate-limiting allowance that can
+# The minimum interval between attempts to reload any config file at
-# accumulate and be used to allow a subsequent burst of requests.
+# backend_ratelimit_conf_path while the server is running. A value of 0 means
-# requests_per_device_rate_buffer = 1.0
+# that the file is loaded at start-up but not subsequently reloaded. Note that
+# config options in this section are never reloaded after start-up.
+# config_reload_interval = 60
 
 [account-replicator]
 # You can override the default log routing for this app here (don't use set!):

etc/backend-ratelimit.conf-sample

@@ -0,0 +1,10 @@
+[backend_ratelimit]
+# Set the maximum rate of requests per second per device per worker. Beyond
+# this rate the server will return 529 responses and emit a 'backend.ratelimit'
+# statsd metric without logging. The default value of zero causes no
+# rate-limiting to be applied.
+# requests_per_device_per_second = 0.0
+#
+# Set the number of seconds of unused rate-limiting allowance that can
+# accumulate and be used to allow a subsequent burst of requests.
+# requests_per_device_rate_buffer = 1.0

etc/container-server.conf-sample

@@ -140,15 +140,17 @@ use = egg:swift#recon
 
 [filter:backend_ratelimit]
 use = egg:swift#backend_ratelimit
-# Set the maximum rate of requests per second per device per worker. Beyond
+# Config options can optionally be loaded from a separate config file. Config
-# this rate the server will return 529 responses and emit a 'backend.ratelimit'
+# options in this section will be used unless the same option is found in the
-# statsd metric without logging. The default value of zero causes no
+# config file, in which case the config file option will be used. See the
-# rate-limiting to be applied.
+# backend-ratelimit.conf-sample file for details of available config options.
-# requests_per_device_per_second = 0.0
+# backend_ratelimit_conf_path = /etc/swift/backend-ratelimit.conf
-#
+
-# Set the number of seconds of unused rate-limiting allowance that can
+# The minimum interval between attempts to reload any config file at
-# accumulate and be used to allow a subsequent burst of requests.
+# backend_ratelimit_conf_path while the server is running. A value of 0 means
-# requests_per_device_rate_buffer = 1.0
+# that the file is loaded at start-up but not subsequently reloaded. Note that
+# config options in this section are never reloaded after start-up.
+# config_reload_interval = 60
 
 [container-replicator]
 # You can override the default log routing for this app here (don't use set!):

etc/object-server.conf-sample

@@ -244,15 +244,17 @@ use = egg:swift#recon
 
 [filter:backend_ratelimit]
 use = egg:swift#backend_ratelimit
-# Set the maximum rate of requests per second per device per worker. Beyond
+# Config options can optionally be loaded from a separate config file. Config
-# this rate the server will return 529 responses and emit a 'backend.ratelimit'
+# options in this section will be used unless the same option is found in the
-# statsd metric without logging. The default value of zero causes no
+# config file, in which case the config file option will be used. See the
-# rate-limiting to be applied.
+# backend-ratelimit.conf-sample file for details of available config options.
-# requests_per_device_per_second = 0.0
+# backend_ratelimit_conf_path = /etc/swift/backend-ratelimit.conf
-#
+
-# Set the number of seconds of unused rate-limiting allowance that can
+# The minimum interval between attempts to reload any config file at
-# accumulate and be used to allow a subsequent burst of requests.
+# backend_ratelimit_conf_path while the server is running. A value of 0 means
-# requests_per_device_rate_buffer = 1.0
+# that the file is loaded at start-up but not subsequently reloaded. Note that
+# config options in this section are never reloaded after start-up.
+# config_reload_interval = 60
 
 [object-replicator]
 # You can override the default log routing for this app here (don't use set!):

swift/common/middleware/backend_ratelimit.py

@@ -12,7 +12,7 @@
 # implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import os
 import time
 from collections import defaultdict
 
@@ -20,10 +20,15 @@ from swift.common.request_helpers import split_and_validate_path
 from swift.common.swob import Request, HTTPTooManyBackendRequests, \
     HTTPException
 from swift.common.utils import get_logger, non_negative_float, \
-    EventletRateLimiter
+    EventletRateLimiter, readconf
 
 RATE_LIMITED_METHODS = ('GET', 'HEAD', 'PUT', 'POST', 'DELETE', 'UPDATE',
                         'REPLICATE')
+BACKEND_RATELIMIT_CONFIG_SECTION = 'backend_ratelimit'
+DEFAULT_BACKEND_RATELIMIT_CONF_FILE = 'backend-ratelimit.conf'
+DEFAULT_CONFIG_RELOAD_INTERVAL = 60.0
+DEFAULT_REQUESTS_PER_DEVICE_PER_SECOND = 0.0
+DEFAULT_REQUESTS_PER_DEVICE_RATE_BUFFER = 1.0
 
 
 class BackendRateLimitMiddleware(object):
@@ -38,14 +43,16 @@ class BackendRateLimitMiddleware(object):
     If a request would cause the rate-limit to be exceeded then a response with
     a 529 status code is returned.
     """
-    def __init__(self, app, conf, logger=None):
+    def __init__(self, app, filter_conf, logger=None):
         self.app = app
-        self.logger = logger or get_logger(conf, log_route='backend_ratelimit')
+        self.filter_conf = filter_conf
-        self.requests_per_device_per_second = non_negative_float(
+        self.current_conf = {}
-            conf.get('requests_per_device_per_second', 0.0))
+        self.logger = logger or get_logger(self.filter_conf,
-        self.requests_per_device_rate_buffer = non_negative_float(
+                                           log_route='backend_ratelimit')
-            conf.get('requests_per_device_rate_buffer', 1.0))
+        self.requests_per_device_per_second = \
-
+            DEFAULT_REQUESTS_PER_DEVICE_PER_SECOND
+        self.requests_per_device_rate_buffer = \
+            DEFAULT_REQUESTS_PER_DEVICE_RATE_BUFFER
         # map device -> RateLimiter
         self.rate_limiters = defaultdict(
             lambda: EventletRateLimiter(
@@ -54,6 +61,95 @@ class BackendRateLimitMiddleware(object):
                 running_time=time.time(),
                 burst_after_idle=True))
 
+        # some config options are *only* read from filter conf at startup...
+        default_conf_path = os.path.join(
+            self.filter_conf.get('swift_dir', '/etc/swift'),
+            DEFAULT_BACKEND_RATELIMIT_CONF_FILE)
+        try:
+            self.conf_path = self.filter_conf['backend_ratelimit_conf_path']
+            self.is_config_file_expected = True
+        except KeyError:
+            self.conf_path = default_conf_path
+            self.is_config_file_expected = False
+        self.config_reload_interval = non_negative_float(
+            filter_conf.get('config_reload_interval',
+                            DEFAULT_CONFIG_RELOAD_INTERVAL))
+
+        # other conf options are read from filter section at startup but may
+        # also be overridden by options in a separate config file...
+        self._last_config_reload_attempt = time.time()
+        self._apply_config(self.filter_conf)
+        self._load_config_file()
+
+    def _refresh_ratelimiters(self):
+        for dev, rl in self.rate_limiters.items():
+            rl.set_max_rate(self.requests_per_device_per_second)
+            rl.set_rate_buffer(self.requests_per_device_rate_buffer)
+
+    def _apply_config(self, conf):
+        self.current_conf = conf
+        modified = False
+        new_value = non_negative_float(
+            conf.get('requests_per_device_per_second',
+                     DEFAULT_REQUESTS_PER_DEVICE_PER_SECOND))
+        if new_value != self.requests_per_device_per_second:
+            self.requests_per_device_per_second = new_value
+            modified = True
+        new_value = non_negative_float(
+            conf.get('requests_per_device_rate_buffer',
+                     DEFAULT_REQUESTS_PER_DEVICE_RATE_BUFFER))
+        if new_value != self.requests_per_device_rate_buffer:
+            self.requests_per_device_rate_buffer = new_value
+            modified = True
+        if modified:
+            self._refresh_ratelimiters()
+        return modified
+
+    def _load_config_file(self):
+        # If conf file can be read then apply its options to the filter conf
+        # options, discarding *all* options previously loaded from the conf
+        # file i.e. options deleted from the conf file will revert to the
+        # filter conf value or default value. If the conf file cannot be read
+        # or is invalid, then the current config is left unchanged.
+        try:
+            new_conf = dict(self.filter_conf)  # filter_conf not current_conf
+            new_conf.update(
+                readconf(self.conf_path, BACKEND_RATELIMIT_CONFIG_SECTION))
+            modified = self._apply_config(new_conf)
+            if modified:
+                self.logger.info('Loaded config file %s, config changed',
+                                 self.conf_path)
+            elif not self.is_config_file_expected:
+                self.logger.info('Loaded new config file %s, config unchanged',
+                                 self.conf_path)
+            else:
+                self.logger.debug(
+                    'Loaded existing config file %s, config unchanged',
+                    self.conf_path)
+            self.is_config_file_expected = True
+        except IOError as err:
+            if self.is_config_file_expected:
+                self.logger.warning(
+                    'Failed to load config file, config unchanged: %s', err)
+            self.is_config_file_expected = False
+        except ValueError as err:
+            # ...but if it exists it should be valid
+            self.logger.warning('Invalid config file %s, config unchanged: %s',
+                                self.conf_path, err)
+
+    def _maybe_reload_config(self):
+        if self.config_reload_interval:
+            now = time.time()
+            if (now - self._last_config_reload_attempt
+                    >= self.config_reload_interval):
+                try:
+                    self._load_config_file()
+                except Exception:  # noqa
+                    self.logger.exception('Error reloading config file')
+                finally:
+                    # always reset last loaded time to avoid re-try storm
+                    self._last_config_reload_attempt = now
+
     def __call__(self, env, start_response):
         """
         WSGI entry point.
@@ -61,9 +157,11 @@ class BackendRateLimitMiddleware(object):
         :param env: WSGI environment dictionary
         :param start_response: WSGI callable
         """
+        self._maybe_reload_config()
         req = Request(env)
         handler = self.app
-        if req.method in RATE_LIMITED_METHODS:
+        if (self.requests_per_device_per_second
+                and req.method in RATE_LIMITED_METHODS):
             try:
                 device, partition, _ = split_and_validate_path(req, 1, 3, True)
                 int(partition)  # check it's a valid partition

swift/common/utils/__init__.py

@@ -2920,13 +2920,19 @@ class AbstractRateLimiter(object):
             running_time < (current time - rate_buffer ms) to allow an initial
             burst.
         """
-        self.max_rate = max_rate
+        self.set_max_rate(max_rate)
-        self.rate_buffer_ms = rate_buffer * self.clock_accuracy
+        self.set_rate_buffer(rate_buffer)
         self.burst_after_idle = burst_after_idle
         self.running_time = running_time
+
+    def set_max_rate(self, max_rate):
+        self.max_rate = max_rate
         self.time_per_incr = (self.clock_accuracy / self.max_rate
                               if self.max_rate else 0)
 
+    def set_rate_buffer(self, rate_buffer):
+        self.rate_buffer_ms = rate_buffer * self.clock_accuracy
+
     def _sleep(self, seconds):
         # subclasses should override to implement a sleep
         raise NotImplementedError

test/unit/common/middleware/test_backend_ratelimit.py

@@ -15,9 +15,12 @@
 
 # Used by get_swift_info and register_swift_info to store information about
 # the swift cluster.
+import os
+import shutil
 import time
 import unittest
 from collections import defaultdict
+from tempfile import mkdtemp
 
 import mock
 
@@ -42,6 +45,10 @@ class TestBackendRatelimitMiddleware(unittest.TestCase):
     def setUp(self):
         super(TestBackendRatelimitMiddleware, self).setUp()
         self.swift = FakeSwift()
+        self.tempdir = mkdtemp()
+
+    def tearDown(self):
+        shutil.rmtree(self.tempdir, ignore_errors=True)
 
     def test_init(self):
         conf = {}
@@ -73,6 +80,600 @@ class TestBackendRatelimitMiddleware(unittest.TestCase):
             'Value must be a non-negative float number, not "-1.0".',
             str(cm.exception))
 
+    def test_init_conf_path(self):
+        conf = {}
+        factory = backend_ratelimit.filter_factory(conf)
+        rl = factory(self.swift)
+        self.assertEqual('/etc/swift/backend-ratelimit.conf', rl.conf_path)
+        conf = {'backend_ratelimit_conf_path': '/etc/other/rl.conf'}
+        factory = backend_ratelimit.filter_factory(conf)
+        rl = factory(self.swift)
+        self.assertEqual('/etc/other/rl.conf', rl.conf_path)
+        conf = {'backend_ratelimit_conf_path': ''}
+        factory = backend_ratelimit.filter_factory(conf)
+        rl = factory(self.swift)
+        self.assertEqual('', rl.conf_path)
+
+    def test_init_conf_reload_interval(self):
+        conf = {}
+        factory = backend_ratelimit.filter_factory(conf)
+        rl = factory(self.swift)
+        self.assertEqual(60, rl.config_reload_interval)
+        conf = {'config_reload_interval': 600}
+        factory = backend_ratelimit.filter_factory(conf)
+        rl = factory(self.swift)
+        self.assertEqual(600, rl.config_reload_interval)
+        conf = {'config_reload_interval': 0}
+        factory = backend_ratelimit.filter_factory(conf)
+        rl = factory(self.swift)
+        self.assertEqual(0, rl.config_reload_interval)
+
+        def test_bad(value):
+            with self.assertRaises(ValueError) as cm:
+                conf = {'config_reload_interval': value}
+                factory = backend_ratelimit.filter_factory(conf)
+                factory(self.swift)
+            self.assertIn('Value must be a non-negative float number',
+                          str(cm.exception))
+        test_bad(-1)
+        test_bad('auto')
+
+    def test_init_config_file_set_and_missing(self):
+        # warn if missing conf file during init (conf_path set)
+        def do_test(conf_path):
+            conf = {'backend_ratelimit_conf_path': '%s' % conf_path,
+                    'requests_per_device_per_second': "1.3"}
+            factory = backend_ratelimit.filter_factory(conf)
+            with mock.patch(
+                    'swift.common.middleware.backend_ratelimit.get_logger',
+                    return_value=debug_logger()):
+                rl = factory(self.swift)
+            self.assertEqual(1.3, rl.requests_per_device_per_second)
+            self.assertEqual(1.0, rl.requests_per_device_rate_buffer)
+            self.assertEqual([], rl.logger.get_lines_for_level('error'))
+            self.assertEqual(
+                ['Failed to load config file, config unchanged: Unable to '
+                 'read config from %s' % conf_path],
+                rl.logger.get_lines_for_level('warning'))
+
+        do_test('')
+        do_test(os.path.join(self.tempdir, 'backend_rl.conf'))
+
+    def test_init_config_file_unset_and_missing(self):
+        # don't warn if missing conf file during init (conf_path not set)
+        conf = {'requests_per_device_per_second': "1.3"}
+        factory = backend_ratelimit.filter_factory(conf)
+        with mock.patch(
+                'swift.common.middleware.backend_ratelimit.get_logger',
+                return_value=debug_logger()):
+            rl = factory(self.swift)
+        self.assertEqual(1.3, rl.requests_per_device_per_second)
+        self.assertEqual(1.0, rl.requests_per_device_rate_buffer)
+        self.assertEqual([], rl.logger.get_lines_for_level('error'))
+        self.assertEqual([], rl.logger.get_lines_for_level('warning'))
+
+    def test_init_config_file_no_section(self):
+        # warn and ignore conf file without section
+        conf_path = os.path.join(self.tempdir, 'backend_rl.conf')
+        with open(conf_path, 'w') as fd:
+            fd.write('[DEFAULT]\n'
+                     'requests_per_device_per_second = 12.3\n')
+        conf = {'backend_ratelimit_conf_path': '%s' % conf_path,
+                'requests_per_device_per_second': 1.3}
+        factory = backend_ratelimit.filter_factory(conf)
+        with mock.patch('swift.common.middleware.backend_ratelimit.get_logger',
+                        return_value=debug_logger()):
+            rl = factory(self.swift)
+        self.assertEqual(1.3, rl.requests_per_device_per_second)
+        lines = rl.logger.get_lines_for_level('warning')
+        self.assertEqual(1, len(lines), lines)
+        self.assertIn('Invalid config file', lines[0])
+        self.assertEqual([], rl.logger.get_lines_for_level('error'))
+
+    def test_read_default_backend_ratelimit_conf(self):
+        conf = {'swift_dir': self.tempdir,
+                'requests_per_device_per_second': "1.3",
+                'requests_per_device_rate_buffer': "2.4",
+                # do not set 'backend_ratelimit_conf_path'
+                'config_reload_interval': 15}
+        # but set it up anyway
+        conf_path = os.path.join(self.tempdir, 'backend-ratelimit.conf')
+        with open(conf_path, 'w') as fd:
+            fd.write('[backend_ratelimit]\n'
+                     'requests_per_device_per_second = 12.3\n')
+        factory = backend_ratelimit.filter_factory(conf)
+        with mock.patch('swift.common.middleware.backend_ratelimit.get_logger',
+                        return_value=debug_logger()):
+            rl = factory(self.swift)
+        # backend-ratelimit.conf overrides options
+        self.assertEqual(12.3, rl.requests_per_device_per_second)
+        # but only the ones that are listed
+        self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
+        lines = rl.logger.get_lines_for_level('info')
+        self.assertEqual(['Loaded config file %s, config changed' % conf_path],
+                         lines)
+
+    def test_config_reload_does_not_override_reload_options(self):
+        conf_path = os.path.join(self.tempdir, 'override-ratelimit.conf')
+        conf = {'swift_dir': self.tempdir,
+                'requests_per_device_per_second': "1.3",
+                'requests_per_device_rate_buffer': "2.4",
+                'backend_ratelimit_conf_path': conf_path,
+                'config_reload_interval': 15}
+        with open(conf_path, 'w') as fd:
+            fd.write('[backend_ratelimit]\n'
+                     'requests_per_device_per_second = 12.3\n'
+                     'requests_per_device_rate_buffer = 12.4\n'
+                     'backend_ratelimit_conf_path = /etc/swift/ignored.conf\n'
+                     'config_reload_interval = 999999\n')
+        factory = backend_ratelimit.filter_factory(conf)
+        with mock.patch('swift.common.middleware.backend_ratelimit.get_logger',
+                        return_value=debug_logger()):
+            rl = factory(self.swift)
+        # we DO read rate limit options
+        self.assertEqual(12.3, rl.requests_per_device_per_second)
+        self.assertEqual(12.4, rl.requests_per_device_rate_buffer)
+        # but we do NOT read config reload options
+        self.assertEqual(conf_path, rl.conf_path)
+        self.assertEqual(15, rl.config_reload_interval)
+        lines = rl.logger.logger.get_lines_for_level('info')
+        self.assertEqual(['Loaded config file %s, config changed' % conf_path],
+                         lines)
+
+    def _do_test_config_file_reload(self, filter_conf, exp_reload_time):
+        # verify that conf file options are periodically reloaded
+        now = time.time()
+        # create the actual file
+        conf_path = os.path.join(self.tempdir, 'backend-ratelimit.conf')
+        with open(conf_path, 'w') as fd:
+            fd.write('[backend_ratelimit]\n'
+                     'requests_per_device_per_second = 12.3\n'
+                     'backend_ratelimit_conf_path = /etc/swift/rl.conf\n'
+                     'config_reload_interval = 999999\n')
+        factory = backend_ratelimit.filter_factory(filter_conf)
+        with mock.patch('swift.common.middleware.backend_ratelimit.time.time',
+                        return_value=now):
+            rl = factory(self.swift)
+        self.assertEqual(12.3, rl.requests_per_device_per_second)
+        self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
+        self.assertEqual(conf_path, rl.conf_path)
+
+        # modify the conf file
+        with open(conf_path, 'w') as fd:
+            fd.write('[backend_ratelimit]\n'
+                     'requests_per_device_per_second = 29.3\n'
+                     'requests_per_device_rate_buffer = 12.4\n'
+                     'backend_ratelimit_conf_path = /etc/swift/rl.conf\n'
+                     'config_reload_interval = 999999\n')
+
+        # send some requests, but too soon for config file to be reloaded
+        req1 = Request.blank('/sda1/99/a/c/o')
+        req2 = Request.blank('/sda2/99/a/c/o')
+        self.swift.register(req1.method, req1.path, HTTPOk, {})
+        self.swift.register(req2.method, req2.path, HTTPOk, {})
+        with mock.patch('swift.common.middleware.backend_ratelimit.time.time',
+                        return_value=now + exp_reload_time - 1):
+            resp1 = req1.get_response(rl)
+            resp2 = req2.get_response(rl)
+        self.assertEqual(200, resp1.status_int)
+        self.assertEqual(200, resp2.status_int)
+        self.assertEqual(12.3, rl.requests_per_device_per_second)
+        self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
+        self.assertEqual(conf_path, rl.conf_path)
+
+        # send some requests, time for config file to be reloaded
+        with mock.patch('swift.common.middleware.backend_ratelimit.time.time',
+                        return_value=now + exp_reload_time):
+            resp1 = req1.get_response(rl)
+            resp2 = req2.get_response(rl)
+        self.assertEqual(200, resp1.status_int)
+        self.assertEqual(200, resp2.status_int)
+        self.assertEqual(29.3, rl.requests_per_device_per_second)
+        self.assertEqual(12.4, rl.requests_per_device_rate_buffer)
+        self.assertEqual(conf_path, rl.conf_path)
+
+        # verify the per dev ratelimiters were updated
+        per_dev_rl_rates = [per_dev_rl.max_rate
+                            for per_dev_rl in list(rl.rate_limiters.values())]
+        self.assertEqual([29.3, 29.3], per_dev_rl_rates)
+        per_dev_rl_buffer = [per_dev_rl.rate_buffer_ms
+                             for per_dev_rl in list(rl.rate_limiters.values())]
+        self.assertEqual([12400, 12400], sorted(per_dev_rl_buffer))
+
+        # modify the config file again
+        # remove requests_per_device_per_second option
+        with open(conf_path, 'w') as fd:
+            fd.write('[backend_ratelimit]\n'
+                     'backend_ratelimit_conf_path = /etc/swift/rl.conf\n'
+                     'config_reload_interval = 999999\n')
+
+        # send some requests, not yet time for config file to be reloaded
+        with mock.patch('swift.common.middleware.backend_ratelimit.time.time',
+                        return_value=now + 2 * exp_reload_time - 1):
+            resp1 = req1.get_response(rl)
+            resp2 = req2.get_response(rl)
+        self.assertEqual(200, resp1.status_int)
+        self.assertEqual(200, resp2.status_int)
+        self.assertEqual(29.3, rl.requests_per_device_per_second)
+        self.assertEqual(12.4, rl.requests_per_device_rate_buffer)
+        self.assertEqual(conf_path, rl.conf_path)
+
+        # verify the per dev ratelimiters were not updated
+        per_dev_rl_rates = [per_dev_rl.max_rate
+                            for per_dev_rl in list(rl.rate_limiters.values())]
+        self.assertEqual([29.3, 29.3], sorted(per_dev_rl_rates))
+        per_dev_rl_buffer = [per_dev_rl.rate_buffer_ms
+                             for per_dev_rl in list(rl.rate_limiters.values())]
+        self.assertEqual([12400, 12400], sorted(per_dev_rl_buffer))
+
+        # send some requests, time for config file to be reloaded
+        with mock.patch('swift.common.middleware.backend_ratelimit.time.time',
+                        return_value=now + 2 * exp_reload_time):
+            resp1 = req1.get_response(rl)
+            resp2 = req2.get_response(rl)
+        self.assertEqual(200, resp1.status_int)
+        self.assertEqual(200, resp2.status_int)
+        # requests_per_device_per_second option reverts to filter conf
+        self.assertEqual(1.3, rl.requests_per_device_per_second)
+        self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
+        self.assertEqual(conf_path, rl.conf_path)
+
+        # verify the per dev ratelimiters were not updated
+        per_dev_rl_rates = [per_dev_rl.max_rate
+                            for per_dev_rl in list(rl.rate_limiters.values())]
+        self.assertEqual([1.3, 1.3], sorted(per_dev_rl_rates))
+        per_dev_rl_buffer = [per_dev_rl.rate_buffer_ms
+                             for per_dev_rl in list(rl.rate_limiters.values())]
+        self.assertEqual([2400, 2400], sorted(per_dev_rl_buffer))
+        return rl
+
+    def test_config_file_reload_default_interval(self):
+        filter_conf = {'swift_dir': self.tempdir,
+                       'requests_per_device_per_second': "1.3",
+                       'requests_per_device_rate_buffer': "2.4"}
+        rl = self._do_test_config_file_reload(filter_conf, 60)
+        self.assertEqual(60, rl.config_reload_interval)
+
+    def test_config_file_reload_custom_interval(self):
+        filter_conf = {'swift_dir': self.tempdir,
+                       'config_reload_interval': "30",
+                       'requests_per_device_per_second': "1.3",
+                       'requests_per_device_rate_buffer': "2.4"}
+        rl = self._do_test_config_file_reload(filter_conf, 30)
+        self.assertEqual(30, rl.config_reload_interval)
+
+    def test_config_file_reload_set_and_missing(self):
+        now = time.time()
+        conf_path = os.path.join(self.tempdir, 'missing')
+        filter_conf = {'swift_dir': self.tempdir,
+                       # path set so expect warning during init
+                       'backend_ratelimit_conf_path': conf_path,
+                       'requests_per_device_per_second': "1.3",
+                       'requests_per_device_rate_buffer': "2.4"}
+        factory = backend_ratelimit.filter_factory(filter_conf)
+
+        # expect warning during init
+        with mock.patch('swift.common.middleware.backend_ratelimit.time.time',
+                        return_value=now):
+            with mock.patch(
+                    'swift.common.middleware.backend_ratelimit.get_logger',
+                    return_value=debug_logger()):
+                rl = factory(self.swift)
+        # filter conf has been applied
+        self.assertEqual(1.3, rl.requests_per_device_per_second)
+        self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
+        self.assertEqual(
+            ['Failed to load config file, config unchanged: Unable to read '
+             'config from %s' % conf_path],
+            rl.logger.get_lines_for_level('warning'))
+        self.assertEqual([], rl.logger.get_lines_for_level('error'))
+
+        # jump into future, send request, config reload attempted
+        # no ongoing warning
+        rl.logger.logger.clear()
+        req1 = Request.blank('/sda1/99/a/c/o')
+        self.swift.register(req1.method, req1.path, HTTPOk, {})
+        with mock.patch('swift.common.middleware.backend_ratelimit.time.time',
+                        return_value=now + 10000):
+            resp1 = req1.get_response(rl)
+        self.assertEqual(200, resp1.status_int)
+        self.assertEqual(1.3, rl.requests_per_device_per_second)
+        self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
+        self.assertEqual([], rl.logger.get_lines_for_level('warning'))
+        self.assertEqual([], rl.logger.get_lines_for_level('error'))
+
+    def test_config_file_reload_unset_and_missing(self):
+        now = time.time()
+        filter_conf = {'swift_dir': self.tempdir,
+                       # conf path not set so expect no warnings
+                       'requests_per_device_per_second': "1.3",
+                       'requests_per_device_rate_buffer': "2.4"}
+        factory = backend_ratelimit.filter_factory(filter_conf)
+
+        # expect NO warning during init
+        with mock.patch('swift.common.middleware.backend_ratelimit.time.time',
+                        return_value=now):
+            with mock.patch(
+                    'swift.common.middleware.backend_ratelimit.get_logger',
+                    return_value=debug_logger()):
+                rl = factory(self.swift)
+        # filter conf has been applied
+        self.assertEqual(1.3, rl.requests_per_device_per_second)
+        self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
+        self.assertEqual([], rl.logger.get_lines_for_level('warning'))
+        self.assertEqual([], rl.logger.get_lines_for_level('error'))
+
+        # jump into future, send request, config reload attempted
+        # no ongoing warning
+        req1 = Request.blank('/sda1/99/a/c/o')
+        self.swift.register(req1.method, req1.path, HTTPOk, {})
+        with mock.patch('swift.common.middleware.backend_ratelimit.time.time',
+                        return_value=now + 10000):
+            resp1 = req1.get_response(rl)
+        self.assertEqual(200, resp1.status_int)
+        # previous conf file value has been retained
+        self.assertEqual(1.3, rl.requests_per_device_per_second)
+        self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
+        self.assertEqual([], rl.logger.get_lines_for_level('warning'))
+        self.assertEqual([], rl.logger.get_lines_for_level('error'))
+
+    def test_config_file_reload_empty_section(self):
+        # verify that empty section is OK
+        now = time.time()
+        filter_conf = {'swift_dir': self.tempdir,
+                       'requests_per_device_per_second': "1.3",
+                       'requests_per_device_rate_buffer': "2.4"}
+        # create the actual file
+        conf_path = os.path.join(self.tempdir, 'backend-ratelimit.conf')
+        with open(conf_path, 'w') as fd:
+            fd.write('[backend_ratelimit]\n')
+        factory = backend_ratelimit.filter_factory(filter_conf)
+        with mock.patch('swift.common.middleware.backend_ratelimit.time.time',
+                        return_value=now):
+            with mock.patch(
+                    'swift.common.middleware.backend_ratelimit.get_logger',
+                    return_value=debug_logger()):
+                rl = factory(self.swift)
+        # conf file value has been applied
+        self.assertEqual(1.3, rl.requests_per_device_per_second)
+        self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
+        self.assertEqual([], rl.logger.get_lines_for_level('warning'))
+        self.assertEqual([], rl.logger.get_lines_for_level('error'))
+
+    def test_config_file_reload_error(self):
+        # verify that current config is preserved if reload fails
+        now = time.time()
+        filter_conf = {'swift_dir': self.tempdir,
+                       'requests_per_device_per_second': "1.3",
+                       'requests_per_device_rate_buffer': "2.4"}
+        # create the actual file
+        conf_path = os.path.join(self.tempdir, 'backend-ratelimit.conf')
+        with open(conf_path, 'w') as fd:
+            fd.write('[backend_ratelimit]\n'
+                     'requests_per_device_per_second = 12.3\n')
+        factory = backend_ratelimit.filter_factory(filter_conf)
+        with mock.patch('swift.common.middleware.backend_ratelimit.time.time',
+                        return_value=now):
+            with mock.patch(
+                    'swift.common.middleware.backend_ratelimit.get_logger',
+                    return_value=debug_logger()):
+                rl = factory(self.swift)
+        # conf file value has been applied
+        self.assertEqual(12.3, rl.requests_per_device_per_second)
+        self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
+        self.assertEqual([], rl.logger.get_lines_for_level('warning'))
+        self.assertEqual([], rl.logger.get_lines_for_level('error'))
+
+        with open(conf_path, 'w') as fd:
+            fd.write('[backend_ratelimit]\n'
+                     'requests_per_device_per_second = 29.3\n')
+
+        # jump into future, send request, config reload attempted but fails
+        req1 = Request.blank('/sda1/99/a/c/o')
+        self.swift.register(req1.method, req1.path, HTTPOk, {})
+        with mock.patch('swift.common.middleware.backend_ratelimit.time.time',
+                        return_value=now + 10000):
+            with mock.patch(
+                    'swift.common.middleware.backend_ratelimit.readconf',
+                    side_effect=ValueError('BOOM')
+            ) as mock_readconf:
+                resp1 = req1.get_response(rl)
+        self.assertEqual(200, resp1.status_int)
+        # previous conf file value has been retained
+        self.assertEqual(12.3, rl.requests_per_device_per_second)
+        self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
+        mock_readconf.assert_called_once()
+        self.assertEqual(
+            ['Invalid config file %s, config unchanged: BOOM' % conf_path],
+            rl.logger.get_lines_for_level('warning'))
+        self.assertEqual([], rl.logger.get_lines_for_level('error'))
+
+        # the reload is not tried again immediately
+        rl.logger = debug_logger()
+        with mock.patch('swift.common.middleware.backend_ratelimit.time.time',
+                        return_value=now + 10059):
+            resp1 = req1.get_response(rl)
+        self.assertEqual(200, resp1.status_int)
+        # previous conf file value has been retained
+        self.assertEqual(12.3, rl.requests_per_device_per_second)
+        self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
+        self.assertEqual([], rl.logger.get_lines_for_level('warning'))
+        self.assertEqual([], rl.logger.get_lines_for_level('error'))
+
+        # ..but will be retried after reload interval
+        rl.logger = debug_logger()
+        with mock.patch('swift.common.middleware.backend_ratelimit.time.time',
+                        return_value=now + 10060):
+            resp1 = req1.get_response(rl)
+        self.assertEqual(200, resp1.status_int)
+        # updated conf file value is applied
+        self.assertEqual(29.3, rl.requests_per_device_per_second)
+        self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
+        self.assertEqual([], rl.logger.get_lines_for_level('warning'))
+        self.assertEqual([], rl.logger.get_lines_for_level('error'))
+
+    def test_config_file_reload_logging(self):
+        # verify that config reload is logged when config changes
+        now = time.time()
+        filter_conf = {'swift_dir': self.tempdir,
+                       'requests_per_device_per_second': "1.3",
+                       'requests_per_device_rate_buffer': "2.4"}
+        # create the actual file
+        conf_path = os.path.join(self.tempdir, 'backend-ratelimit.conf')
+        with open(conf_path, 'w') as fd:
+            fd.write('[backend_ratelimit]\n'
+                     'requests_per_device_per_second = 12.3\n')
+        factory = backend_ratelimit.filter_factory(filter_conf)
+        with mock.patch('swift.common.middleware.backend_ratelimit.time.time',
+                        return_value=now):
+            with mock.patch(
+                    'swift.common.middleware.backend_ratelimit.get_logger',
+                    return_value=debug_logger()):
+                rl = factory(self.swift)
+        # conf file value has been applied
+        self.assertEqual(12.3, rl.requests_per_device_per_second)
+        self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
+        lines = rl.logger.get_lines_for_level('info')
+        self.assertEqual(['Loaded config file %s, config changed' % conf_path],
+                         lines)
+
+        # jump into future, send request, config reload attempted, no change
+        rl.logger.logger.clear()
+        req1 = Request.blank('/sda1/99/a/c/o')
+        self.swift.register(req1.method, req1.path, HTTPOk, {})
+        with mock.patch('swift.common.middleware.backend_ratelimit.time.time',
+                        return_value=now + 10000):
+            resp1 = req1.get_response(rl)
+        self.assertEqual(200, resp1.status_int)
+        self.assertEqual(12.3, rl.requests_per_device_per_second)
+        self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
+        lines = rl.logger.get_lines_for_level('info')
+        self.assertEqual([], lines)
+
+        # modify config file, jump into future, change logged
+        with open(conf_path, 'w') as fd:
+            fd.write('[backend_ratelimit]\n'
+                     'requests_per_device_per_second = 23.4\n')
+        rl.logger = debug_logger()
+        with mock.patch('swift.common.middleware.backend_ratelimit.time.time',
+                        return_value=now + 10060):
+            resp1 = req1.get_response(rl)
+        self.assertEqual(200, resp1.status_int)
+        # previous conf file value has been retained
+        self.assertEqual(23.4, rl.requests_per_device_per_second)
+        self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
+        lines = rl.logger.get_lines_for_level('info')
+        self.assertEqual(['Loaded config file %s, config changed' % conf_path],
+                         lines)
+
+    def test_config_file_disappears_appears_logging(self):
+        # verify that config reload is logged when file reappears
+        now = time.time()
+        filter_conf = {'swift_dir': self.tempdir,
+                       'requests_per_device_per_second': "1.3",
+                       'requests_per_device_rate_buffer': "2.4"}
+        # create the actual file
+        conf_path = os.path.join(self.tempdir, 'backend-ratelimit.conf')
+        conf_str = ('[backend_ratelimit]\n'
+                    'requests_per_device_per_second = 12.3\n')
+        with open(conf_path, 'w') as fd:
+            fd.write(conf_str)
+        factory = backend_ratelimit.filter_factory(filter_conf)
+        with mock.patch('swift.common.middleware.backend_ratelimit.time.time',
+                        return_value=now):
+            with mock.patch(
+                    'swift.common.middleware.backend_ratelimit.get_logger',
+                    return_value=debug_logger()):
+                rl = factory(self.swift)
+        # conf file value has been applied
+        self.assertEqual(12.3, rl.requests_per_device_per_second)
+        self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
+        lines = rl.logger.get_lines_for_level('info')
+        self.assertEqual(
+            ['Loaded config file %s, config changed' % conf_path],
+            lines)
+        lines = rl.logger.get_lines_for_level('warning')
+        self.assertFalse(lines)
+
+        def do_request(now):
+            rl.logger.logger.clear()
+            req1 = Request.blank('/sda1/99/a/c/o')
+            self.swift.register(req1.method, req1.path, HTTPOk, {})
+            with mock.patch(
+                    'swift.common.middleware.backend_ratelimit.time.time',
+                    return_value=now):
+                resp = req1.get_response(rl)
+            self.assertEqual(200, resp.status_int)
+            info_lines = rl.logger.get_lines_for_level('info')
+            warning_lines = rl.logger.get_lines_for_level('warning')
+            return info_lines, warning_lines
+
+        # jump into future, send request, config reload fails - warning
+        os.unlink(conf_path)
+        now += 100
+        info_lines, warning_lines = do_request(now)
+        self.assertFalse(info_lines)
+        self.assertEqual(
+            ['Failed to load config file, config unchanged: Unable to '
+             'read config from %s' % conf_path], warning_lines)
+
+        # jump into future, send request, config reload fails - no warning
+        now += 100
+        info_lines, warning_lines = do_request(now)
+        self.assertFalse(info_lines)
+        self.assertFalse(warning_lines)
+
+        # reinstate conf file
+        with open(conf_path, 'w') as fd:
+            fd.write(conf_str)
+
+        # jump into future, send request, config reload succeeds - logged
+        now += 100
+        info_lines, warning_lines = do_request(now)
+        self.assertEqual('Loaded new config file %s, config unchanged'
+                         % conf_path, info_lines[0])
+        self.assertFalse(warning_lines)
+
+        # jump into future, send request, config reload succeeds - not logged
+        now += 100
+        info_lines, warning_lines = do_request(now)
+        self.assertFalse(info_lines)
+        self.assertFalse(warning_lines)
+
+    def test_config_file_reload_disabled(self):
+        # verify that conf file options are not periodically reloaded when
+        # interval is zero
+        now = time.time()
+        filter_conf = {'swift_dir': self.tempdir,
+                       'requests_per_device_per_second': "1.3",
+                       'requests_per_device_rate_buffer': "2.4",
+                       'config_reload_interval': 0}
+        conf_path = os.path.join(self.tempdir, 'backend-ratelimit.conf')
+        with open(conf_path, 'w') as fd:
+            fd.write('[backend_ratelimit]\n'
+                     'requests_per_device_per_second = 12.3\n')
+        factory = backend_ratelimit.filter_factory(filter_conf)
+        with mock.patch('swift.common.middleware.backend_ratelimit.time.time',
+                        return_value=now):
+            rl = factory(self.swift)
+        self.assertEqual(12.3, rl.requests_per_device_per_second)
+        self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
+
+        with open(conf_path, 'w') as fd:
+            fd.write('[backend_ratelimit]\n'
+                     'requests_per_device_per_second = 29.3\n')
+
+        req = Request.blank('/sda1/99/a/c/o')
+        self.swift.register(req.method, req.path, HTTPOk, {})
+        # jump way into the future...
+        with mock.patch('swift.common.middleware.backend_ratelimit.time.time',
+                        return_value=now + 100000):
+            resp = req.get_response(rl)
+        self.assertEqual(200, resp.status_int)
+        # no change
+        self.assertEqual(12.3, rl.requests_per_device_per_second)
+        self.assertEqual(2.4, rl.requests_per_device_rate_buffer)
+
     def _do_test_ratelimit(self, method, req_per_sec, rate_buffer):
         # send 20 requests, time increments by 0.01 between each request
         start = time.time()

test/unit/common/test_utils.py

@@ -5268,6 +5268,20 @@ class TestEventletRateLimiter(unittest.TestCase):
         self.assertEqual(1234567.8, rl.running_time)
         self.assertEqual(2000, rl.rate_buffer_ms)
 
+    def test_set_max_rate(self):
+        rl = utils.EventletRateLimiter(0.1)
+        self.assertEqual(0.1, rl.max_rate)
+        self.assertEqual(10000, rl.time_per_incr)
+        rl.set_max_rate(2)
+        self.assertEqual(2, rl.max_rate)
+        self.assertEqual(500, rl.time_per_incr)
+
+    def test_set_rate_buffer(self):
+        rl = utils.EventletRateLimiter(0.1)
+        self.assertEqual(5000.0, rl.rate_buffer_ms)
+        rl.set_rate_buffer(2.3)
+        self.assertEqual(2300, rl.rate_buffer_ms)
+
     def test_non_blocking(self):
         rate_limiter = utils.EventletRateLimiter(0.1, rate_buffer=0)
         with patch('time.time',) as mock_time:
@@ -5300,6 +5314,37 @@ class TestEventletRateLimiter(unittest.TestCase):
                 self.assertFalse(rate_limiter.is_allowed())
                 mock_sleep.assert_not_called()
 
+    def test_non_blocking_max_rate_adjusted(self):
+        rate_limiter = utils.EventletRateLimiter(0.1, rate_buffer=0)
+        with patch('time.time',) as mock_time:
+            with patch('eventlet.sleep') as mock_sleep:
+                mock_time.return_value = 0
+                self.assertTrue(rate_limiter.is_allowed())
+                self.assertFalse(rate_limiter.is_allowed())
+                mock_time.return_value = 9.99
+                self.assertFalse(rate_limiter.is_allowed())
+                mock_time.return_value = 10.0
+                self.assertTrue(rate_limiter.is_allowed())
+                self.assertFalse(rate_limiter.is_allowed())
+                # increase max_rate...but the new max_rate won't have impact
+                # until the running time is next incremented, i.e. when
+                # a call to is_allowed() next returns True
+                rate_limiter.set_max_rate(0.2)
+                self.assertFalse(rate_limiter.is_allowed())
+                mock_time.return_value = 19.99
+                self.assertFalse(rate_limiter.is_allowed())
+                mock_time.return_value = 20.0
+                self.assertTrue(rate_limiter.is_allowed())
+                # now we can go faster...
+                self.assertFalse(rate_limiter.is_allowed())
+                mock_time.return_value = 24.99
+                self.assertFalse(rate_limiter.is_allowed())
+                mock_time.return_value = 25.0
+                self.assertTrue(rate_limiter.is_allowed())
+                self.assertFalse(rate_limiter.is_allowed())
+
+        mock_sleep.assert_not_called()
+
     def _do_test(self, max_rate, running_time, start_time, rate_buffer,
                  burst_after_idle=False, incr_by=1.0):
         rate_limiter = utils.EventletRateLimiter(