diff --git a/etc/account-server.conf-sample b/etc/account-server.conf-sample index ade806819f..8df91428fd 100644 --- a/etc/account-server.conf-sample +++ b/etc/account-server.conf-sample @@ -130,15 +130,17 @@ use = egg:swift#recon [filter:backend_ratelimit] use = egg:swift#backend_ratelimit -# Set the maximum rate of requests per second per device per worker. Beyond -# this rate the server will return 529 responses and emit a 'backend.ratelimit' -# statsd metric without logging. The default value of zero causes no -# rate-limiting to be applied. -# requests_per_device_per_second = 0.0 -# -# Set the number of seconds of unused rate-limiting allowance that can -# accumulate and be used to allow a subsequent burst of requests. -# requests_per_device_rate_buffer = 1.0 +# Config options can optionally be loaded from a separate config file. Config +# options in this section will be used unless the same option is found in the +# config file, in which case the config file option will be used. See the +# backend-ratelimit.conf-sample file for details of available config options. +# backend_ratelimit_conf_path = /etc/swift/backend-ratelimit.conf + +# The minimum interval between attempts to reload any config file at +# backend_ratelimit_conf_path while the server is running. A value of 0 means +# that the file is loaded at start-up but not subsequently reloaded. Note that +# config options in this section are never reloaded after start-up. +# config_reload_interval = 60 [account-replicator] # You can override the default log routing for this app here (don't use set!): diff --git a/etc/backend-ratelimit.conf-sample b/etc/backend-ratelimit.conf-sample new file mode 100644 index 0000000000..4c38c20a2f --- /dev/null +++ b/etc/backend-ratelimit.conf-sample @@ -0,0 +1,10 @@ +[backend_ratelimit] +# Set the maximum rate of requests per second per device per worker. Beyond +# this rate the server will return 529 responses and emit a 'backend.ratelimit' +# statsd metric without logging. The default value of zero causes no +# rate-limiting to be applied. +# requests_per_device_per_second = 0.0 +# +# Set the number of seconds of unused rate-limiting allowance that can +# accumulate and be used to allow a subsequent burst of requests. +# requests_per_device_rate_buffer = 1.0 diff --git a/etc/container-server.conf-sample b/etc/container-server.conf-sample index 25cf917efa..78d52a17b9 100644 --- a/etc/container-server.conf-sample +++ b/etc/container-server.conf-sample @@ -140,15 +140,17 @@ use = egg:swift#recon [filter:backend_ratelimit] use = egg:swift#backend_ratelimit -# Set the maximum rate of requests per second per device per worker. Beyond -# this rate the server will return 529 responses and emit a 'backend.ratelimit' -# statsd metric without logging. The default value of zero causes no -# rate-limiting to be applied. -# requests_per_device_per_second = 0.0 -# -# Set the number of seconds of unused rate-limiting allowance that can -# accumulate and be used to allow a subsequent burst of requests. -# requests_per_device_rate_buffer = 1.0 +# Config options can optionally be loaded from a separate config file. Config +# options in this section will be used unless the same option is found in the +# config file, in which case the config file option will be used. See the +# backend-ratelimit.conf-sample file for details of available config options. +# backend_ratelimit_conf_path = /etc/swift/backend-ratelimit.conf + +# The minimum interval between attempts to reload any config file at +# backend_ratelimit_conf_path while the server is running. A value of 0 means +# that the file is loaded at start-up but not subsequently reloaded. Note that +# config options in this section are never reloaded after start-up. +# config_reload_interval = 60 [container-replicator] # You can override the default log routing for this app here (don't use set!): diff --git a/etc/object-server.conf-sample b/etc/object-server.conf-sample index ac7d497adb..c0a1287b98 100644 --- a/etc/object-server.conf-sample +++ b/etc/object-server.conf-sample @@ -244,15 +244,17 @@ use = egg:swift#recon [filter:backend_ratelimit] use = egg:swift#backend_ratelimit -# Set the maximum rate of requests per second per device per worker. Beyond -# this rate the server will return 529 responses and emit a 'backend.ratelimit' -# statsd metric without logging. The default value of zero causes no -# rate-limiting to be applied. -# requests_per_device_per_second = 0.0 -# -# Set the number of seconds of unused rate-limiting allowance that can -# accumulate and be used to allow a subsequent burst of requests. -# requests_per_device_rate_buffer = 1.0 +# Config options can optionally be loaded from a separate config file. Config +# options in this section will be used unless the same option is found in the +# config file, in which case the config file option will be used. See the +# backend-ratelimit.conf-sample file for details of available config options. +# backend_ratelimit_conf_path = /etc/swift/backend-ratelimit.conf + +# The minimum interval between attempts to reload any config file at +# backend_ratelimit_conf_path while the server is running. A value of 0 means +# that the file is loaded at start-up but not subsequently reloaded. Note that +# config options in this section are never reloaded after start-up. +# config_reload_interval = 60 [object-replicator] # You can override the default log routing for this app here (don't use set!): diff --git a/swift/common/middleware/backend_ratelimit.py b/swift/common/middleware/backend_ratelimit.py index b4922005f0..a9e6e8925f 100644 --- a/swift/common/middleware/backend_ratelimit.py +++ b/swift/common/middleware/backend_ratelimit.py @@ -12,7 +12,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. - +import os import time from collections import defaultdict @@ -20,10 +20,15 @@ from swift.common.request_helpers import split_and_validate_path from swift.common.swob import Request, HTTPTooManyBackendRequests, \ HTTPException from swift.common.utils import get_logger, non_negative_float, \ - EventletRateLimiter + EventletRateLimiter, readconf RATE_LIMITED_METHODS = ('GET', 'HEAD', 'PUT', 'POST', 'DELETE', 'UPDATE', 'REPLICATE') +BACKEND_RATELIMIT_CONFIG_SECTION = 'backend_ratelimit' +DEFAULT_BACKEND_RATELIMIT_CONF_FILE = 'backend-ratelimit.conf' +DEFAULT_CONFIG_RELOAD_INTERVAL = 60.0 +DEFAULT_REQUESTS_PER_DEVICE_PER_SECOND = 0.0 +DEFAULT_REQUESTS_PER_DEVICE_RATE_BUFFER = 1.0 class BackendRateLimitMiddleware(object): @@ -38,14 +43,16 @@ class BackendRateLimitMiddleware(object): If a request would cause the rate-limit to be exceeded then a response with a 529 status code is returned. """ - def __init__(self, app, conf, logger=None): + def __init__(self, app, filter_conf, logger=None): self.app = app - self.logger = logger or get_logger(conf, log_route='backend_ratelimit') - self.requests_per_device_per_second = non_negative_float( - conf.get('requests_per_device_per_second', 0.0)) - self.requests_per_device_rate_buffer = non_negative_float( - conf.get('requests_per_device_rate_buffer', 1.0)) - + self.filter_conf = filter_conf + self.current_conf = {} + self.logger = logger or get_logger(self.filter_conf, + log_route='backend_ratelimit') + self.requests_per_device_per_second = \ + DEFAULT_REQUESTS_PER_DEVICE_PER_SECOND + self.requests_per_device_rate_buffer = \ + DEFAULT_REQUESTS_PER_DEVICE_RATE_BUFFER # map device -> RateLimiter self.rate_limiters = defaultdict( lambda: EventletRateLimiter( @@ -54,6 +61,95 @@ class BackendRateLimitMiddleware(object): running_time=time.time(), burst_after_idle=True)) + # some config options are *only* read from filter conf at startup... + default_conf_path = os.path.join( + self.filter_conf.get('swift_dir', '/etc/swift'), + DEFAULT_BACKEND_RATELIMIT_CONF_FILE) + try: + self.conf_path = self.filter_conf['backend_ratelimit_conf_path'] + self.is_config_file_expected = True + except KeyError: + self.conf_path = default_conf_path + self.is_config_file_expected = False + self.config_reload_interval = non_negative_float( + filter_conf.get('config_reload_interval', + DEFAULT_CONFIG_RELOAD_INTERVAL)) + + # other conf options are read from filter section at startup but may + # also be overridden by options in a separate config file... + self._last_config_reload_attempt = time.time() + self._apply_config(self.filter_conf) + self._load_config_file() + + def _refresh_ratelimiters(self): + for dev, rl in self.rate_limiters.items(): + rl.set_max_rate(self.requests_per_device_per_second) + rl.set_rate_buffer(self.requests_per_device_rate_buffer) + + def _apply_config(self, conf): + self.current_conf = conf + modified = False + new_value = non_negative_float( + conf.get('requests_per_device_per_second', + DEFAULT_REQUESTS_PER_DEVICE_PER_SECOND)) + if new_value != self.requests_per_device_per_second: + self.requests_per_device_per_second = new_value + modified = True + new_value = non_negative_float( + conf.get('requests_per_device_rate_buffer', + DEFAULT_REQUESTS_PER_DEVICE_RATE_BUFFER)) + if new_value != self.requests_per_device_rate_buffer: + self.requests_per_device_rate_buffer = new_value + modified = True + if modified: + self._refresh_ratelimiters() + return modified + + def _load_config_file(self): + # If conf file can be read then apply its options to the filter conf + # options, discarding *all* options previously loaded from the conf + # file i.e. options deleted from the conf file will revert to the + # filter conf value or default value. If the conf file cannot be read + # or is invalid, then the current config is left unchanged. + try: + new_conf = dict(self.filter_conf) # filter_conf not current_conf + new_conf.update( + readconf(self.conf_path, BACKEND_RATELIMIT_CONFIG_SECTION)) + modified = self._apply_config(new_conf) + if modified: + self.logger.info('Loaded config file %s, config changed', + self.conf_path) + elif not self.is_config_file_expected: + self.logger.info('Loaded new config file %s, config unchanged', + self.conf_path) + else: + self.logger.debug( + 'Loaded existing config file %s, config unchanged', + self.conf_path) + self.is_config_file_expected = True + except IOError as err: + if self.is_config_file_expected: + self.logger.warning( + 'Failed to load config file, config unchanged: %s', err) + self.is_config_file_expected = False + except ValueError as err: + # ...but if it exists it should be valid + self.logger.warning('Invalid config file %s, config unchanged: %s', + self.conf_path, err) + + def _maybe_reload_config(self): + if self.config_reload_interval: + now = time.time() + if (now - self._last_config_reload_attempt + >= self.config_reload_interval): + try: + self._load_config_file() + except Exception: # noqa + self.logger.exception('Error reloading config file') + finally: + # always reset last loaded time to avoid re-try storm + self._last_config_reload_attempt = now + def __call__(self, env, start_response): """ WSGI entry point. @@ -61,9 +157,11 @@ class BackendRateLimitMiddleware(object): :param env: WSGI environment dictionary :param start_response: WSGI callable """ + self._maybe_reload_config() req = Request(env) handler = self.app - if req.method in RATE_LIMITED_METHODS: + if (self.requests_per_device_per_second + and req.method in RATE_LIMITED_METHODS): try: device, partition, _ = split_and_validate_path(req, 1, 3, True) int(partition) # check it's a valid partition diff --git a/swift/common/utils/__init__.py b/swift/common/utils/__init__.py index 52d235b5c7..8df82afcc1 100644 --- a/swift/common/utils/__init__.py +++ b/swift/common/utils/__init__.py @@ -2920,13 +2920,19 @@ class AbstractRateLimiter(object): running_time < (current time - rate_buffer ms) to allow an initial burst. """ - self.max_rate = max_rate - self.rate_buffer_ms = rate_buffer * self.clock_accuracy + self.set_max_rate(max_rate) + self.set_rate_buffer(rate_buffer) self.burst_after_idle = burst_after_idle self.running_time = running_time + + def set_max_rate(self, max_rate): + self.max_rate = max_rate self.time_per_incr = (self.clock_accuracy / self.max_rate if self.max_rate else 0) + def set_rate_buffer(self, rate_buffer): + self.rate_buffer_ms = rate_buffer * self.clock_accuracy + def _sleep(self, seconds): # subclasses should override to implement a sleep raise NotImplementedError diff --git a/test/unit/common/middleware/test_backend_ratelimit.py b/test/unit/common/middleware/test_backend_ratelimit.py index 353b040288..68b106c31c 100644 --- a/test/unit/common/middleware/test_backend_ratelimit.py +++ b/test/unit/common/middleware/test_backend_ratelimit.py @@ -15,9 +15,12 @@ # Used by get_swift_info and register_swift_info to store information about # the swift cluster. +import os +import shutil import time import unittest from collections import defaultdict +from tempfile import mkdtemp import mock @@ -42,6 +45,10 @@ class TestBackendRatelimitMiddleware(unittest.TestCase): def setUp(self): super(TestBackendRatelimitMiddleware, self).setUp() self.swift = FakeSwift() + self.tempdir = mkdtemp() + + def tearDown(self): + shutil.rmtree(self.tempdir, ignore_errors=True) def test_init(self): conf = {} @@ -73,6 +80,600 @@ class TestBackendRatelimitMiddleware(unittest.TestCase): 'Value must be a non-negative float number, not "-1.0".', str(cm.exception)) + def test_init_conf_path(self): + conf = {} + factory = backend_ratelimit.filter_factory(conf) + rl = factory(self.swift) + self.assertEqual('/etc/swift/backend-ratelimit.conf', rl.conf_path) + conf = {'backend_ratelimit_conf_path': '/etc/other/rl.conf'} + factory = backend_ratelimit.filter_factory(conf) + rl = factory(self.swift) + self.assertEqual('/etc/other/rl.conf', rl.conf_path) + conf = {'backend_ratelimit_conf_path': ''} + factory = backend_ratelimit.filter_factory(conf) + rl = factory(self.swift) + self.assertEqual('', rl.conf_path) + + def test_init_conf_reload_interval(self): + conf = {} + factory = backend_ratelimit.filter_factory(conf) + rl = factory(self.swift) + self.assertEqual(60, rl.config_reload_interval) + conf = {'config_reload_interval': 600} + factory = backend_ratelimit.filter_factory(conf) + rl = factory(self.swift) + self.assertEqual(600, rl.config_reload_interval) + conf = {'config_reload_interval': 0} + factory = backend_ratelimit.filter_factory(conf) + rl = factory(self.swift) + self.assertEqual(0, rl.config_reload_interval) + + def test_bad(value): + with self.assertRaises(ValueError) as cm: + conf = {'config_reload_interval': value} + factory = backend_ratelimit.filter_factory(conf) + factory(self.swift) + self.assertIn('Value must be a non-negative float number', + str(cm.exception)) + test_bad(-1) + test_bad('auto') + + def test_init_config_file_set_and_missing(self): + # warn if missing conf file during init (conf_path set) + def do_test(conf_path): + conf = {'backend_ratelimit_conf_path': '%s' % conf_path, + 'requests_per_device_per_second': "1.3"} + factory = backend_ratelimit.filter_factory(conf) + with mock.patch( + 'swift.common.middleware.backend_ratelimit.get_logger', + return_value=debug_logger()): + rl = factory(self.swift) + self.assertEqual(1.3, rl.requests_per_device_per_second) + self.assertEqual(1.0, rl.requests_per_device_rate_buffer) + self.assertEqual([], rl.logger.get_lines_for_level('error')) + self.assertEqual( + ['Failed to load config file, config unchanged: Unable to ' + 'read config from %s' % conf_path], + rl.logger.get_lines_for_level('warning')) + + do_test('') + do_test(os.path.join(self.tempdir, 'backend_rl.conf')) + + def test_init_config_file_unset_and_missing(self): + # don't warn if missing conf file during init (conf_path not set) + conf = {'requests_per_device_per_second': "1.3"} + factory = backend_ratelimit.filter_factory(conf) + with mock.patch( + 'swift.common.middleware.backend_ratelimit.get_logger', + return_value=debug_logger()): + rl = factory(self.swift) + self.assertEqual(1.3, rl.requests_per_device_per_second) + self.assertEqual(1.0, rl.requests_per_device_rate_buffer) + self.assertEqual([], rl.logger.get_lines_for_level('error')) + self.assertEqual([], rl.logger.get_lines_for_level('warning')) + + def test_init_config_file_no_section(self): + # warn and ignore conf file without section + conf_path = os.path.join(self.tempdir, 'backend_rl.conf') + with open(conf_path, 'w') as fd: + fd.write('[DEFAULT]\n' + 'requests_per_device_per_second = 12.3\n') + conf = {'backend_ratelimit_conf_path': '%s' % conf_path, + 'requests_per_device_per_second': 1.3} + factory = backend_ratelimit.filter_factory(conf) + with mock.patch('swift.common.middleware.backend_ratelimit.get_logger', + return_value=debug_logger()): + rl = factory(self.swift) + self.assertEqual(1.3, rl.requests_per_device_per_second) + lines = rl.logger.get_lines_for_level('warning') + self.assertEqual(1, len(lines), lines) + self.assertIn('Invalid config file', lines[0]) + self.assertEqual([], rl.logger.get_lines_for_level('error')) + + def test_read_default_backend_ratelimit_conf(self): + conf = {'swift_dir': self.tempdir, + 'requests_per_device_per_second': "1.3", + 'requests_per_device_rate_buffer': "2.4", + # do not set 'backend_ratelimit_conf_path' + 'config_reload_interval': 15} + # but set it up anyway + conf_path = os.path.join(self.tempdir, 'backend-ratelimit.conf') + with open(conf_path, 'w') as fd: + fd.write('[backend_ratelimit]\n' + 'requests_per_device_per_second = 12.3\n') + factory = backend_ratelimit.filter_factory(conf) + with mock.patch('swift.common.middleware.backend_ratelimit.get_logger', + return_value=debug_logger()): + rl = factory(self.swift) + # backend-ratelimit.conf overrides options + self.assertEqual(12.3, rl.requests_per_device_per_second) + # but only the ones that are listed + self.assertEqual(2.4, rl.requests_per_device_rate_buffer) + lines = rl.logger.get_lines_for_level('info') + self.assertEqual(['Loaded config file %s, config changed' % conf_path], + lines) + + def test_config_reload_does_not_override_reload_options(self): + conf_path = os.path.join(self.tempdir, 'override-ratelimit.conf') + conf = {'swift_dir': self.tempdir, + 'requests_per_device_per_second': "1.3", + 'requests_per_device_rate_buffer': "2.4", + 'backend_ratelimit_conf_path': conf_path, + 'config_reload_interval': 15} + with open(conf_path, 'w') as fd: + fd.write('[backend_ratelimit]\n' + 'requests_per_device_per_second = 12.3\n' + 'requests_per_device_rate_buffer = 12.4\n' + 'backend_ratelimit_conf_path = /etc/swift/ignored.conf\n' + 'config_reload_interval = 999999\n') + factory = backend_ratelimit.filter_factory(conf) + with mock.patch('swift.common.middleware.backend_ratelimit.get_logger', + return_value=debug_logger()): + rl = factory(self.swift) + # we DO read rate limit options + self.assertEqual(12.3, rl.requests_per_device_per_second) + self.assertEqual(12.4, rl.requests_per_device_rate_buffer) + # but we do NOT read config reload options + self.assertEqual(conf_path, rl.conf_path) + self.assertEqual(15, rl.config_reload_interval) + lines = rl.logger.logger.get_lines_for_level('info') + self.assertEqual(['Loaded config file %s, config changed' % conf_path], + lines) + + def _do_test_config_file_reload(self, filter_conf, exp_reload_time): + # verify that conf file options are periodically reloaded + now = time.time() + # create the actual file + conf_path = os.path.join(self.tempdir, 'backend-ratelimit.conf') + with open(conf_path, 'w') as fd: + fd.write('[backend_ratelimit]\n' + 'requests_per_device_per_second = 12.3\n' + 'backend_ratelimit_conf_path = /etc/swift/rl.conf\n' + 'config_reload_interval = 999999\n') + factory = backend_ratelimit.filter_factory(filter_conf) + with mock.patch('swift.common.middleware.backend_ratelimit.time.time', + return_value=now): + rl = factory(self.swift) + self.assertEqual(12.3, rl.requests_per_device_per_second) + self.assertEqual(2.4, rl.requests_per_device_rate_buffer) + self.assertEqual(conf_path, rl.conf_path) + + # modify the conf file + with open(conf_path, 'w') as fd: + fd.write('[backend_ratelimit]\n' + 'requests_per_device_per_second = 29.3\n' + 'requests_per_device_rate_buffer = 12.4\n' + 'backend_ratelimit_conf_path = /etc/swift/rl.conf\n' + 'config_reload_interval = 999999\n') + + # send some requests, but too soon for config file to be reloaded + req1 = Request.blank('/sda1/99/a/c/o') + req2 = Request.blank('/sda2/99/a/c/o') + self.swift.register(req1.method, req1.path, HTTPOk, {}) + self.swift.register(req2.method, req2.path, HTTPOk, {}) + with mock.patch('swift.common.middleware.backend_ratelimit.time.time', + return_value=now + exp_reload_time - 1): + resp1 = req1.get_response(rl) + resp2 = req2.get_response(rl) + self.assertEqual(200, resp1.status_int) + self.assertEqual(200, resp2.status_int) + self.assertEqual(12.3, rl.requests_per_device_per_second) + self.assertEqual(2.4, rl.requests_per_device_rate_buffer) + self.assertEqual(conf_path, rl.conf_path) + + # send some requests, time for config file to be reloaded + with mock.patch('swift.common.middleware.backend_ratelimit.time.time', + return_value=now + exp_reload_time): + resp1 = req1.get_response(rl) + resp2 = req2.get_response(rl) + self.assertEqual(200, resp1.status_int) + self.assertEqual(200, resp2.status_int) + self.assertEqual(29.3, rl.requests_per_device_per_second) + self.assertEqual(12.4, rl.requests_per_device_rate_buffer) + self.assertEqual(conf_path, rl.conf_path) + + # verify the per dev ratelimiters were updated + per_dev_rl_rates = [per_dev_rl.max_rate + for per_dev_rl in list(rl.rate_limiters.values())] + self.assertEqual([29.3, 29.3], per_dev_rl_rates) + per_dev_rl_buffer = [per_dev_rl.rate_buffer_ms + for per_dev_rl in list(rl.rate_limiters.values())] + self.assertEqual([12400, 12400], sorted(per_dev_rl_buffer)) + + # modify the config file again + # remove requests_per_device_per_second option + with open(conf_path, 'w') as fd: + fd.write('[backend_ratelimit]\n' + 'backend_ratelimit_conf_path = /etc/swift/rl.conf\n' + 'config_reload_interval = 999999\n') + + # send some requests, not yet time for config file to be reloaded + with mock.patch('swift.common.middleware.backend_ratelimit.time.time', + return_value=now + 2 * exp_reload_time - 1): + resp1 = req1.get_response(rl) + resp2 = req2.get_response(rl) + self.assertEqual(200, resp1.status_int) + self.assertEqual(200, resp2.status_int) + self.assertEqual(29.3, rl.requests_per_device_per_second) + self.assertEqual(12.4, rl.requests_per_device_rate_buffer) + self.assertEqual(conf_path, rl.conf_path) + + # verify the per dev ratelimiters were not updated + per_dev_rl_rates = [per_dev_rl.max_rate + for per_dev_rl in list(rl.rate_limiters.values())] + self.assertEqual([29.3, 29.3], sorted(per_dev_rl_rates)) + per_dev_rl_buffer = [per_dev_rl.rate_buffer_ms + for per_dev_rl in list(rl.rate_limiters.values())] + self.assertEqual([12400, 12400], sorted(per_dev_rl_buffer)) + + # send some requests, time for config file to be reloaded + with mock.patch('swift.common.middleware.backend_ratelimit.time.time', + return_value=now + 2 * exp_reload_time): + resp1 = req1.get_response(rl) + resp2 = req2.get_response(rl) + self.assertEqual(200, resp1.status_int) + self.assertEqual(200, resp2.status_int) + # requests_per_device_per_second option reverts to filter conf + self.assertEqual(1.3, rl.requests_per_device_per_second) + self.assertEqual(2.4, rl.requests_per_device_rate_buffer) + self.assertEqual(conf_path, rl.conf_path) + + # verify the per dev ratelimiters were not updated + per_dev_rl_rates = [per_dev_rl.max_rate + for per_dev_rl in list(rl.rate_limiters.values())] + self.assertEqual([1.3, 1.3], sorted(per_dev_rl_rates)) + per_dev_rl_buffer = [per_dev_rl.rate_buffer_ms + for per_dev_rl in list(rl.rate_limiters.values())] + self.assertEqual([2400, 2400], sorted(per_dev_rl_buffer)) + return rl + + def test_config_file_reload_default_interval(self): + filter_conf = {'swift_dir': self.tempdir, + 'requests_per_device_per_second': "1.3", + 'requests_per_device_rate_buffer': "2.4"} + rl = self._do_test_config_file_reload(filter_conf, 60) + self.assertEqual(60, rl.config_reload_interval) + + def test_config_file_reload_custom_interval(self): + filter_conf = {'swift_dir': self.tempdir, + 'config_reload_interval': "30", + 'requests_per_device_per_second': "1.3", + 'requests_per_device_rate_buffer': "2.4"} + rl = self._do_test_config_file_reload(filter_conf, 30) + self.assertEqual(30, rl.config_reload_interval) + + def test_config_file_reload_set_and_missing(self): + now = time.time() + conf_path = os.path.join(self.tempdir, 'missing') + filter_conf = {'swift_dir': self.tempdir, + # path set so expect warning during init + 'backend_ratelimit_conf_path': conf_path, + 'requests_per_device_per_second': "1.3", + 'requests_per_device_rate_buffer': "2.4"} + factory = backend_ratelimit.filter_factory(filter_conf) + + # expect warning during init + with mock.patch('swift.common.middleware.backend_ratelimit.time.time', + return_value=now): + with mock.patch( + 'swift.common.middleware.backend_ratelimit.get_logger', + return_value=debug_logger()): + rl = factory(self.swift) + # filter conf has been applied + self.assertEqual(1.3, rl.requests_per_device_per_second) + self.assertEqual(2.4, rl.requests_per_device_rate_buffer) + self.assertEqual( + ['Failed to load config file, config unchanged: Unable to read ' + 'config from %s' % conf_path], + rl.logger.get_lines_for_level('warning')) + self.assertEqual([], rl.logger.get_lines_for_level('error')) + + # jump into future, send request, config reload attempted + # no ongoing warning + rl.logger.logger.clear() + req1 = Request.blank('/sda1/99/a/c/o') + self.swift.register(req1.method, req1.path, HTTPOk, {}) + with mock.patch('swift.common.middleware.backend_ratelimit.time.time', + return_value=now + 10000): + resp1 = req1.get_response(rl) + self.assertEqual(200, resp1.status_int) + self.assertEqual(1.3, rl.requests_per_device_per_second) + self.assertEqual(2.4, rl.requests_per_device_rate_buffer) + self.assertEqual([], rl.logger.get_lines_for_level('warning')) + self.assertEqual([], rl.logger.get_lines_for_level('error')) + + def test_config_file_reload_unset_and_missing(self): + now = time.time() + filter_conf = {'swift_dir': self.tempdir, + # conf path not set so expect no warnings + 'requests_per_device_per_second': "1.3", + 'requests_per_device_rate_buffer': "2.4"} + factory = backend_ratelimit.filter_factory(filter_conf) + + # expect NO warning during init + with mock.patch('swift.common.middleware.backend_ratelimit.time.time', + return_value=now): + with mock.patch( + 'swift.common.middleware.backend_ratelimit.get_logger', + return_value=debug_logger()): + rl = factory(self.swift) + # filter conf has been applied + self.assertEqual(1.3, rl.requests_per_device_per_second) + self.assertEqual(2.4, rl.requests_per_device_rate_buffer) + self.assertEqual([], rl.logger.get_lines_for_level('warning')) + self.assertEqual([], rl.logger.get_lines_for_level('error')) + + # jump into future, send request, config reload attempted + # no ongoing warning + req1 = Request.blank('/sda1/99/a/c/o') + self.swift.register(req1.method, req1.path, HTTPOk, {}) + with mock.patch('swift.common.middleware.backend_ratelimit.time.time', + return_value=now + 10000): + resp1 = req1.get_response(rl) + self.assertEqual(200, resp1.status_int) + # previous conf file value has been retained + self.assertEqual(1.3, rl.requests_per_device_per_second) + self.assertEqual(2.4, rl.requests_per_device_rate_buffer) + self.assertEqual([], rl.logger.get_lines_for_level('warning')) + self.assertEqual([], rl.logger.get_lines_for_level('error')) + + def test_config_file_reload_empty_section(self): + # verify that empty section is OK + now = time.time() + filter_conf = {'swift_dir': self.tempdir, + 'requests_per_device_per_second': "1.3", + 'requests_per_device_rate_buffer': "2.4"} + # create the actual file + conf_path = os.path.join(self.tempdir, 'backend-ratelimit.conf') + with open(conf_path, 'w') as fd: + fd.write('[backend_ratelimit]\n') + factory = backend_ratelimit.filter_factory(filter_conf) + with mock.patch('swift.common.middleware.backend_ratelimit.time.time', + return_value=now): + with mock.patch( + 'swift.common.middleware.backend_ratelimit.get_logger', + return_value=debug_logger()): + rl = factory(self.swift) + # conf file value has been applied + self.assertEqual(1.3, rl.requests_per_device_per_second) + self.assertEqual(2.4, rl.requests_per_device_rate_buffer) + self.assertEqual([], rl.logger.get_lines_for_level('warning')) + self.assertEqual([], rl.logger.get_lines_for_level('error')) + + def test_config_file_reload_error(self): + # verify that current config is preserved if reload fails + now = time.time() + filter_conf = {'swift_dir': self.tempdir, + 'requests_per_device_per_second': "1.3", + 'requests_per_device_rate_buffer': "2.4"} + # create the actual file + conf_path = os.path.join(self.tempdir, 'backend-ratelimit.conf') + with open(conf_path, 'w') as fd: + fd.write('[backend_ratelimit]\n' + 'requests_per_device_per_second = 12.3\n') + factory = backend_ratelimit.filter_factory(filter_conf) + with mock.patch('swift.common.middleware.backend_ratelimit.time.time', + return_value=now): + with mock.patch( + 'swift.common.middleware.backend_ratelimit.get_logger', + return_value=debug_logger()): + rl = factory(self.swift) + # conf file value has been applied + self.assertEqual(12.3, rl.requests_per_device_per_second) + self.assertEqual(2.4, rl.requests_per_device_rate_buffer) + self.assertEqual([], rl.logger.get_lines_for_level('warning')) + self.assertEqual([], rl.logger.get_lines_for_level('error')) + + with open(conf_path, 'w') as fd: + fd.write('[backend_ratelimit]\n' + 'requests_per_device_per_second = 29.3\n') + + # jump into future, send request, config reload attempted but fails + req1 = Request.blank('/sda1/99/a/c/o') + self.swift.register(req1.method, req1.path, HTTPOk, {}) + with mock.patch('swift.common.middleware.backend_ratelimit.time.time', + return_value=now + 10000): + with mock.patch( + 'swift.common.middleware.backend_ratelimit.readconf', + side_effect=ValueError('BOOM') + ) as mock_readconf: + resp1 = req1.get_response(rl) + self.assertEqual(200, resp1.status_int) + # previous conf file value has been retained + self.assertEqual(12.3, rl.requests_per_device_per_second) + self.assertEqual(2.4, rl.requests_per_device_rate_buffer) + mock_readconf.assert_called_once() + self.assertEqual( + ['Invalid config file %s, config unchanged: BOOM' % conf_path], + rl.logger.get_lines_for_level('warning')) + self.assertEqual([], rl.logger.get_lines_for_level('error')) + + # the reload is not tried again immediately + rl.logger = debug_logger() + with mock.patch('swift.common.middleware.backend_ratelimit.time.time', + return_value=now + 10059): + resp1 = req1.get_response(rl) + self.assertEqual(200, resp1.status_int) + # previous conf file value has been retained + self.assertEqual(12.3, rl.requests_per_device_per_second) + self.assertEqual(2.4, rl.requests_per_device_rate_buffer) + self.assertEqual([], rl.logger.get_lines_for_level('warning')) + self.assertEqual([], rl.logger.get_lines_for_level('error')) + + # ..but will be retried after reload interval + rl.logger = debug_logger() + with mock.patch('swift.common.middleware.backend_ratelimit.time.time', + return_value=now + 10060): + resp1 = req1.get_response(rl) + self.assertEqual(200, resp1.status_int) + # updated conf file value is applied + self.assertEqual(29.3, rl.requests_per_device_per_second) + self.assertEqual(2.4, rl.requests_per_device_rate_buffer) + self.assertEqual([], rl.logger.get_lines_for_level('warning')) + self.assertEqual([], rl.logger.get_lines_for_level('error')) + + def test_config_file_reload_logging(self): + # verify that config reload is logged when config changes + now = time.time() + filter_conf = {'swift_dir': self.tempdir, + 'requests_per_device_per_second': "1.3", + 'requests_per_device_rate_buffer': "2.4"} + # create the actual file + conf_path = os.path.join(self.tempdir, 'backend-ratelimit.conf') + with open(conf_path, 'w') as fd: + fd.write('[backend_ratelimit]\n' + 'requests_per_device_per_second = 12.3\n') + factory = backend_ratelimit.filter_factory(filter_conf) + with mock.patch('swift.common.middleware.backend_ratelimit.time.time', + return_value=now): + with mock.patch( + 'swift.common.middleware.backend_ratelimit.get_logger', + return_value=debug_logger()): + rl = factory(self.swift) + # conf file value has been applied + self.assertEqual(12.3, rl.requests_per_device_per_second) + self.assertEqual(2.4, rl.requests_per_device_rate_buffer) + lines = rl.logger.get_lines_for_level('info') + self.assertEqual(['Loaded config file %s, config changed' % conf_path], + lines) + + # jump into future, send request, config reload attempted, no change + rl.logger.logger.clear() + req1 = Request.blank('/sda1/99/a/c/o') + self.swift.register(req1.method, req1.path, HTTPOk, {}) + with mock.patch('swift.common.middleware.backend_ratelimit.time.time', + return_value=now + 10000): + resp1 = req1.get_response(rl) + self.assertEqual(200, resp1.status_int) + self.assertEqual(12.3, rl.requests_per_device_per_second) + self.assertEqual(2.4, rl.requests_per_device_rate_buffer) + lines = rl.logger.get_lines_for_level('info') + self.assertEqual([], lines) + + # modify config file, jump into future, change logged + with open(conf_path, 'w') as fd: + fd.write('[backend_ratelimit]\n' + 'requests_per_device_per_second = 23.4\n') + rl.logger = debug_logger() + with mock.patch('swift.common.middleware.backend_ratelimit.time.time', + return_value=now + 10060): + resp1 = req1.get_response(rl) + self.assertEqual(200, resp1.status_int) + # previous conf file value has been retained + self.assertEqual(23.4, rl.requests_per_device_per_second) + self.assertEqual(2.4, rl.requests_per_device_rate_buffer) + lines = rl.logger.get_lines_for_level('info') + self.assertEqual(['Loaded config file %s, config changed' % conf_path], + lines) + + def test_config_file_disappears_appears_logging(self): + # verify that config reload is logged when file reappears + now = time.time() + filter_conf = {'swift_dir': self.tempdir, + 'requests_per_device_per_second': "1.3", + 'requests_per_device_rate_buffer': "2.4"} + # create the actual file + conf_path = os.path.join(self.tempdir, 'backend-ratelimit.conf') + conf_str = ('[backend_ratelimit]\n' + 'requests_per_device_per_second = 12.3\n') + with open(conf_path, 'w') as fd: + fd.write(conf_str) + factory = backend_ratelimit.filter_factory(filter_conf) + with mock.patch('swift.common.middleware.backend_ratelimit.time.time', + return_value=now): + with mock.patch( + 'swift.common.middleware.backend_ratelimit.get_logger', + return_value=debug_logger()): + rl = factory(self.swift) + # conf file value has been applied + self.assertEqual(12.3, rl.requests_per_device_per_second) + self.assertEqual(2.4, rl.requests_per_device_rate_buffer) + lines = rl.logger.get_lines_for_level('info') + self.assertEqual( + ['Loaded config file %s, config changed' % conf_path], + lines) + lines = rl.logger.get_lines_for_level('warning') + self.assertFalse(lines) + + def do_request(now): + rl.logger.logger.clear() + req1 = Request.blank('/sda1/99/a/c/o') + self.swift.register(req1.method, req1.path, HTTPOk, {}) + with mock.patch( + 'swift.common.middleware.backend_ratelimit.time.time', + return_value=now): + resp = req1.get_response(rl) + self.assertEqual(200, resp.status_int) + info_lines = rl.logger.get_lines_for_level('info') + warning_lines = rl.logger.get_lines_for_level('warning') + return info_lines, warning_lines + + # jump into future, send request, config reload fails - warning + os.unlink(conf_path) + now += 100 + info_lines, warning_lines = do_request(now) + self.assertFalse(info_lines) + self.assertEqual( + ['Failed to load config file, config unchanged: Unable to ' + 'read config from %s' % conf_path], warning_lines) + + # jump into future, send request, config reload fails - no warning + now += 100 + info_lines, warning_lines = do_request(now) + self.assertFalse(info_lines) + self.assertFalse(warning_lines) + + # reinstate conf file + with open(conf_path, 'w') as fd: + fd.write(conf_str) + + # jump into future, send request, config reload succeeds - logged + now += 100 + info_lines, warning_lines = do_request(now) + self.assertEqual('Loaded new config file %s, config unchanged' + % conf_path, info_lines[0]) + self.assertFalse(warning_lines) + + # jump into future, send request, config reload succeeds - not logged + now += 100 + info_lines, warning_lines = do_request(now) + self.assertFalse(info_lines) + self.assertFalse(warning_lines) + + def test_config_file_reload_disabled(self): + # verify that conf file options are not periodically reloaded when + # interval is zero + now = time.time() + filter_conf = {'swift_dir': self.tempdir, + 'requests_per_device_per_second': "1.3", + 'requests_per_device_rate_buffer': "2.4", + 'config_reload_interval': 0} + conf_path = os.path.join(self.tempdir, 'backend-ratelimit.conf') + with open(conf_path, 'w') as fd: + fd.write('[backend_ratelimit]\n' + 'requests_per_device_per_second = 12.3\n') + factory = backend_ratelimit.filter_factory(filter_conf) + with mock.patch('swift.common.middleware.backend_ratelimit.time.time', + return_value=now): + rl = factory(self.swift) + self.assertEqual(12.3, rl.requests_per_device_per_second) + self.assertEqual(2.4, rl.requests_per_device_rate_buffer) + + with open(conf_path, 'w') as fd: + fd.write('[backend_ratelimit]\n' + 'requests_per_device_per_second = 29.3\n') + + req = Request.blank('/sda1/99/a/c/o') + self.swift.register(req.method, req.path, HTTPOk, {}) + # jump way into the future... + with mock.patch('swift.common.middleware.backend_ratelimit.time.time', + return_value=now + 100000): + resp = req.get_response(rl) + self.assertEqual(200, resp.status_int) + # no change + self.assertEqual(12.3, rl.requests_per_device_per_second) + self.assertEqual(2.4, rl.requests_per_device_rate_buffer) + def _do_test_ratelimit(self, method, req_per_sec, rate_buffer): # send 20 requests, time increments by 0.01 between each request start = time.time() diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 0414d92cf1..57fe6a162e 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -5268,6 +5268,20 @@ class TestEventletRateLimiter(unittest.TestCase): self.assertEqual(1234567.8, rl.running_time) self.assertEqual(2000, rl.rate_buffer_ms) + def test_set_max_rate(self): + rl = utils.EventletRateLimiter(0.1) + self.assertEqual(0.1, rl.max_rate) + self.assertEqual(10000, rl.time_per_incr) + rl.set_max_rate(2) + self.assertEqual(2, rl.max_rate) + self.assertEqual(500, rl.time_per_incr) + + def test_set_rate_buffer(self): + rl = utils.EventletRateLimiter(0.1) + self.assertEqual(5000.0, rl.rate_buffer_ms) + rl.set_rate_buffer(2.3) + self.assertEqual(2300, rl.rate_buffer_ms) + def test_non_blocking(self): rate_limiter = utils.EventletRateLimiter(0.1, rate_buffer=0) with patch('time.time',) as mock_time: @@ -5300,6 +5314,37 @@ class TestEventletRateLimiter(unittest.TestCase): self.assertFalse(rate_limiter.is_allowed()) mock_sleep.assert_not_called() + def test_non_blocking_max_rate_adjusted(self): + rate_limiter = utils.EventletRateLimiter(0.1, rate_buffer=0) + with patch('time.time',) as mock_time: + with patch('eventlet.sleep') as mock_sleep: + mock_time.return_value = 0 + self.assertTrue(rate_limiter.is_allowed()) + self.assertFalse(rate_limiter.is_allowed()) + mock_time.return_value = 9.99 + self.assertFalse(rate_limiter.is_allowed()) + mock_time.return_value = 10.0 + self.assertTrue(rate_limiter.is_allowed()) + self.assertFalse(rate_limiter.is_allowed()) + # increase max_rate...but the new max_rate won't have impact + # until the running time is next incremented, i.e. when + # a call to is_allowed() next returns True + rate_limiter.set_max_rate(0.2) + self.assertFalse(rate_limiter.is_allowed()) + mock_time.return_value = 19.99 + self.assertFalse(rate_limiter.is_allowed()) + mock_time.return_value = 20.0 + self.assertTrue(rate_limiter.is_allowed()) + # now we can go faster... + self.assertFalse(rate_limiter.is_allowed()) + mock_time.return_value = 24.99 + self.assertFalse(rate_limiter.is_allowed()) + mock_time.return_value = 25.0 + self.assertTrue(rate_limiter.is_allowed()) + self.assertFalse(rate_limiter.is_allowed()) + + mock_sleep.assert_not_called() + def _do_test(self, max_rate, running_time, start_time, rate_buffer, burst_after_idle=False, incr_by=1.0): rate_limiter = utils.EventletRateLimiter(