Merge "Added container listing ratelimiting"

This commit is contained in:
Jenkins 2013-08-23 15:46:53 +00:00 committed by Gerrit Code Review
commit 621ea520a5
4 changed files with 217 additions and 103 deletions

View File

@ -15,38 +15,49 @@ Configuration
All configuration is optional. If no account or container limits are provided All configuration is optional. If no account or container limits are provided
there will be no rate limiting. Configuration available: there will be no rate limiting. Configuration available:
======================== ========= =========================================== ================================ ======= ======================================
Option Default Description Option Default Description
------------------------ --------- ------------------------------------------- -------------------------------- ------- --------------------------------------
clock_accuracy 1000 Represents how accurate the proxy servers' clock_accuracy 1000 Represents how accurate the proxy
system clocks are with each other. 1000 servers' system clocks are with each
means that all the proxies' clock are other. 1000 means that all the
accurate to each other within 1 proxies' clock are accurate to each
millisecond. No ratelimit should be other within 1 millisecond. No
higher than the clock accuracy. ratelimit should be higher than the
max_sleep_time_seconds 60 App will immediately return a 498 response clock accuracy.
if the necessary sleep time ever exceeds max_sleep_time_seconds 60 App will immediately return a 498
the given max_sleep_time_seconds. response if the necessary sleep time
log_sleep_time_seconds 0 To allow visibility into rate limiting set ever exceeds the given
this value > 0 and all sleeps greater than max_sleep_time_seconds.
the number will be logged. log_sleep_time_seconds 0 To allow visibility into rate limiting
rate_buffer_seconds 5 Number of seconds the rate counter can set this value > 0 and all sleeps
drop and be allowed to catch up (at a greater than the number will be
faster than listed rate). A larger number logged.
will result in larger spikes in rate but rate_buffer_seconds 5 Number of seconds the rate counter can
better average accuracy. drop and be allowed to catch up (at a
account_ratelimit 0 If set, will limit PUT and DELETE requests faster than listed rate). A larger
to /account_name/container_name. number will result in larger spikes in
Number is in requests per second. rate but better average accuracy.
account_whitelist '' Comma separated lists of account names that account_ratelimit 0 If set, will limit PUT and DELETE
will not be rate limited. requests to
account_blacklist '' Comma separated lists of account names that /account_name/container_name. Number
will not be allowed. Returns a 497 response. is in requests per second.
container_ratelimit_size '' When set with container_limit_x = r: account_whitelist '' Comma separated lists of account names
for containers of size x, limit requests that will not be rate limited.
per second to r. Will limit PUT, DELETE, account_blacklist '' Comma separated lists of account names
and POST requests to /a/c/o. that will not be allowed. Returns a
======================== ========= =========================================== 497 response.
container_ratelimit_size '' When set with container_ratelimit_x =
r: for containers of size x, limit
requests per second to r. Will limit
PUT, DELETE, and POST requests to
/a/c/o.
container_listing_ratelimit_size '' When set with
container_listing_ratelimit_x = r: for
containers of size x, limit listing
requests per second to r. Will limit
GET requests to /a/c.
================================ ======= ======================================
The container rate limits are linearly interpolated from the values given. A The container rate limits are linearly interpolated from the values given. A
sample container rate limiting could be: sample container rate limiting could be:

View File

@ -328,13 +328,19 @@ use = egg:swift#ratelimit
# account_blacklist = c,d # account_blacklist = c,d
# with container_limit_x = r # with container_limit_x = r
# for containers of size x limit requests per second to r. The container # for containers of size x limit write requests per second to r. The container
# rate will be linearly interpolated from the values given. With the values # rate will be linearly interpolated from the values given. With the values
# below, a container of size 5 will get a rate of 75. # below, a container of size 5 will get a rate of 75.
# container_ratelimit_0 = 100 # container_ratelimit_0 = 100
# container_ratelimit_10 = 50 # container_ratelimit_10 = 50
# container_ratelimit_50 = 20 # container_ratelimit_50 = 20
# Similarly to the above container-level write limits, the following will limit
# container GET (listing) requests.
# container_listing_ratelimit_0 = 100
# container_listing_ratelimit_10 = 50
# container_listing_ratelimit_50 = 20
[filter:domain_remap] [filter:domain_remap]
use = egg:swift#domain_remap use = egg:swift#domain_remap
# You can override the default log routing for this filter here: # You can override the default log routing for this filter here:

View File

@ -23,6 +23,51 @@ from swift.common.memcached import MemcacheConnectionError
from swift.common.swob import Request, Response from swift.common.swob import Request, Response
def interpret_conf_limits(conf, name_prefix):
conf_limits = []
for conf_key in conf.keys():
if conf_key.startswith(name_prefix):
cont_size = int(conf_key[len(name_prefix):])
rate = float(conf[conf_key])
conf_limits.append((cont_size, rate))
conf_limits.sort()
ratelimits = []
while conf_limits:
cur_size, cur_rate = conf_limits.pop(0)
if conf_limits:
next_size, next_rate = conf_limits[0]
slope = (float(next_rate) - float(cur_rate)) \
/ (next_size - cur_size)
def new_scope(cur_size, slope, cur_rate):
# making new scope for variables
return lambda x: (x - cur_size) * slope + cur_rate
line_func = new_scope(cur_size, slope, cur_rate)
else:
line_func = lambda x: cur_rate
ratelimits.append((cur_size, cur_rate, line_func))
return ratelimits
def get_maxrate(ratelimits, size):
"""
Returns number of requests allowed per second for given size.
"""
last_func = None
if size:
size = int(size)
for ratesize, rate, func in ratelimits:
if size < ratesize:
break
last_func = func
if last_func:
return last_func(size)
return None
class MaxSleepTimeHitError(Exception): class MaxSleepTimeHitError(Exception):
pass pass
@ -57,45 +102,20 @@ class RateLimitMiddleware(object):
[acc.strip() for acc in [acc.strip() for acc in
conf.get('account_blacklist', '').split(',') if acc.strip()] conf.get('account_blacklist', '').split(',') if acc.strip()]
self.memcache_client = None self.memcache_client = None
conf_limits = [] self.container_ratelimits = interpret_conf_limits(
for conf_key in conf.keys(): conf, 'container_ratelimit_')
if conf_key.startswith('container_ratelimit_'): self.container_listing_ratelimits = interpret_conf_limits(
cont_size = int(conf_key[len('container_ratelimit_'):]) conf, 'container_listing_ratelimit_')
rate = float(conf[conf_key])
conf_limits.append((cont_size, rate))
conf_limits.sort() def get_container_size(self, account_name, container_name):
self.container_ratelimits = [] rv = 0
while conf_limits: memcache_key = get_container_memcache_key(account_name,
cur_size, cur_rate = conf_limits.pop(0) container_name)
if conf_limits: container_info = self.memcache_client.get(memcache_key)
next_size, next_rate = conf_limits[0] if isinstance(container_info, dict):
slope = (float(next_rate) - float(cur_rate)) \ rv = container_info.get(
/ (next_size - cur_size) 'object_count', container_info.get('container_size', 0))
return rv
def new_scope(cur_size, slope, cur_rate):
# making new scope for variables
return lambda x: (x - cur_size) * slope + cur_rate
line_func = new_scope(cur_size, slope, cur_rate)
else:
line_func = lambda x: cur_rate
self.container_ratelimits.append((cur_size, cur_rate, line_func))
def get_container_maxrate(self, container_size):
"""
Returns number of requests allowed per second for given container size.
"""
last_func = None
if container_size:
container_size = int(container_size)
for size, rate, func in self.container_ratelimits:
if container_size < size:
break
last_func = func
if last_func:
return last_func(container_size)
return None
def get_ratelimitable_key_tuples(self, req_method, account_name, def get_ratelimitable_key_tuples(self, req_method, account_name,
container_name=None, obj_name=None): container_name=None, obj_name=None):
@ -118,18 +138,26 @@ class RateLimitMiddleware(object):
if account_name and container_name and obj_name and \ if account_name and container_name and obj_name and \
req_method in ('PUT', 'DELETE', 'POST'): req_method in ('PUT', 'DELETE', 'POST'):
container_size = None container_size = self.get_container_size(
memcache_key = get_container_memcache_key(account_name, account_name, container_name)
container_name) container_rate = get_maxrate(
container_info = self.memcache_client.get(memcache_key) self.container_ratelimits, container_size)
if isinstance(container_info, dict): if container_rate:
container_size = container_info.get( keys.append((
'object_count', container_info.get('container_size', 0)) "ratelimit/%s/%s" % (account_name, container_name),
container_rate = self.get_container_maxrate(container_size) container_rate))
if container_rate:
keys.append(("ratelimit/%s/%s" % (account_name, if account_name and container_name and not obj_name and \
container_name), req_method == 'GET':
container_rate)) container_size = self.get_container_size(
account_name, container_name)
container_rate = get_maxrate(
self.container_listing_ratelimits, container_size)
if container_rate:
keys.append((
"ratelimit_listing/%s/%s" % (account_name, container_name),
container_rate))
return keys return keys
def _get_sleep_time(self, key, max_rate): def _get_sleep_time(self, key, max_rate):

View File

@ -161,23 +161,28 @@ class TestRateLimit(unittest.TestCase):
for x in range(0, num): for x in range(0, num):
callable_func() callable_func()
end = time.time() end = time.time()
total_time = float(num) / rate - 1.0 / rate # 1st request isn't limited total_time = float(num) / rate - 1.0 / rate # 1st request not limited
# Allow for one second of variation in the total time. # Allow for one second of variation in the total time.
time_diff = abs(total_time - (end - begin)) time_diff = abs(total_time - (end - begin))
if check_time: if check_time:
self.assertEquals(round(total_time, 1), round(time_ticker, 1)) self.assertEquals(round(total_time, 1), round(time_ticker, 1))
return time_diff return time_diff
def test_get_container_maxrate(self): def test_get_maxrate(self):
conf_dict = {'container_ratelimit_10': 200, conf_dict = {'container_ratelimit_10': 200,
'container_ratelimit_50': 100, 'container_ratelimit_50': 100,
'container_ratelimit_75': 30} 'container_ratelimit_75': 30}
test_ratelimit = dummy_filter_factory(conf_dict)(FakeApp()) test_ratelimit = dummy_filter_factory(conf_dict)(FakeApp())
self.assertEquals(test_ratelimit.get_container_maxrate(0), None) self.assertEquals(ratelimit.get_maxrate(
self.assertEquals(test_ratelimit.get_container_maxrate(5), None) test_ratelimit.container_ratelimits, 0), None)
self.assertEquals(test_ratelimit.get_container_maxrate(10), 200) self.assertEquals(ratelimit.get_maxrate(
self.assertEquals(test_ratelimit.get_container_maxrate(60), 72) test_ratelimit.container_ratelimits, 5), None)
self.assertEquals(test_ratelimit.get_container_maxrate(160), 30) self.assertEquals(ratelimit.get_maxrate(
test_ratelimit.container_ratelimits, 10), 200)
self.assertEquals(ratelimit.get_maxrate(
test_ratelimit.container_ratelimits, 60), 72)
self.assertEquals(ratelimit.get_maxrate(
test_ratelimit.container_ratelimits, 160), 30)
def test_get_ratelimitable_key_tuples(self): def test_get_ratelimitable_key_tuples(self):
current_rate = 13 current_rate = 13
@ -190,15 +195,15 @@ class TestRateLimit(unittest.TestCase):
logger=FakeLogger()) logger=FakeLogger())
the_app.memcache_client = fake_memcache the_app.memcache_client = fake_memcache
self.assertEquals(len(the_app.get_ratelimitable_key_tuples( self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
'DELETE', 'a', None, None)), 0) 'DELETE', 'a', None, None)), 0)
self.assertEquals(len(the_app.get_ratelimitable_key_tuples( self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
'PUT', 'a', 'c', None)), 1) 'PUT', 'a', 'c', None)), 1)
self.assertEquals(len(the_app.get_ratelimitable_key_tuples( self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
'DELETE', 'a', 'c', None)), 1) 'DELETE', 'a', 'c', None)), 1)
self.assertEquals(len(the_app.get_ratelimitable_key_tuples( self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
'GET', 'a', 'c', 'o')), 0) 'GET', 'a', 'c', 'o')), 0)
self.assertEquals(len(the_app.get_ratelimitable_key_tuples( self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
'PUT', 'a', 'c', 'o')), 1) 'PUT', 'a', 'c', 'o')), 1)
def test_memcached_container_info_dict(self): def test_memcached_container_info_dict(self):
mdict = headers_to_container_info({'x-container-object-count': '45'}) mdict = headers_to_container_info({'x-container-object-count': '45'})
@ -223,8 +228,8 @@ class TestRateLimit(unittest.TestCase):
conf_dict = {'account_ratelimit': current_rate} conf_dict = {'account_ratelimit': current_rate}
self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp())
ratelimit.http_connect = mock_http_connect(204) ratelimit.http_connect = mock_http_connect(204)
for meth, exp_time in [('DELETE', 9.8), ('GET', 0), for meth, exp_time in [
('POST', 0), ('PUT', 9.8)]: ('DELETE', 9.8), ('GET', 0), ('POST', 0), ('PUT', 9.8)]:
req = Request.blank('/v/a%s/c' % meth) req = Request.blank('/v/a%s/c' % meth)
req.method = meth req.method = meth
req.environ['swift.cache'] = FakeMemcache() req.environ['swift.cache'] = FakeMemcache()
@ -281,8 +286,8 @@ class TestRateLimit(unittest.TestCase):
threads.append(rc) threads.append(rc)
for thread in threads: for thread in threads:
thread.join() thread.join()
the_498s = [t for t in threads if \ the_498s = [
''.join(t.result).startswith('Slow down')] t for t in threads if ''.join(t.result).startswith('Slow down')]
self.assertEquals(len(the_498s), 0) self.assertEquals(len(the_498s), 0)
self.assertEquals(time_ticker, 0) self.assertEquals(time_ticker, 0)
@ -316,8 +321,8 @@ class TestRateLimit(unittest.TestCase):
threads.append(rc) threads.append(rc)
for thread in threads: for thread in threads:
thread.join() thread.join()
the_497s = [t for t in threads if \ the_497s = [
''.join(t.result).startswith('Your account')] t for t in threads if ''.join(t.result).startswith('Your account')]
self.assertEquals(len(the_497s), 5) self.assertEquals(len(the_497s), 5)
self.assertEquals(time_ticker, 0) self.assertEquals(time_ticker, 0)
@ -350,6 +355,70 @@ class TestRateLimit(unittest.TestCase):
r = self.test_ratelimit(req.environ, start_response) r = self.test_ratelimit(req.environ, start_response)
self.assertEquals(r[0], '204 No Content') self.assertEquals(r[0], '204 No Content')
def test_ratelimit_max_rate_double_container(self):
global time_ticker
global time_override
current_rate = 2
conf_dict = {'container_ratelimit_0': current_rate,
'clock_accuracy': 100,
'max_sleep_time_seconds': 1}
self.test_ratelimit = dummy_filter_factory(conf_dict)(FakeApp())
ratelimit.http_connect = mock_http_connect(204)
self.test_ratelimit.log_sleep_time_seconds = .00001
req = Request.blank('/v/a/c/o')
req.method = 'PUT'
req.environ['swift.cache'] = FakeMemcache()
req.environ['swift.cache'].set(
ratelimit.get_container_memcache_key('a', 'c'),
{'container_size': 1})
time_override = [0, 0, 0, 0, None]
# simulates 4 requests coming in at same time, then sleeping
r = self.test_ratelimit(req.environ, start_response)
mock_sleep(.1)
r = self.test_ratelimit(req.environ, start_response)
mock_sleep(.1)
r = self.test_ratelimit(req.environ, start_response)
self.assertEquals(r[0], 'Slow down')
mock_sleep(.1)
r = self.test_ratelimit(req.environ, start_response)
self.assertEquals(r[0], 'Slow down')
mock_sleep(.1)
r = self.test_ratelimit(req.environ, start_response)
self.assertEquals(r[0], '204 No Content')
def test_ratelimit_max_rate_double_container_listing(self):
global time_ticker
global time_override
current_rate = 2
conf_dict = {'container_listing_ratelimit_0': current_rate,
'clock_accuracy': 100,
'max_sleep_time_seconds': 1}
self.test_ratelimit = dummy_filter_factory(conf_dict)(FakeApp())
ratelimit.http_connect = mock_http_connect(204)
self.test_ratelimit.log_sleep_time_seconds = .00001
req = Request.blank('/v/a/c')
req.method = 'GET'
req.environ['swift.cache'] = FakeMemcache()
req.environ['swift.cache'].set(
ratelimit.get_container_memcache_key('a', 'c'),
{'container_size': 1})
time_override = [0, 0, 0, 0, None]
# simulates 4 requests coming in at same time, then sleeping
r = self.test_ratelimit(req.environ, start_response)
mock_sleep(.1)
r = self.test_ratelimit(req.environ, start_response)
mock_sleep(.1)
r = self.test_ratelimit(req.environ, start_response)
self.assertEquals(r[0], 'Slow down')
mock_sleep(.1)
r = self.test_ratelimit(req.environ, start_response)
self.assertEquals(r[0], 'Slow down')
mock_sleep(.1)
r = self.test_ratelimit(req.environ, start_response)
self.assertEquals(r[0], '204 No Content')
def test_ratelimit_max_rate_multiple_acc(self): def test_ratelimit_max_rate_multiple_acc(self):
num_calls = 4 num_calls = 4
current_rate = 2 current_rate = 2
@ -420,7 +489,7 @@ class TestRateLimit(unittest.TestCase):
begin = time.time() begin = time.time()
self._run(make_app_call, num_calls, current_rate, check_time=False) self._run(make_app_call, num_calls, current_rate, check_time=False)
time_took = time.time() - begin time_took = time.time() - begin
self.assertEquals(round(time_took, 1), 0) # no memcache, no limiting self.assertEquals(round(time_took, 1), 0) # no memcache, no limiting
def test_restarting_memcache(self): def test_restarting_memcache(self):
current_rate = 2 current_rate = 2
@ -437,7 +506,7 @@ class TestRateLimit(unittest.TestCase):
begin = time.time() begin = time.time()
self._run(make_app_call, num_calls, current_rate, check_time=False) self._run(make_app_call, num_calls, current_rate, check_time=False)
time_took = time.time() - begin time_took = time.time() - begin
self.assertEquals(round(time_took, 1), 0) # no memcache, no limiting self.assertEquals(round(time_took, 1), 0) # no memcache, no limiting
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()