openstack/swift
Code Issues Proposed changes

Add a way to ratelimit all writes to an account

Browse Source 
This is in case a cluster gets a problem user who has distributed the
writes to a bunch of containers but is just taking too much of the
cluster's resources.

Change-Id: Ibd2ffd0e911463a432117b478585b9f8bc4a2495
This commit is contained in:
David Goetz 2014-01-29 08:33:27 -08:00
parent 9034558f0b
commit b89ac55c05
3 changed files with 151 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
doc/source/ratelimit.rst
View File

@ -81,3 +81,20 @@ Container Size Rate Limit
================ ============
-----------------------------
Account Specific Ratelimiting
-----------------------------
The above ratelimiting is to prevent the "many writes to a single container"
bottleneck from causing a problem. There could also be a problem where a single
account is just using too much of the cluster's resources. In this case, the
container ratelimits may not help because the customer could be doing thousands
of reqs/sec to distributed containers each getting a small fraction of the
total so those limits would never trigger. If a system adminstrator notices
this, he/she can set the X-Account-Sysmeta-Global-Write-Ratelimit on an account
and that will limit the total number of write requests (PUT, POST, DELETE,
COPY) that account can do for the whole account. This limit will be in addition
to the applicable account/container limits from above. This header will be
hidden from the user, because of the gatekeeper middleware, and can only be set
using a direct client to the account nodes. It accepts a float value and will
only limit requests if the value is > 0.

29
swift/common/middleware/ratelimit.py
View File

@ -18,7 +18,8 @@ from swift import gettext_ as _
import eventlet
from swift.common.utils import cache_from_env, get_logger, register_swift_info
from swift.proxy.controllers.base import get_container_memcache_key
from swift.proxy.controllers.base import get_container_memcache_key, \
get_account_info
from swift.common.memcached import MemcacheConnectionError
from swift.common.swob import Request, Response
@ -117,13 +118,13 @@ class RateLimitMiddleware(object):
'object_count', container_info.get('container_size', 0))
return rv
def get_ratelimitable_key_tuples(self, req_method, account_name,
def get_ratelimitable_key_tuples(self, req, account_name,
container_name=None, obj_name=None):
"""
Returns a list of key (used in memcache), ratelimit tuples. Keys
should be checked in order.
:param req_method: HTTP method
:param req: swob request
:param account_name: account name from path
:param container_name: container name from path
:param obj_name: object name from path
@ -132,12 +133,12 @@ class RateLimitMiddleware(object):
# COPYs are not limited
if self.account_ratelimit and \
account_name and container_name and not obj_name and \
req_method in ('PUT', 'DELETE'):
req.method in ('PUT', 'DELETE'):
keys.append(("ratelimit/%s" % account_name,
self.account_ratelimit))
if account_name and container_name and obj_name and \
req_method in ('PUT', 'DELETE', 'POST'):
req.method in ('PUT', 'DELETE', 'POST', 'COPY'):
container_size = self.get_container_size(
account_name, container_name)
container_rate = get_maxrate(
@ -148,7 +149,7 @@ class RateLimitMiddleware(object):
container_rate))
if account_name and container_name and not obj_name and \
req_method == 'GET':
req.method == 'GET':
container_size = self.get_container_size(
account_name, container_name)
container_rate = get_maxrate(
@ -158,6 +159,20 @@ class RateLimitMiddleware(object):
"ratelimit_listing/%s/%s" % (account_name, container_name),
container_rate))
if account_name and req.method in ('PUT', 'DELETE', 'POST', 'COPY'):
account_info = get_account_info(req.environ, self.app)
account_global_ratelimit = \
account_info.get('sysmeta', {}).get('global-write-ratelimit')
if account_global_ratelimit:
try:
account_global_ratelimit = float(account_global_ratelimit)
if account_global_ratelimit > 0:
keys.append((
"ratelimit/global-write/%s" % account_name,
account_global_ratelimit))
except ValueError:
pass
return keys
def _get_sleep_time(self, key, max_rate):
@ -218,7 +233,7 @@ class RateLimitMiddleware(object):
if account_name in self.ratelimit_whitelist:
return None
for key, max_rate in self.get_ratelimitable_key_tuples(
req.method, account_name, container_name=container_name,
req, account_name, container_name=container_name,
obj_name=obj_name):
try:
need_to_sleep = self._get_sleep_time(key, max_rate)

176
test/unit/common/middleware/test_ratelimit.py
View File

@ -16,6 +16,7 @@
import unittest
import time
import eventlet
import mock
from contextlib import contextmanager
from threading import Thread
@ -194,16 +195,45 @@ class TestRateLimit(unittest.TestCase):
the_app = ratelimit.RateLimitMiddleware(None, conf_dict,
logger=FakeLogger())
the_app.memcache_client = fake_memcache
self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
'DELETE', 'a', None, None)), 0)
self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
'PUT', 'a', 'c', None)), 1)
self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
'DELETE', 'a', 'c', None)), 1)
self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
'GET', 'a', 'c', 'o')), 0)
self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
'PUT', 'a', 'c', 'o')), 1)
req = lambda: None
req.environ = {}
with mock.patch('swift.common.middleware.ratelimit.get_account_info',
lambda *args, **kwargs: {}):
req.method = 'DELETE'
self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
req, 'a', None, None)), 0)
req.method = 'PUT'
self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
req, 'a', 'c', None)), 1)
req.method = 'DELETE'
self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
req, 'a', 'c', None)), 1)
req.method = 'GET'
self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
req, 'a', 'c', 'o')), 0)
req.method = 'PUT'
self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
req, 'a', 'c', 'o')), 1)
def get_fake_ratelimit(*args, **kwargs):
return {'sysmeta': {'global-write-ratelimit': 10}}
with mock.patch('swift.common.middleware.ratelimit.get_account_info',
get_fake_ratelimit):
req.method = 'PUT'
self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
req, 'a', 'c', None)), 2)
self.assertEquals(the_app.get_ratelimitable_key_tuples(
req, 'a', 'c', None)[1], ('ratelimit/global-write/a', 10))
def get_fake_ratelimit(*args, **kwargs):
return {'sysmeta': {'global-write-ratelimit': 'notafloat'}}
with mock.patch('swift.common.middleware.ratelimit.get_account_info',
get_fake_ratelimit):
req.method = 'PUT'
self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
req, 'a', 'c', None)), 1)
def test_memcached_container_info_dict(self):
mdict = headers_to_container_info({'x-container-object-count': '45'})
@ -219,8 +249,13 @@ class TestRateLimit(unittest.TestCase):
the_app = ratelimit.RateLimitMiddleware(None, conf_dict,
logger=FakeLogger())
the_app.memcache_client = fake_memcache
tuples = the_app.get_ratelimitable_key_tuples('PUT', 'a', 'c', 'o')
self.assertEquals(tuples, [('ratelimit/a/c', 200.0)])
req = lambda: None
req.method = 'PUT'
req.environ = {}
with mock.patch('swift.common.middleware.ratelimit.get_account_info',
lambda *args, **kwargs: {}):
tuples = the_app.get_ratelimitable_key_tuples(req, 'a', 'c', 'o')
self.assertEquals(tuples, [('ratelimit/a/c', 200.0)])
def test_account_ratelimit(self):
current_rate = 5
@ -228,18 +263,20 @@ class TestRateLimit(unittest.TestCase):
conf_dict = {'account_ratelimit': current_rate}
self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp())
ratelimit.http_connect = mock_http_connect(204)
for meth, exp_time in [
('DELETE', 9.8), ('GET', 0), ('POST', 0), ('PUT', 9.8)]:
req = Request.blank('/v/a%s/c' % meth)
req.method = meth
req.environ['swift.cache'] = FakeMemcache()
make_app_call = lambda: self.test_ratelimit(req.environ,
start_response)
begin = time.time()
self._run(make_app_call, num_calls, current_rate,
check_time=bool(exp_time))
self.assertEquals(round(time.time() - begin, 1), exp_time)
self._reset_time()
with mock.patch('swift.common.middleware.ratelimit.get_account_info',
lambda *args, **kwargs: {}):
for meth, exp_time in [
('DELETE', 9.8), ('GET', 0), ('POST', 0), ('PUT', 9.8)]:
req = Request.blank('/v/a%s/c' % meth)
req.method = meth
req.environ['swift.cache'] = FakeMemcache()
make_app_call = lambda: self.test_ratelimit(req.environ,
start_response)
begin = time.time()
self._run(make_app_call, num_calls, current_rate,
check_time=bool(exp_time))
self.assertEquals(round(time.time() - begin, 1), exp_time)
self._reset_time()
def test_ratelimit_set_incr(self):
current_rate = 5
@ -254,8 +291,10 @@ class TestRateLimit(unittest.TestCase):
make_app_call = lambda: self.test_ratelimit(req.environ,
start_response)
begin = time.time()
self._run(make_app_call, num_calls, current_rate, check_time=False)
self.assertEquals(round(time.time() - begin, 1), 9.8)
with mock.patch('swift.common.middleware.ratelimit.get_account_info',
lambda *args, **kwargs: {}):
self._run(make_app_call, num_calls, current_rate, check_time=False)
self.assertEquals(round(time.time() - begin, 1), 9.8)
def test_ratelimit_whitelist(self):
global time_ticker
@ -342,18 +381,20 @@ class TestRateLimit(unittest.TestCase):
time_override = [0, 0, 0, 0, None]
# simulates 4 requests coming in at same time, then sleeping
r = self.test_ratelimit(req.environ, start_response)
mock_sleep(.1)
r = self.test_ratelimit(req.environ, start_response)
mock_sleep(.1)
r = self.test_ratelimit(req.environ, start_response)
self.assertEquals(r[0], 'Slow down')
mock_sleep(.1)
r = self.test_ratelimit(req.environ, start_response)
self.assertEquals(r[0], 'Slow down')
mock_sleep(.1)
r = self.test_ratelimit(req.environ, start_response)
self.assertEquals(r[0], '204 No Content')
with mock.patch('swift.common.middleware.ratelimit.get_account_info',
lambda *args, **kwargs: {}):
r = self.test_ratelimit(req.environ, start_response)
mock_sleep(.1)
r = self.test_ratelimit(req.environ, start_response)
mock_sleep(.1)
r = self.test_ratelimit(req.environ, start_response)
self.assertEquals(r[0], 'Slow down')
mock_sleep(.1)
r = self.test_ratelimit(req.environ, start_response)
self.assertEquals(r[0], 'Slow down')
mock_sleep(.1)
r = self.test_ratelimit(req.environ, start_response)
self.assertEquals(r[0], '204 No Content')
def test_ratelimit_max_rate_double_container(self):
global time_ticker
@ -374,18 +415,20 @@ class TestRateLimit(unittest.TestCase):
time_override = [0, 0, 0, 0, None]
# simulates 4 requests coming in at same time, then sleeping
r = self.test_ratelimit(req.environ, start_response)
mock_sleep(.1)
r = self.test_ratelimit(req.environ, start_response)
mock_sleep(.1)
r = self.test_ratelimit(req.environ, start_response)
self.assertEquals(r[0], 'Slow down')
mock_sleep(.1)
r = self.test_ratelimit(req.environ, start_response)
self.assertEquals(r[0], 'Slow down')
mock_sleep(.1)
r = self.test_ratelimit(req.environ, start_response)
self.assertEquals(r[0], '204 No Content')
with mock.patch('swift.common.middleware.ratelimit.get_account_info',
lambda *args, **kwargs: {}):
r = self.test_ratelimit(req.environ, start_response)
mock_sleep(.1)
r = self.test_ratelimit(req.environ, start_response)
mock_sleep(.1)
r = self.test_ratelimit(req.environ, start_response)
self.assertEquals(r[0], 'Slow down')
mock_sleep(.1)
r = self.test_ratelimit(req.environ, start_response)
self.assertEquals(r[0], 'Slow down')
mock_sleep(.1)
r = self.test_ratelimit(req.environ, start_response)
self.assertEquals(r[0], '204 No Content')
def test_ratelimit_max_rate_double_container_listing(self):
global time_ticker
@ -431,6 +474,7 @@ class TestRateLimit(unittest.TestCase):
the_app.memcache_client = fake_memcache
req = lambda: None
req.method = 'PUT'
req.environ = {}
class rate_caller(Thread):
@ -443,18 +487,20 @@ class TestRateLimit(unittest.TestCase):
self.result = the_app.handle_ratelimit(req, self.myname,
'c', None)
nt = 15
begin = time.time()
threads = []
for i in range(nt):
rc = rate_caller('a%s' % i)
rc.start()
threads.append(rc)
for thread in threads:
thread.join()
with mock.patch('swift.common.middleware.ratelimit.get_account_info',
lambda *args, **kwargs: {}):
nt = 15
begin = time.time()
threads = []
for i in range(nt):
rc = rate_caller('a%s' % i)
rc.start()
threads.append(rc)
for thread in threads:
thread.join()
time_took = time.time() - begin
self.assertEquals(1.5, round(time_took, 1))
time_took = time.time() - begin
self.assertEquals(1.5, round(time_took, 1))
def test_call_invalid_path(self):
env = {'REQUEST_METHOD': 'GET',
@ -504,9 +550,11 @@ class TestRateLimit(unittest.TestCase):
make_app_call = lambda: self.test_ratelimit(req.environ,
start_response)
begin = time.time()
self._run(make_app_call, num_calls, current_rate, check_time=False)
time_took = time.time() - begin
self.assertEquals(round(time_took, 1), 0) # no memcache, no limiting
with mock.patch('swift.common.middleware.ratelimit.get_account_info',
lambda *args, **kwargs: {}):
self._run(make_app_call, num_calls, current_rate, check_time=False)
time_took = time.time() - begin
self.assertEquals(round(time_took, 1), 0) # no memcache, no limit
if __name__ == '__main__':
unittest.main()