openstack/swift
Code Issues Proposed changes

minor bug fixes- pre ratelimit caching of container size

Browse Source
This commit is contained in:
David Goetz 2010-10-06 13:11:58 -07:00
parent 72d40bd9f6
commit db4689689a
4 changed files with 24 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.functests
View File

@ -1,4 +1,4 @@
#!/bin/bash #!/bin/bash
python test/functional/tests.py nosetests test/functional --exe
nosetests test/functionalnosetests --exe nosetests test/functionalnosetests --exe

14
etc/proxy-server.conf-sample
View File

@ -59,12 +59,22 @@ use = egg:swift#memcache
[filter:ratelimit] [filter:ratelimit]
use = egg:swift#ratelimit use = egg:swift#ratelimit
# clock_accuracy should represent how accurate the proxy servers' system clocks
# are with each other. 1000 means that all the proxies' clock are accurate to
# each other within 1 millisecond. No ratelimit should be higher than the
# clock accuracy.
clock_accuracy = 1000
max_sleep_time_seconds = 60
account_ratelimit = 200 account_ratelimit = 200
# these are comma separated lists of account names
account_whitelist = a,b account_whitelist = a,b
# account_blacklist = # account_blacklist =
# with container_limit_x = r # with container_limit_x = r
# for containers of size x limit requests per second to r # for containers of size x limit requests per second to r. The container
# rate will be linearly interpolated from the values given. With the values
# below, a container of size 5 will get a rate of 75.
container_limit_0 = 100 container_limit_0 = 100
container_limit_10 = 50 container_limit_10 = 50
container_limit_50 = 10 container_limit_50 = 20

2
setup.py
View File

@ -88,7 +88,7 @@ setup(
'auth=swift.common.middleware.auth:filter_factory', 'auth=swift.common.middleware.auth:filter_factory',
'healthcheck=swift.common.middleware.healthcheck:filter_factory', 'healthcheck=swift.common.middleware.healthcheck:filter_factory',
'memcache=swift.common.middleware.memcache:filter_factory', 'memcache=swift.common.middleware.memcache:filter_factory',
# 'ratelimit=swift.common.middeware.ratelimit:filter_factory', 'ratelimit=swift.common.middleware.ratelimit:filter_factory',
], ],
}, },
) )

20
swift/common/middleware/ratelimit.py
View File

@ -35,9 +35,9 @@ class RateLimitMiddleware(object):
else: else:
self.logger = logger self.logger = logger
self.account_rate_limit = float(conf.get('account_ratelimit', 1))#200.0)) self.account_rate_limit = float(conf.get('account_ratelimit', 200.0))
self.max_sleep_time_seconds = int(conf.get('max_sleep_time_seconds', self.max_sleep_time_seconds = float(conf.get('max_sleep_time_seconds',
2))#60)) 60))
self.clock_accuracy = int(conf.get('clock_accuracy', 1000)) self.clock_accuracy = int(conf.get('clock_accuracy', 1000))
self.rate_limit_whitelist = [acc.strip() for acc in self.rate_limit_whitelist = [acc.strip() for acc in
@ -101,14 +101,14 @@ class RateLimitMiddleware(object):
if account_name: if account_name:
keys.append(("ratelimit/%s" % account_name, keys.append(("ratelimit/%s" % account_name,
self.account_rate_limit)) self.account_rate_limit))
if account_name and container_name and not obj_name: if account_name and container_name and not obj_name:
container_size = None container_size = None
memcache_key = get_container_memcache_key(account_name, memcache_key = get_container_memcache_key(account_name,
container_name) container_name)
container_info = self.memcache_client.get(memcache_key) container_info = self.memcache_client.get(memcache_key)
if type(container_info) == dict: if type(container_info) == dict:
container_size = container_info.get('container_size') container_size = int(container_info.get('container_size', 0))
container_rate = self.get_container_maxrate(container_size) container_rate = self.get_container_maxrate(container_size)
if container_rate: if container_rate:
keys.append(("ratelimit/%s/%s" % (account_name, keys.append(("ratelimit/%s/%s" % (account_name,
@ -139,7 +139,7 @@ class RateLimitMiddleware(object):
if max_sleep_m - need_to_sleep_m <= self.clock_accuracy * 0.01: if max_sleep_m - need_to_sleep_m <= self.clock_accuracy * 0.01:
# make it accurate to 1% of clock accuracy # make it accurate to 1% of clock accuracy
# treat as no-op decrement time # treat as no-op decrement time
self.memcache_client.decr(key, delta=time_per_request_m) self.memcache_client.incr(key, delta=-time_per_request_m)
raise MaxSleepTimeHit("Max Sleep Time Exceeded: %s" % raise MaxSleepTimeHit("Max Sleep Time Exceeded: %s" %
need_to_sleep_m) need_to_sleep_m)
@ -161,9 +161,8 @@ class RateLimitMiddleware(object):
container_name, container_name,
obj_name): obj_name):
try: try:
need_to_sleep = self._get_sleep_time(key, need_to_sleep = self._get_sleep_time(key, max_rate)
max_rate) if need_to_sleep > 0:
if need_to_sleep > 0:
time.sleep(need_to_sleep) time.sleep(need_to_sleep)
except MaxSleepTimeHit, e: except MaxSleepTimeHit, e:
@ -177,6 +176,7 @@ class RateLimitMiddleware(object):
def __call__(self, env, start_response, name=None): def __call__(self, env, start_response, name=None):
#TODO : David- get rid of the name thing- used for debugging
req = Request(env) req = Request(env)
if self.memcache_client is None: if self.memcache_client is None:
self.memcache_client = cache_from_env(env) self.memcache_client = cache_from_env(env)