diff --git a/doc/source/deployment_guide.rst b/doc/source/deployment_guide.rst index 40854b0a1f..133f8043df 100644 --- a/doc/source/deployment_guide.rst +++ b/doc/source/deployment_guide.rst @@ -229,6 +229,7 @@ Option Default Description log_name object-auditor Label used when logging log_facility LOG_LOCAL0 Syslog log facility log_level INFO Logging level +log_time 3600 Frequency of status logs in seconds. files_per_second 20 Maximum files audited per second. Should be tuned according to individual system specs. 0 is unlimited. diff --git a/doc/source/ratelimit.rst b/doc/source/ratelimit.rst index 80db870773..4924b71153 100644 --- a/doc/source/ratelimit.rst +++ b/doc/source/ratelimit.rst @@ -30,6 +30,11 @@ max_sleep_time_seconds 60 App will immediately return a 498 response log_sleep_time_seconds 0 To allow visibility into rate limiting set this value > 0 and all sleeps greater than the number will be logged. +rate_buffer_seconds 5 Number of seconds the rate counter can + drop and be allowed to catch up (at a + faster than listed rate). A larger number + will result in larger spikes in rate but + better average accuracy. account_ratelimit 0 If set, will limit all requests to /account_name and PUTs to /account_name/container_name. Number is in diff --git a/etc/object-server.conf-sample b/etc/object-server.conf-sample index cc80c18c07..92e37ed58c 100644 --- a/etc/object-server.conf-sample +++ b/etc/object-server.conf-sample @@ -57,3 +57,4 @@ use = egg:swift#object # log_name = object-auditor # files_per_second = 20 # bytes_per_second = 10000000 +# log_time = 3600 \ No newline at end of file diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index a3f64f8415..6b31d9dafe 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -99,6 +99,8 @@ use = egg:swift#ratelimit # max_sleep_time_seconds = 60 # log_sleep_time_seconds of 0 means disabled # log_sleep_time_seconds = 0 +# allows for slow rates (e.g. running up to 5 sec's behind) to catch up. +# rate_buffer_seconds = 5 # account_ratelimit of 0 means disabled # account_ratelimit = 0 diff --git a/swift/common/middleware/ratelimit.py b/swift/common/middleware/ratelimit.py index c0827da88b..cc2c8b1417 100644 --- a/swift/common/middleware/ratelimit.py +++ b/swift/common/middleware/ratelimit.py @@ -44,6 +44,7 @@ class RateLimitMiddleware(object): self.log_sleep_time_seconds = float(conf.get('log_sleep_time_seconds', 0)) self.clock_accuracy = int(conf.get('clock_accuracy', 1000)) + self.rate_buffer_seconds = int(conf.get('rate_buffer_seconds', 5)) self.ratelimit_whitelist = [acc.strip() for acc in conf.get('account_whitelist', '').split(',') if acc.strip()] @@ -140,8 +141,8 @@ class RateLimitMiddleware(object): running_time_m = self.memcache_client.incr(key, delta=time_per_request_m) need_to_sleep_m = 0 - request_time_limit = now_m + (time_per_request_m * max_rate) - if running_time_m < now_m: + if (now_m - running_time_m > + self.rate_buffer_seconds * self.clock_accuracy): next_avail_time = int(now_m + time_per_request_m) self.memcache_client.set(key, str(next_avail_time), serialize=False) diff --git a/swift/common/utils.py b/swift/common/utils.py index 299980493a..80f8125e1c 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -820,7 +820,7 @@ def audit_location_generator(devices, datadir, mount_check=True, logger=None): yield path, device, partition -def ratelimit_sleep(running_time, max_rate, incr_by=1): +def ratelimit_sleep(running_time, max_rate, incr_by=1, rate_buffer=5): ''' Will eventlet.sleep() for the appropriate time so that the max_rate is never exceeded. If max_rate is 0, will not ratelimit. The @@ -834,13 +834,17 @@ def ratelimit_sleep(running_time, max_rate, incr_by=1): :param incr_by: How much to increment the counter. Useful if you want to ratelimit 1024 bytes/sec and have differing sizes of requests. Must be >= 0. + :param rate_buffer: Number of seconds the rate counter can drop and be + allowed to catch up (at a faster than listed rate). + A larger number will result in larger spikes in rate + but better average accuracy. ''' if not max_rate or incr_by <= 0: return running_time clock_accuracy = 1000.0 now = time.time() * clock_accuracy time_per_request = clock_accuracy * (float(incr_by) / max_rate) - if running_time < now: + if now - running_time > rate_buffer * clock_accuracy: running_time = now elif running_time - now > time_per_request: eventlet.sleep((running_time - now) / clock_accuracy) diff --git a/swift/obj/auditor.py b/swift/obj/auditor.py index 62fc747e86..09fdd77774 100644 --- a/swift/obj/auditor.py +++ b/swift/obj/auditor.py @@ -38,6 +38,7 @@ class ObjectAuditor(Daemon): self.max_files_per_second = float(conf.get('files_per_second', 20)) self.max_bytes_per_second = float(conf.get('bytes_per_second', 10000000)) + self.log_time = int(conf.get('log_time', 3600)) self.files_running_time = 0 self.bytes_running_time = 0 self.bytes_processed = 0 @@ -46,7 +47,6 @@ class ObjectAuditor(Daemon): self.passes = 0 self.quarantines = 0 self.errors = 0 - self.log_time = 3600 # once an hour def run_forever(self): """Run the object audit until stopped.""" diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 1f5a94edd5..ee95f5f129 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -456,15 +456,6 @@ log_name = yarr''' # make sure its accurate to 10th of a second self.assertTrue(abs(25 - (time.time() - start) * 100) < 10) - def test_ratelimit_sleep_with_sleep(self): - running_time = 0 - start = time.time() - for i in range(25): - running_time = utils.ratelimit_sleep(running_time, 50) - time.sleep(1.0 / 75) - # make sure its accurate to 10th of a second - self.assertTrue(abs(50 - (time.time() - start) * 100) < 10) - def test_ratelimit_sleep_with_incr(self): running_time = 0 start = time.time() @@ -477,6 +468,17 @@ log_name = yarr''' total += i self.assertTrue(abs(50 - (time.time() - start) * 100) < 10) + def test_ratelimit_sleep_with_sleep(self): + running_time = 0 + start = time.time() + sleeps = [0] * 7 + [.2] * 3 + [0] * 30 + for i in sleeps: + running_time = utils.ratelimit_sleep(running_time, 40, + rate_buffer = 1) + time.sleep(i) + # make sure its accurate to 10th of a second + self.assertTrue(abs(100 - (time.time() - start) * 100) < 10) + if __name__ == '__main__': unittest.main()