Dynamic Large Object sleeps too much

DLO is supposed to sleep 1 second per segment after 10 (default) initial
segments are loaded to prevent abuse of the system. Currently, if the user
did a range request to the 11th segment the 1 sec delay would start immediately
on the 11th segment because the skipped segments were counted as initial
segments. This path fixes that.

Change-Id: I53bbc24af54921ddc04d9cb3b704366b51b3f740

swift/proxy/controllers/obj.py

@@ -111,7 +111,7 @@ class SegmentedIterable(object):
         self.container = container
         self.listing = segment_listing_iter(listing)
         self.is_slo = is_slo
-        self.segment = 0
+        self.ratelimit_index = 0
         self.segment_dict = None
         self.segment_peek = None
         self.seek = 0
@@ -132,7 +132,7 @@ class SegmentedIterable(object):
                  segment no longer matches SLO manifest specifications.
         """
         try:
-            self.segment += 1
+            self.ratelimit_index += 1
             self.segment_dict = self.segment_peek or self.listing.next()
             self.segment_peek = None
             if self.container is None:
@@ -147,7 +147,7 @@ class SegmentedIterable(object):
             if self.seek:
                 req.range = 'bytes=%s-' % self.seek
                 self.seek = 0
-            if not self.is_slo and self.segment > \
+            if not self.is_slo and self.ratelimit_index > \
                     self.controller.app.rate_limit_after_segment:
                 sleep(max(self.next_get_time - time.time(), 0))
             self.next_get_time = time.time() + \
@@ -246,7 +246,6 @@ class SegmentedIterable(object):
             if start:
                 self.segment_peek = self.listing.next()
                 while start >= self.position + self.segment_peek['bytes']:
-                    self.segment += 1
                     self.position += self.segment_peek['bytes']
                     self.segment_peek = self.listing.next()
                 self.seek = start - self.position

test/unit/proxy/test_server.py

@@ -5467,10 +5467,51 @@ class TestSegmentedIterable(unittest.TestCase):
         finally:
             swift.proxy.controllers.obj.sleep = orig_sleep
 
+    def test_load_next_segment_range_req_rate_limiting(self):
+        sleep_calls = []
+
+        def _stub_sleep(sleepy_time):
+            sleep_calls.append(sleepy_time)
+        orig_sleep = swift.proxy.controllers.obj.sleep
+        try:
+            swift.proxy.controllers.obj.sleep = _stub_sleep
+            segit = SegmentedIterable(
+                self.controller, 'lc', [
+                    {'name': 'o0', 'bytes': 5}, {'name': 'o1', 'bytes': 5},
+                    {'name': 'o2', 'bytes': 1}, {'name': 'o3'}, {'name': 'o4'},
+                    {'name': 'o5'}, {'name': 'o6'}])
+
+            # this tests for a range request which skips over the whole first
+            # segment, after that 3 segments will be read in because the
+            # rate_limit_after_segment == 3, then sleeping starts
+            segit_iter = segit.app_iter_range(10, None)
+            segit_iter.next()
+            for _ in xrange(2):
+                # this is set to 2 instead of 3 because o2 was loaded after
+                # o0 and o1 were skipped.
+                segit._load_next_segment()
+            self.assertEquals([], sleep_calls)
+            self.assertEquals(self.controller.GETorHEAD_base_args[4],
+                              '/a/lc/o4')
+
+            # Loading of next (5th) segment starts rate-limiting.
+            segit._load_next_segment()
+            self.assertAlmostEqual(0.5, sleep_calls[0], places=2)
+            self.assertEquals(self.controller.GETorHEAD_base_args[4],
+                              '/a/lc/o5')
+
+            sleep_calls = []
+            segit._load_next_segment()
+            self.assertAlmostEqual(0.5, sleep_calls[0], places=2)
+            self.assertEquals(self.controller.GETorHEAD_base_args[4],
+                              '/a/lc/o6')
+        finally:
+            swift.proxy.controllers.obj.sleep = orig_sleep
+
     def test_load_next_segment_with_two_segments_skip_first(self):
         segit = SegmentedIterable(self.controller, 'lc', [{'name':
                                   'o1'}, {'name': 'o2'}])
-        segit.segment = 0
+        segit.ratelimit_index = 0
         segit.listing.next()
         segit._load_next_segment()
         self.assertEquals(self.controller.GETorHEAD_base_args[4], '/a/lc/o2')
@@ -5480,7 +5521,7 @@ class TestSegmentedIterable(unittest.TestCase):
     def test_load_next_segment_with_seek(self):
         segit = SegmentedIterable(self.controller, 'lc', [{'name':
                                   'o1'}, {'name': 'o2'}])
-        segit.segment = 0
+        segit.ratelimit_index = 0
         segit.listing.next()
         segit.seek = 1
         segit._load_next_segment()