Merge "Add round_robin_iter function to common/utils"

swift/common/db_replicator.py

@@ -33,7 +33,7 @@ from swift.common.direct_client import quote
 from swift.common.utils import get_logger, whataremyips, storage_directory, \
     renamer, mkdirs, lock_parent_directory, config_true_value, \
     unlink_older_than, dump_recon_cache, rsync_module_interpolation, \
-    json, Timestamp, parse_overrides
+    json, Timestamp, parse_overrides, round_robin_iter
 from swift.common import ring
 from swift.common.ring.utils import is_local_device
 from swift.common.http import HTTP_NOT_FOUND, HTTP_INSUFFICIENT_STORAGE
@@ -127,12 +127,10 @@ def roundrobin_datadirs(datadirs):
 
     its = [walk_datadir(datadir, node_id, filt)
            for datadir, node_id, filt in datadirs]
-    while its:
+
-        for it in its:
+    rr_its = round_robin_iter(its)
-            try:
+    for datadir in rr_its:
-                yield next(it)
+        yield datadir
-            except StopIteration:
-                its.remove(it)
 
 
 class ReplConnection(BufferedHTTPConnection):

swift/common/utils.py

@@ -4642,3 +4642,17 @@ class PipeMutex(object):
 class ThreadSafeSysLogHandler(SysLogHandler):
     def createLock(self):
         self.lock = PipeMutex()
+
+
+def round_robin_iter(its):
+    """
+    Takes a list of iterators, yield an element from each in a round-robin
+    fashion until all of them are exhausted.
+    :param its: list of iterators
+    """
+    while its:
+        for it in its:
+            try:
+                yield next(it)
+            except StopIteration:
+                its.remove(it)

test/unit/common/test_utils.py

@@ -4027,6 +4027,19 @@ cluster_dfw1 = http://dfw1.host/v1/
         self.assertEqual(utils.replace_partition_in_path(old, 10), old)
         self.assertEqual(utils.replace_partition_in_path(new, 11), new)
 
+    def test_round_robin_iter(self):
+        it1 = iter([1, 2, 3])
+        it2 = iter([4, 5])
+        it3 = iter([6, 7, 8, 9])
+        it4 = iter([])
+
+        rr_its = utils.round_robin_iter([it1, it2, it3, it4])
+        got = list(rr_its)
+
+        # Expect that items get fetched in a round-robin fashion from the
+        # iterators
+        self.assertListEqual([1, 4, 6, 2, 5, 7, 3, 8, 9], got)
+
 
 class ResellerConfReader(unittest.TestCase):