From 0d45e99ff0b2e1564c4cd6f35f47a0545538c233 Mon Sep 17 00:00:00 2001 From: Samuel Merritt Date: Wed, 27 Nov 2013 18:24:17 -0800 Subject: [PATCH] Expose bulk-operation limits in /info. These will allow clients to perform the minimal number of requests required to accomplish some bulk tasks. For example, a client with many objects to delete can learn that the cluster's limit on deletes-per-request is, say, 128, and then batch up their deletes in groups of 128. Without this, the client has to either discover the limit out-of-band somehow (and get notified if it changes), or do some sort of binary search to figure out the limit. Similar reasoning applies to the containers-per-request value. The errors-per-request values are included so that clients may size their requests such that everything is attempted regardless of failure. I split the 'bulk' entry into 'bulk_delete' and 'bulk_upload' because, from a client's standpoint, they're separate operations. It so happens that Swift implements both in one piece of middleware, but clients don't care. Bonus fix: documented a missing config setting for the bulk middleware. Change-Id: Ic3549aef79682fd5b798145c3545c1609aa1592b --- etc/proxy-server.conf-sample | 1 + swift/common/middleware/bulk.py | 43 +++++++++++++++++------- test/unit/common/middleware/test_bulk.py | 26 ++++++++++++++ 3 files changed, 58 insertions(+), 12 deletions(-) diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index 26e31828c6..01032d24b0 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -492,6 +492,7 @@ use = egg:swift#bulk # max_containers_per_extraction = 10000 # max_failed_extractions = 1000 # max_deletes_per_request = 10000 +# max_failed_deletes = 1000 # yield_frequency = 60 # Note: Put after auth in the pipeline. diff --git a/swift/common/middleware/bulk.py b/swift/common/middleware/bulk.py index 7dcb79e782..f19815c5a8 100644 --- a/swift/common/middleware/bulk.py +++ b/swift/common/middleware/bulk.py @@ -184,18 +184,16 @@ class Bulk(object): payload sent to the proxy (the list of objects/containers to be deleted). """ - def __init__(self, app, conf): + def __init__(self, app, conf, max_containers_per_extraction=10000, + max_failed_extractions=1000, max_deletes_per_request=10000, + max_failed_deletes=1000, yield_frequency=60): self.app = app self.logger = get_logger(conf, log_route='bulk') - self.max_containers = int( - conf.get('max_containers_per_extraction', 10000)) - self.max_failed_extractions = int( - conf.get('max_failed_extractions', 1000)) - self.max_failed_deletes = int( - conf.get('max_failed_deletes', 1000)) - self.max_deletes_per_request = int( - conf.get('max_deletes_per_request', 10000)) - self.yield_frequency = int(conf.get('yield_frequency', 60)) + self.max_containers = max_containers_per_extraction + self.max_failed_extractions = max_failed_extractions + self.max_failed_deletes = max_failed_deletes + self.max_deletes_per_request = max_deletes_per_request + self.yield_frequency = yield_frequency def create_container(self, req, container_path): """ @@ -542,8 +540,29 @@ class Bulk(object): def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) - register_swift_info('bulk') + + max_containers_per_extraction = \ + int(conf.get('max_containers_per_extraction', 10000)) + max_failed_extractions = int(conf.get('max_failed_extractions', 1000)) + max_deletes_per_request = int(conf.get('max_deletes_per_request', 10000)) + max_failed_deletes = int(conf.get('max_failed_deletes', 1000)) + yield_frequency = int(conf.get('yield_frequency', 60)) + + register_swift_info( + 'bulk_upload', + max_containers_per_extraction=max_containers_per_extraction, + max_failed_extractions=max_failed_extractions) + register_swift_info( + 'bulk_delete', + max_deletes_per_request=max_deletes_per_request, + max_failed_deletes=max_failed_deletes) def bulk_filter(app): - return Bulk(app, conf) + return Bulk( + app, conf, + max_containers_per_extraction=max_containers_per_extraction, + max_failed_extractions=max_failed_extractions, + max_deletes_per_request=max_deletes_per_request, + max_failed_deletes=max_failed_deletes, + yield_frequency=yield_frequency) return bulk_filter diff --git a/test/unit/common/middleware/test_bulk.py b/test/unit/common/middleware/test_bulk.py index d5b72bb35a..165808f2ae 100644 --- a/test/unit/common/middleware/test_bulk.py +++ b/test/unit/common/middleware/test_bulk.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import numbers import unittest import os import tarfile @@ -22,6 +23,7 @@ from shutil import rmtree from tempfile import mkdtemp from StringIO import StringIO from mock import patch +from swift.common import utils from swift.common.middleware import bulk from swift.common.swob import Request, Response, HTTPException from swift.common.http import HTTP_NOT_FOUND, HTTP_UNAUTHORIZED @@ -736,5 +738,29 @@ class TestDelete(unittest.TestCase): ['/c/f2', '401 Unauthorized']]) +class TestSwiftInfo(unittest.TestCase): + def setUp(self): + utils._swift_info = {} + utils._swift_admin_info = {} + + def test_registered_defaults(self): + bulk.filter_factory({}) + swift_info = utils.get_swift_info() + self.assertTrue('bulk_upload' in swift_info) + self.assertTrue(isinstance( + swift_info['bulk_upload'].get('max_containers_per_extraction'), + numbers.Integral)) + self.assertTrue(isinstance( + swift_info['bulk_upload'].get('max_failed_extractions'), + numbers.Integral)) + + self.assertTrue('bulk_delete' in swift_info) + self.assertTrue(isinstance( + swift_info['bulk_delete'].get('max_deletes_per_request'), + numbers.Integral)) + self.assertTrue(isinstance( + swift_info['bulk_delete'].get('max_failed_deletes'), + numbers.Integral)) + if __name__ == '__main__': unittest.main()