Misc. swift-bench improvements.

swift-bench now honors the environment variables, ST_AUTH, ST_USER, and
ST_KEY like python-swiftclient does.

Added --lower-object-size (or -l) command-line option which, if
specified, will turn a specified --object-size into --upper-object-size.
It will raise a ValueError if --object-size is not specified or is <=
--lower-object-size.

BenchController how handles SIGINT (KeyboardInterrupt) similarly to the
swift command-line client: the first Ctrl-C will make it fast-track to
completion (no new PUT or GET operations are started, but everything PUT
is DELETE'ed).  A second Ctrl-C will immediately exit.  The behavior for
SIGTERM is unchanged (a single SIGTERM will immediately terminate the
process).

Added a sample configuration file for swift-bench, with documenting
comments.

Change-Id: I6f394ad995300fc8af3d565d95c3b45559ee510a

bin/swift-bench

@@ -26,43 +26,43 @@ from swift.common.utils import readconf, LogAdapter
 
 # The defaults should be sufficient to run swift-bench on a SAIO
 CONF_DEFAULTS = {
-    'auth': '',
+    'auth': os.environ.get('ST_AUTH', ''),
-    'user': '',
+    'user': os.environ.get('ST_USER', ''),
-    'key': '',
+    'key': os.environ.get('ST_KEY', ''),
-    'object_sources': '',
+    'auth_version': '1.0',
+    'use_proxy': 'yes',
     'put_concurrency': '10',
     'get_concurrency': '10',
     'del_concurrency': '10',
-    'concurrency': '',
+    'concurrency': '',  # set all 3 in one shot
-    'object_size': '1',
+    'object_sources': '',  # set of file contents to read and use for PUTs
-    'lower_object_size': '10',
+    'lower_object_size': '10',  # bounded random size used if these differ
     'upper_object_size': '10',
+    'object_size': '1',  # only if not object_sources and lower == upper
     'num_objects': '1000',
     'num_gets': '10000',
     'delete': 'yes',
-    'container_name': uuid.uuid4().hex,
+    'container_name': uuid.uuid4().hex,  # really "container name base"
     'num_containers': '20',
-    'use_proxy': 'yes',
+    'url': '',  # used when use_proxy = no or overrides auth X-Storage-Url
-    'url': '',
+    'account': '',  # used when use_proxy = no
-    'account': '',
+    'devices': 'sdb1',  # space-sep list
-    'devices': 'sdb1',
     'log_level': 'INFO',
     'timeout': '10',
-    'auth_version': '1.0',
+}
-    }
 
 SAIO_DEFAULTS = {
     'auth': 'http://localhost:8080/auth/v1.0',
     'user': 'test:tester',
     'key': 'testing',
-    }
+}
 
 if __name__ == '__main__':
     usage = "usage: %prog [OPTIONS] [CONF_FILE]"
     usage += """\n\nConf file with SAIO defaults:
 
     [bench]
-    auth = http://localhost:8080/v1.0
+    auth = http://localhost:8080/auth/v1.0
     user = test:tester
     key = testing
     concurrency = 10
@@ -87,6 +87,9 @@ if __name__ == '__main__':
                       help='Number of concurrent connections to use')
     parser.add_option('-s', '--object-size', dest='object_size',
                       help='Size of objects to PUT (in bytes)')
+    parser.add_option('-l', '--lower-object-size', dest='lower_object_size',
+                      help=('Lower size of objects (in bytes); '
+                            '--object-size will be upper-object-size'))
     parser.add_option('-n', '--num-objects', dest='num_objects',
                       help='Number of objects to PUT')
     parser.add_option('-g', '--num-gets', dest='num_gets',
@@ -102,12 +105,18 @@ if __name__ == '__main__':
     options, args = parser.parse_args()
     if options.saio:
         CONF_DEFAULTS.update(SAIO_DEFAULTS)
+    if getattr(options, 'lower_object_size', None):
+        if options.object_size <= options.lower_object_size:
+            raise ValueError('--lower-object-size (%s) must be '
+                             '< --object-size (%s)' %
+                             (options.lower_object_size, options.object_size))
+        CONF_DEFAULTS['upper_object_size'] = options.object_size
     if args:
         conf = args[0]
         if not os.path.exists(conf):
             sys.exit("No such conf file: %s" % conf)
         conf = readconf(conf, 'bench', log_name='swift-bench',
-            defaults=CONF_DEFAULTS)
+                        defaults=CONF_DEFAULTS)
     else:
         conf = CONF_DEFAULTS
     parser.set_defaults(**conf)

etc/swift-bench.conf-sample

@@ -0,0 +1,60 @@
+[bench]
+# auth = http://localhost:8080/auth/v1.0
+# user = test:tester
+# key = testing
+# auth_version = 1.0
+# log-level = INFO
+# timeout = 10
+
+# You can configure PUT, GET, and DELETE concurrency independently or set all
+# three with "concurrency"
+# put_concurrency = 10
+# get_concurrency = 10
+# del_concurrency = 10
+# concurrency =
+
+# A space-sep list of files whose contents will be read and randomly chosen
+# as the body (object contents) for each PUT.
+# object_sources =
+
+# If object_sources is not set and lower_object_size != upper_object_size,
+# each PUT will randomly select an object size between the two values.  Units
+# are bytes.
+# lower_object_size = 10
+# upper_object_size = 10
+
+# If object_sources is not set and lower_object_size == upper_object_size,
+# every object PUT will contain this many bytes.
+# object_size = 1
+
+# num_objects = 1000
+# num_gets = 10000
+# num_containers = 20
+
+# The base name for created containers.
+# container_name = (randomly-chosen uuid4)
+
+# Should swift-bench benchmark DELETEing the created objects and then delete
+# all created containers?
+# delete = yes
+
+# Without use_proxy, swift-bench will talk directly to the backend Swift
+# servers.  Doing that will require "url", "account", and at least one
+# "devices" entry.
+# use_proxy = yes
+
+# If use_proxy = yes, this will override any returned X-Storage-Url returned
+# by authenticaion (the account name will still be extracted from
+# X-Storage-Url though and may NOT be set with the "account" conf var).  If
+# use_proxy = no, this setting is required and used as the X-Storage-Url when
+# deleting containers and as a source for IP and port for back-end Swift server
+# connections.  The IP and port specified in this setting must have local
+# storage access to every device specified in "devices".
+# url =
+
+# Only used (and required) when use_proxy = no.
+# account =
+
+# A space-sep list of devices names; only relevant (and required) when
+# use_proxy = no.
+# devices = sdb1

swift/common/bench.py

@@ -13,9 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import sys
 import uuid
 import time
 import random
+import signal
 from contextlib import contextmanager
 
 import eventlet.pools
@@ -41,6 +43,7 @@ class Bench(object):
 
     def __init__(self, logger, conf, names):
         self.logger = logger
+        self.aborted = False
         self.user = conf.user
         self.key = conf.key
         self.auth_url = conf.auth
@@ -116,6 +119,8 @@ class Bench(object):
         self.failures = 0
         self.complete = 0
         for i in xrange(self.total):
+            if self.aborted:
+                break
             pool.spawn_n(self._run, i)
         pool.waitall()
         self._log_status(self.msg + ' **FINAL**')
@@ -132,15 +137,35 @@ class BenchController(object):
         self.names = []
         self.delete = conf.delete.lower() in TRUE_VALUES
         self.gets = int(conf.num_gets)
+        self.aborted = False
+
+    def sigint1(self, signum, frame):
+        if self.delete:
+            print >>sys.stderr, (
+                'SIGINT received; finishing up and running DELETE.\n'
+                'Send one more SIGINT to exit *immediately*.')
+            self.aborted = True
+            if self.running and not isinstance(self.running, BenchDELETE):
+                self.running.aborted = True
+            signal.signal(signal.SIGINT, self.sigint2)
+        else:
+            self.sigint2(signum, frame)
+
+    def sigint2(self, signum, frame):
+        sys.exit('Final SIGINT received.')
 
     def run(self):
+        signal.signal(signal.SIGINT, self.sigint1)
         puts = BenchPUT(self.logger, self.conf, self.names)
+        self.running = puts
         puts.run()
-        if self.gets:
+        if self.gets and not self.aborted:
             gets = BenchGET(self.logger, self.conf, self.names)
+            self.running = gets
             gets.run()
         if self.delete:
             dels = BenchDELETE(self.logger, self.conf, self.names)
+            self.running = dels
             dels.run()