Merge "Faster swift-dispersion-populate"
This commit is contained in:
commit
920680ffdd
@ -20,7 +20,6 @@ from cStringIO import StringIO
|
|||||||
from optparse import OptionParser
|
from optparse import OptionParser
|
||||||
from sys import exit, stdout
|
from sys import exit, stdout
|
||||||
from time import time
|
from time import time
|
||||||
from uuid import uuid4
|
|
||||||
|
|
||||||
from eventlet import GreenPool, patcher, sleep
|
from eventlet import GreenPool, patcher, sleep
|
||||||
from eventlet.pools import Pool
|
from eventlet.pools import Pool
|
||||||
@ -87,6 +86,14 @@ if __name__ == '__main__':
|
|||||||
Usage: %%prog [options] [conf_file]
|
Usage: %%prog [options] [conf_file]
|
||||||
|
|
||||||
[conf_file] defaults to %s'''.strip() % conffile)
|
[conf_file] defaults to %s'''.strip() % conffile)
|
||||||
|
parser.add_option('--container-only', action='store_true', default=False,
|
||||||
|
help='Only run container population')
|
||||||
|
parser.add_option('--object-only', action='store_true', default=False,
|
||||||
|
help='Only run object population')
|
||||||
|
parser.add_option('--container-suffix-start', type=int, default=0,
|
||||||
|
help='container suffix start value, defaults to 0')
|
||||||
|
parser.add_option('--object-suffix-start', type=int, default=0,
|
||||||
|
help='object suffix start value, defaults to 0')
|
||||||
parser.add_option('--insecure', action='store_true', default=False,
|
parser.add_option('--insecure', action='store_true', default=False,
|
||||||
help='Allow accessing insecure keystone server. '
|
help='Allow accessing insecure keystone server. '
|
||||||
'The keystone\'s certificate will not be verified.')
|
'The keystone\'s certificate will not be verified.')
|
||||||
@ -106,6 +113,13 @@ Usage: %%prog [options] [conf_file]
|
|||||||
endpoint_type = str(conf.get('endpoint_type', 'publicURL'))
|
endpoint_type = str(conf.get('endpoint_type', 'publicURL'))
|
||||||
insecure = options.insecure \
|
insecure = options.insecure \
|
||||||
or config_true_value(conf.get('keystone_api_insecure', 'no'))
|
or config_true_value(conf.get('keystone_api_insecure', 'no'))
|
||||||
|
container_populate = config_true_value(
|
||||||
|
conf.get('container_populate', 'yes')) and not options.object_only
|
||||||
|
object_populate = config_true_value(
|
||||||
|
conf.get('object_populate', 'yes')) and not options.container_only
|
||||||
|
|
||||||
|
if not (object_populate or container_populate):
|
||||||
|
exit("Neither container or object populate is set to run")
|
||||||
|
|
||||||
coropool = GreenPool(size=concurrency)
|
coropool = GreenPool(size=concurrency)
|
||||||
retries_done = 0
|
retries_done = 0
|
||||||
@ -126,8 +140,10 @@ Usage: %%prog [options] [conf_file]
|
|||||||
os_options=os_options,
|
os_options=os_options,
|
||||||
insecure=insecure)
|
insecure=insecure)
|
||||||
|
|
||||||
|
if container_populate:
|
||||||
container_ring = Ring(swift_dir, ring_name='container')
|
container_ring = Ring(swift_dir, ring_name='container')
|
||||||
parts_left = dict((x, x) for x in xrange(container_ring.partition_count))
|
parts_left = dict((x, x)
|
||||||
|
for x in xrange(container_ring.partition_count))
|
||||||
item_type = 'containers'
|
item_type = 'containers'
|
||||||
created = 0
|
created = 0
|
||||||
retries_done = 0
|
retries_done = 0
|
||||||
@ -135,21 +151,27 @@ Usage: %%prog [options] [conf_file]
|
|||||||
dispersion_coverage / 100.0 * container_ring.partition_count
|
dispersion_coverage / 100.0 * container_ring.partition_count
|
||||||
begun = next_report = time()
|
begun = next_report = time()
|
||||||
next_report += 2
|
next_report += 2
|
||||||
|
suffix = 0
|
||||||
while need_to_queue >= 1:
|
while need_to_queue >= 1:
|
||||||
container = 'dispersion_%s' % uuid4().hex
|
container = 'dispersion_%d' % suffix
|
||||||
part, _junk = container_ring.get_nodes(account, container)
|
part, _junk = container_ring.get_nodes(account, container)
|
||||||
if part in parts_left:
|
if part in parts_left:
|
||||||
|
if suffix >= options.container_suffix_start:
|
||||||
coropool.spawn(put_container, connpool, container, report)
|
coropool.spawn(put_container, connpool, container, report)
|
||||||
sleep()
|
sleep()
|
||||||
|
else:
|
||||||
|
report(True)
|
||||||
del parts_left[part]
|
del parts_left[part]
|
||||||
need_to_queue -= 1
|
need_to_queue -= 1
|
||||||
|
suffix += 1
|
||||||
coropool.waitall()
|
coropool.waitall()
|
||||||
elapsed, elapsed_unit = get_time_units(time() - begun)
|
elapsed, elapsed_unit = get_time_units(time() - begun)
|
||||||
print '\r\x1B[KCreated %d containers for dispersion reporting, %d%s, %d ' \
|
print '\r\x1B[KCreated %d containers for dispersion reporting, ' \
|
||||||
'retries' % \
|
'%d%s, %d retries' % \
|
||||||
(need_to_create, round(elapsed), elapsed_unit, retries_done)
|
(need_to_create, round(elapsed), elapsed_unit, retries_done)
|
||||||
stdout.flush()
|
stdout.flush()
|
||||||
|
|
||||||
|
if object_populate:
|
||||||
container = 'dispersion_objects'
|
container = 'dispersion_objects'
|
||||||
put_container(connpool, container, None)
|
put_container(connpool, container, None)
|
||||||
object_ring = Ring(swift_dir, ring_name='object')
|
object_ring = Ring(swift_dir, ring_name='object')
|
||||||
@ -161,17 +183,23 @@ Usage: %%prog [options] [conf_file]
|
|||||||
dispersion_coverage / 100.0 * object_ring.partition_count
|
dispersion_coverage / 100.0 * object_ring.partition_count
|
||||||
begun = next_report = time()
|
begun = next_report = time()
|
||||||
next_report += 2
|
next_report += 2
|
||||||
|
suffix = 0
|
||||||
while need_to_queue >= 1:
|
while need_to_queue >= 1:
|
||||||
obj = 'dispersion_%s' % uuid4().hex
|
obj = 'dispersion_%d' % suffix
|
||||||
part, _junk = object_ring.get_nodes(account, container, obj)
|
part, _junk = object_ring.get_nodes(account, container, obj)
|
||||||
if part in parts_left:
|
if part in parts_left:
|
||||||
coropool.spawn(put_object, connpool, container, obj, report)
|
if suffix >= options.object_suffix_start:
|
||||||
|
coropool.spawn(
|
||||||
|
put_object, connpool, container, obj, report)
|
||||||
sleep()
|
sleep()
|
||||||
|
else:
|
||||||
|
report(True)
|
||||||
del parts_left[part]
|
del parts_left[part]
|
||||||
need_to_queue -= 1
|
need_to_queue -= 1
|
||||||
|
suffix += 1
|
||||||
coropool.waitall()
|
coropool.waitall()
|
||||||
elapsed, elapsed_unit = get_time_units(time() - begun)
|
elapsed, elapsed_unit = get_time_units(time() - begun)
|
||||||
print '\r\x1B[KCreated %d objects for dispersion reporting, %d%s, %d ' \
|
print '\r\x1B[KCreated %d objects for dispersion reporting, ' \
|
||||||
'retries' % \
|
'%d%s, %d retries' % \
|
||||||
(need_to_create, round(elapsed), elapsed_unit, retries_done)
|
(need_to_create, round(elapsed), elapsed_unit, retries_done)
|
||||||
stdout.flush()
|
stdout.flush()
|
||||||
|
@ -24,7 +24,7 @@
|
|||||||
|
|
||||||
.SH SYNOPSIS
|
.SH SYNOPSIS
|
||||||
.LP
|
.LP
|
||||||
.B swift-dispersion-populate [--insecure] [conf_file]
|
.B swift-dispersion-populate [--container-suffix-start] [--object-suffix-start] [--container-only|--object-only] [--insecure] [conf_file]
|
||||||
|
|
||||||
.SH DESCRIPTION
|
.SH DESCRIPTION
|
||||||
.PP
|
.PP
|
||||||
@ -62,6 +62,16 @@ privileges.
|
|||||||
.IP "\fB--insecure\fR"
|
.IP "\fB--insecure\fR"
|
||||||
Allow accessing insecure keystone server. The keystone's certificate will not
|
Allow accessing insecure keystone server. The keystone's certificate will not
|
||||||
be verified.
|
be verified.
|
||||||
|
.IP "\fB--container-suffix-start=NUMBER\fR"
|
||||||
|
Start container suffix at NUMBER and resume population at this point; default: 0
|
||||||
|
.IP "\fB--object-suffix-start=NUMBER\fR"
|
||||||
|
Start object suffix at NUMBER and resume population at this point; default: 0
|
||||||
|
.IP "\fB--object-only\fR"
|
||||||
|
Only run object population
|
||||||
|
.IP "\fB--container-only\fR"
|
||||||
|
Only run container population
|
||||||
|
.IP "\fB--object-only\fR"
|
||||||
|
Only run object population
|
||||||
|
|
||||||
.SH CONFIGURATION
|
.SH CONFIGURATION
|
||||||
.PD 0
|
.PD 0
|
||||||
|
@ -13,6 +13,8 @@ auth_key = testing
|
|||||||
# dispersion_coverage = 1.0
|
# dispersion_coverage = 1.0
|
||||||
# retries = 5
|
# retries = 5
|
||||||
# concurrency = 25
|
# concurrency = 25
|
||||||
|
# container_populate = yes
|
||||||
|
# object_populate = yes
|
||||||
# container_report = yes
|
# container_report = yes
|
||||||
# object_report = yes
|
# object_report = yes
|
||||||
# dump_json = no
|
# dump_json = no
|
||||||
|
Loading…
Reference in New Issue
Block a user