openstack/swift
Code Issues Proposed changes

Add a region tier to Swift's ring.

Browse Source 
The region is one level above the zone; it is intended to represent a
chunk of machines that is distant from others with respect to
bandwidth and latency.

Old rings will default to having all their devices in region 1. Since
everything is in the same region by default, the ring builder will
simply distribute across zones as it did before, so your partition
assignment won't move because of this change. If you start adding
devices in other regions, of course, the assignment will change to
take that into account.

swift-ring-builder still accepts the same syntax as before, but will
default added devices to region 1 if no region is specified.

Examples:

$ swift-ring-builder foo.builder add r2z1-1.2.3.4:555/sda

$ swift-ring-builder foo.builder add r1z3-1.2.3.4:555/sda

$ swift-ring-builder foo.builder add z3-1.2.3.4:555/sda

Also, some updates to ring-overview doc.

Change-Id: Ifefbb839cdcf033e6c9201fadca95224c7303a29
This commit is contained in:
Samuel Merritt 2013-03-04 17:05:43 -08:00
parent f6d1fa1c15
commit ebcd60f7d9
8 changed files with 589 additions and 364 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101
bin/swift-ring-builder
View File

@ -20,7 +20,7 @@ from errno import EEXIST
from itertools import islice, izip from itertools import islice, izip
from os import mkdir from os import mkdir
from os.path import basename, abspath, dirname, exists, join as pathjoin from os.path import basename, abspath, dirname, exists, join as pathjoin
from sys import argv, exit from sys import argv, exit, stderr
from textwrap import wrap from textwrap import wrap
from time import time from time import time
@ -40,9 +40,11 @@ def format_device(dev):
Format a device for display. Format a device for display.
""" """
if ':' in dev['ip']: if ':' in dev['ip']:
return 'd%(id)sz%(zone)s-[%(ip)s]:%(port)s/%(device)s_"%(meta)s"' % dev return ('d%(id)sr%(region)z%(zone)s-'
'[%(ip)s]:%(port)s/%(device)s_"%(meta)s"') % dev
else: else:
return 'd%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_"%(meta)s"' % dev return ('d%(id)sr%(region)z%(zone)s-'
'%(ip)s:%(port)s/%(device)s_"%(meta)s"') % dev
class Commands: class Commands:
@ -80,19 +82,27 @@ swift-ring-builder <builder_file>
Shows information about the ring and the devices within. Shows information about the ring and the devices within.
""" """
print '%s, build version %d' % (argv[1], builder.version) print '%s, build version %d' % (argv[1], builder.version)
regions = 0
zones = 0 zones = 0
balance = 0 balance = 0
dev_count = 0
if builder.devs: if builder.devs:
zones = len(set(d['zone'] for d in builder.devs if d is not None)) regions = len(set(d['region'] for d in builder.devs
if d is not None))
zones = len(set((d['region'], d['zone']) for d in builder.devs
if d is not None))
dev_count = len([d for d in builder.devs
if d is not None])
balance = builder.get_balance() balance = builder.get_balance()
print '%d partitions, %.6f replicas, %d zones, %d devices, %.02f ' \ print '%d partitions, %.6f replicas, %d regions, %d zones, ' \
'balance' % (builder.parts, builder.replicas, zones, '%d devices, %.02f balance' % (builder.parts, builder.replicas,
len([d for d in builder.devs if d]), balance) regions, zones, dev_count,
balance)
print 'The minimum number of hours before a partition can be ' \ print 'The minimum number of hours before a partition can be ' \
'reassigned is %s' % builder.min_part_hours 'reassigned is %s' % builder.min_part_hours
if builder.devs: if builder.devs:
print 'Devices: id zone ip address port name ' \ print 'Devices: id region zone ip address port' \
'weight partitions balance meta' ' name weight partitions balance meta'
weighted_parts = builder.parts * builder.replicas / \ weighted_parts = builder.parts * builder.replicas / \
sum(d['weight'] for d in builder.devs if d is not None) sum(d['weight'] for d in builder.devs if d is not None)
for dev in builder.devs: for dev in builder.devs:
@ -106,10 +116,11 @@ swift-ring-builder <builder_file>
else: else:
balance = 100.0 * dev['parts'] / \ balance = 100.0 * dev['parts'] / \
(dev['weight'] * weighted_parts) - 100.0 (dev['weight'] * weighted_parts) - 100.0
print ' %5d %5d %15s %5d %9s %6.02f %10s %7.02f %s' % \ print(' %5d %5d %5d %15s %5d %9s %6.02f %10s'
(dev['id'], dev['zone'], dev['ip'], dev['port'], '%7.02f %s' %
dev['device'], dev['weight'], dev['parts'], balance, (dev['id'], dev['region'], dev['zone'], dev['ip'],
dev['meta']) dev['port'], dev['device'], dev['weight'], dev['parts'],
balance, dev['meta']))
exit(EXIT_SUCCESS) exit(EXIT_SUCCESS)
def search(): def search():
@ -126,7 +137,7 @@ swift-ring-builder <builder_file> search <search-value>
if not devs: if not devs:
print 'No matching devices found' print 'No matching devices found'
exit(EXIT_ERROR) exit(EXIT_ERROR)
print 'Devices: id zone ip address port name ' \ print 'Devices: id region zone ip address port name ' \
'weight partitions balance meta' 'weight partitions balance meta'
weighted_parts = builder.parts * builder.replicas / \ weighted_parts = builder.parts * builder.replicas / \
sum(d['weight'] for d in builder.devs if d is not None) sum(d['weight'] for d in builder.devs if d is not None)
@ -139,10 +150,10 @@ swift-ring-builder <builder_file> search <search-value>
else: else:
balance = 100.0 * dev['parts'] / \ balance = 100.0 * dev['parts'] / \
(dev['weight'] * weighted_parts) - 100.0 (dev['weight'] * weighted_parts) - 100.0
print ' %5d %5d %15s %5d %9s %6.02f %10s %7.02f %s' % \ print(' %5d %5d %5d %15s %5d %9s %6.02f %10s %7.02f %s' %
(dev['id'], dev['zone'], dev['ip'], dev['port'], (dev['id'], dev['region'], dev['zone'], dev['ip'],
dev['device'], dev['weight'], dev['parts'], balance, dev['port'], dev['device'], dev['weight'], dev['parts'],
dev['meta']) balance, dev['meta']))
exit(EXIT_SUCCESS) exit(EXIT_SUCCESS)
def list_parts(): def list_parts():
@ -182,8 +193,8 @@ swift-ring-builder <builder_file> list_parts <search-value> [<search-value>] ..
def add(): def add():
""" """
swift-ring-builder <builder_file> add swift-ring-builder <builder_file> add
z<zone>-<ip>:<port>/<device_name>_<meta> <weight> [r<region>]z<zone>-<ip>:<port>/<device_name>_<meta> <weight>
[z<zone>-<ip>:<port>/<device_name>_<meta> <weight>] ... [[r<region>]z<zone>-<ip>:<port>/<device_name>_<meta> <weight>] ...
Adds devices to the ring with the given information. No partitions will be Adds devices to the ring with the given information. No partitions will be
assigned to the new device until after running 'rebalance'. This is so you assigned to the new device until after running 'rebalance'. This is so you
@ -196,14 +207,26 @@ swift-ring-builder <builder_file> add
devs_and_weights = izip(islice(argv, 3, len(argv), 2), devs_and_weights = izip(islice(argv, 3, len(argv), 2),
islice(argv, 4, len(argv), 2)) islice(argv, 4, len(argv), 2))
for devstr, weightstr in devs_and_weights: for devstr, weightstr in devs_and_weights:
if not devstr.startswith('z'): region = 1
rest = devstr
if devstr.startswith('r'):
i = 1
while i < len(devstr) and devstr[i].isdigit():
i += 1
region = int(devstr[1:i])
rest = devstr[i:]
else:
stderr.write("WARNING: No region specified for %s. "
"Defaulting to region 1.\n" % devstr)
if not rest.startswith('z'):
print 'Invalid add value: %s' % devstr print 'Invalid add value: %s' % devstr
exit(EXIT_ERROR) exit(EXIT_ERROR)
i = 1 i = 1
while i < len(devstr) and devstr[i].isdigit(): while i < len(rest) and rest[i].isdigit():
i += 1 i += 1
zone = int(devstr[1:i]) zone = int(rest[1:i])
rest = devstr[i:] rest = rest[i:]
if not rest.startswith('-'): if not rest.startswith('-'):
print 'Invalid add value: %s' % devstr print 'Invalid add value: %s' % devstr
@ -269,17 +292,21 @@ swift-ring-builder <builder_file> add
print "The on-disk ring builder is unchanged.\n" print "The on-disk ring builder is unchanged.\n"
exit(EXIT_ERROR) exit(EXIT_ERROR)
builder.add_dev({'zone': zone, 'ip': ip, 'port': port, builder.add_dev({'region': region, 'zone': zone, 'ip': ip,
'device': device_name, 'weight': weight, 'port': port, 'device': device_name,
'meta': meta}) 'weight': weight, 'meta': meta})
new_dev = builder.search_devs( new_dev = builder.search_devs(
'z%s-%s:%s/%s' % (zone, ip, port, device_name))[0]['id'] 'r%dz%d-%s:%s/%s' %
(region, zone, ip, port, device_name))[0]['id']
if ':' in ip: if ':' in ip:
print 'Device z%s-[%s]:%s/%s_"%s" with %s weight got id %s' % \ print(
(zone, ip, port, device_name, meta, weight, new_dev) 'Device r%dz%d-[%s]:%s/%s_"%s" with %s weight got id %s' %
(region, zone, ip, port,
device_name, meta, weight, new_dev))
else: else:
print 'Device z%s-%s:%s/%s_"%s" with %s weight got id %s' % \ print('Device r%dz%d-%s:%s/%s_"%s" with %s weight got id %s' %
(zone, ip, port, device_name, meta, weight, new_dev) (region, zone, ip, port,
device_name, meta, weight, new_dev))
pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2) pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2)
exit(EXIT_SUCCESS) exit(EXIT_SUCCESS)
@ -442,8 +469,8 @@ swift-ring-builder <builder_file> remove <search-value> [search-value ...]
if len(devs) > 1: if len(devs) > 1:
print 'Matched more than one device:' print 'Matched more than one device:'
for dev in devs: for dev in devs:
print ' d%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_' \ print ' d%(id)sr%(region)z%(zone)s-%(ip)s:%(port)s/' \
'"%(meta)s"' % dev '%(device)s_"%(meta)s"' % dev
if raw_input('Are you sure you want to remove these %s ' if raw_input('Are you sure you want to remove these %s '
'devices? (y/N) ' % len(devs)) != 'y': 'devices? (y/N) ' % len(devs)) != 'y':
print 'Aborting device removals' print 'Aborting device removals'
@ -465,9 +492,9 @@ swift-ring-builder <builder_file> remove <search-value> [search-value ...]
print '-' * 79 print '-' * 79
exit(EXIT_ERROR) exit(EXIT_ERROR)
print 'd%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_' \ print 'd%(id)sr%(region)z%(zone)s-%(ip)s:%(port)s/' \
'"%(meta)s" marked for removal and will be removed' \ '%(device)s_"%(meta)s" marked for removal and will ' \
' next rebalance.' % dev 'be removed next rebalance.' % dev
pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2) pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2)
exit(EXIT_SUCCESS) exit(EXIT_SUCCESS)

56
doc/source/overview_ring.rst
View File

@ -98,9 +98,9 @@ device and device['weight']]``
Partition Assignment List Partition Assignment List
************************* *************************
This is a list of array('I') of devices ids. The outermost list contains an This is a list of array('H') of devices ids. The outermost list contains an
array('I') for each replica. Each array('I') has a length equal to the array('H') for each replica. Each array('H') has a length equal to the
partition count for the ring. Each integer in the array('I') is an index into partition count for the ring. Each integer in the array('H') is an index into
the above list of devices. The partition list is known internally to the Ring the above list of devices. The partition list is known internally to the Ring
class as _replica2part2dev_id. class as _replica2part2dev_id.
@ -108,9 +108,29 @@ So, to create a list of device dictionaries assigned to a partition, the Python
code would look like: ``devices = [self.devs[part2dev_id[partition]] for code would look like: ``devices = [self.devs[part2dev_id[partition]] for
part2dev_id in self._replica2part2dev_id]`` part2dev_id in self._replica2part2dev_id]``
array('I') is used for memory conservation as there may be millions of That code is a little simplistic, as it does not account for the
removal of duplicate devices. If a ring has more replicas than
devices, then a partition will have more than one replica on one
device; that's simply the pigeonhole principle at work.
array('H') is used for memory conservation as there may be millions of
partitions. partitions.
*******************
Fractional Replicas
*******************
A ring is not restricted to having an integer number of replicas. In order to
support the gradual changing of replica counts, the ring is able to have a real
number of replicas.
When the number of replicas is not an integer, then the last element of
_replica2part2dev_id will have a length that is less than the partition count
for the ring. This means that some partitions will have more replicas than
others. For example, if a ring has 3.25 replicas, then 25% of its partitions
will have four replicas, while the remaining 75% will have just three.
********************* *********************
Partition Shift Value Partition Shift Value
********************* *********************
@ -123,25 +143,37 @@ in this process. For example, to compute the partition for the path
unpack_from('>I', md5('/account/container/object').digest())[0] >> unpack_from('>I', md5('/account/container/object').digest())[0] >>
self._part_shift`` self._part_shift``
For a ring generated with part_power P, the partition shift value is
32 - P.
----------------- -----------------
Building the Ring Building the Ring
----------------- -----------------
The initial building of the ring first calculates the number of partitions that The initial building of the ring first calculates the number of partitions that
should ideally be assigned to each device based the device's weight. For should ideally be assigned to each device based the device's weight. For
example, if the partition power of 20 the ring will have 1,048,576 partitions. example, given a partition power of 20, the ring will have 1,048,576 partitions.
If there are 1,000 devices of equal weight they will each desire 1,048.576 If there are 1,000 devices of equal weight they will each desire 1,048.576
partitions. The devices are then sorted by the number of partitions they desire partitions. The devices are then sorted by the number of partitions they desire
and kept in order throughout the initialization process. and kept in order throughout the initialization process.
Then, the ring builder assigns each replica of each partition to the device Note: each device is also assigned a random tiebreaker value that is used when
that desires the most partitions at that point while keeping it as far away as two devices desire the same number of partitions. This tiebreaker is not stored
on disk anywhere, and so two different rings created with the same parameters
will have different partition assignments. For repeatable partition assignments,
``RingBuilder.rebalance()`` takes an optional seed value that will be used to
seed Python's pseudo-random number generator.
Then, the ring builder assigns each replica of each partition to the device that
desires the most partitions at that point while keeping it as far away as
possible from other replicas. The ring builder prefers to assign a replica to a possible from other replicas. The ring builder prefers to assign a replica to a
device in a zone that has no replicas already; should there be no such zone device in a regions that has no replicas already; should there be no such region
available, the ring builder will try to find a device on a different server; available, the ring builder will try to find a device in a different zone; if
failing that, it will just look for a device that has no replicas; finally, if not possible, it will look on a different server; failing that, it will just
all other options are exhausted, the ring builder will assign the replica to look for a device that has no replicas; finally, if all other options are
the device that has the fewest replicas already assigned. exhausted, the ring builder will assign the replica to the device that has the
fewest replicas already assigned. Note that assignment of multiple replicas to
one device will only happen if the ring has fewer devices than it has replicas.
When building a new ring based on an old ring, the desired number of partitions When building a new ring based on an old ring, the desired number of partitions
each device wants is recalculated. Next the partitions to be reassigned are each device wants is recalculated. Next the partitions to be reassigned are

41
swift/common/ring/builder.py
View File

@ -129,6 +129,11 @@ class RingBuilder(object):
self._remove_devs = builder['_remove_devs'] self._remove_devs = builder['_remove_devs']
self._ring = None self._ring = None
# Old builders may not have a region defined for their devices, in
# which case we default it to 1.
for dev in self._iter_devs():
dev.setdefault("region", 1)
def to_dict(self): def to_dict(self):
""" """
Returns a dict that can be used later with copy_from to Returns a dict that can be used later with copy_from to
@ -224,9 +229,10 @@ class RingBuilder(object):
weight a float of the relative weight of this device as compared to weight a float of the relative weight of this device as compared to
others; this indicates how many partitions the builder will try others; this indicates how many partitions the builder will try
to assign to this device to assign to this device
region integer indicating which region the device is in
zone integer indicating which zone the device is in; a given zone integer indicating which zone the device is in; a given
partition will not be assigned to multiple devices within the partition will not be assigned to multiple devices within the
same zone same (region, zone) pair if there is any alternative
ip the ip address of the device ip the ip address of the device
port the tcp port of the device port the tcp port of the device
device the device's name on disk (sdb1, for example) device the device's name on disk (sdb1, for example)
@ -413,11 +419,11 @@ class RingBuilder(object):
def get_balance(self): def get_balance(self):
""" """
Get the balance of the ring. The balance value is the highest Get the balance of the ring. The balance value is the highest
percentage off the desired amount of partitions a given device wants. percentage off the desired amount of partitions a given device
For instance, if the "worst" device wants (based on its relative weight wants. For instance, if the "worst" device wants (based on its
and its zone's relative weight) 123 partitions and it has 124 weight relative to the sum of all the devices' weights) 123
partitions, the balance value would be 0.83 (1 extra / 123 wanted * 100 partitions and it has 124 partitions, the balance value would
for percentage). be 0.83 (1 extra / 123 wanted * 100 for percentage).
:returns: balance of the ring :returns: balance of the ring
""" """
@ -712,10 +718,10 @@ class RingBuilder(object):
they still want and kept in that order throughout the process. The they still want and kept in that order throughout the process. The
gathered partitions are iterated through, assigning them to devices gathered partitions are iterated through, assigning them to devices
according to the "most wanted" while keeping the replicas as "far according to the "most wanted" while keeping the replicas as "far
apart" as possible. Two different zones are considered the apart" as possible. Two different regions are considered the
farthest-apart things, followed by different ip/port pairs within a farthest-apart things, followed by zones, then different ip/port pairs
zone; the least-far-apart things are different devices with the same within a zone; the least-far-apart things are different devices with
ip/port pair in the same zone. the same ip/port pair in the same zone.
If you want more replicas than devices, you won't get all your If you want more replicas than devices, you won't get all your
replicas. replicas.
@ -761,8 +767,8 @@ class RingBuilder(object):
depth += 1 depth += 1
for part, replace_replicas in reassign_parts: for part, replace_replicas in reassign_parts:
# Gather up what other tiers (zones, ip_ports, and devices) the # Gather up what other tiers (regions, zones, ip/ports, and
# replicas not-to-be-moved are in for this part. # devices) the replicas not-to-be-moved are in for this part.
other_replicas = defaultdict(int) other_replicas = defaultdict(int)
unique_tiers_by_tier_len = defaultdict(set) unique_tiers_by_tier_len = defaultdict(set)
for replica in self._replicas_for_part(part): for replica in self._replicas_for_part(part):
@ -977,13 +983,14 @@ class RingBuilder(object):
""" """
The <search-value> can be of the form:: The <search-value> can be of the form::
d<device_id>z<zone>-<ip>:<port>/<device_name>_<meta> d<device_id>r<region>z<zone>-<ip>:<port>/<device_name>_<meta>
Any part is optional, but you must include at least one part. Any part is optional, but you must include at least one part.
Examples:: Examples::
d74 Matches the device id 74 d74 Matches the device id 74
r4 Matches devices in region 4
z1 Matches devices in zone 1 z1 Matches devices in zone 1
z1-1.2.3.4 Matches devices in zone 1 with the ip 1.2.3.4 z1-1.2.3.4 Matches devices in zone 1 with the ip 1.2.3.4
1.2.3.4 Matches devices in any zone with the ip 1.2.3.4 1.2.3.4 Matches devices in any zone with the ip 1.2.3.4
@ -997,7 +1004,7 @@ class RingBuilder(object):
Most specific example:: Most specific example::
d74z1-1.2.3.4:5678/sdb1_"snet: 5.6.7.8" d74r4z1-1.2.3.4:5678/sdb1_"snet: 5.6.7.8"
Nerd explanation: Nerd explanation:
@ -1012,6 +1019,12 @@ class RingBuilder(object):
i += 1 i += 1
match.append(('id', int(search_value[1:i]))) match.append(('id', int(search_value[1:i])))
search_value = search_value[i:] search_value = search_value[i:]
if search_value.startswith('r'):
i = 1
while i < len(search_value) and search_value[i].isdigit():
i += 1
match.append(('region', int(search_value[1:i])))
search_value = search_value[i:]
if search_value.startswith('z'): if search_value.startswith('z'):
i = 1 i = 1
while i < len(search_value) and search_value[i].isdigit(): while i < len(search_value) and search_value[i].isdigit():

49
swift/common/ring/ring.py
View File

@ -37,6 +37,10 @@ class RingData(object):
self._replica2part2dev_id = replica2part2dev_id self._replica2part2dev_id = replica2part2dev_id
self._part_shift = part_shift self._part_shift = part_shift
for dev in self.devs:
if dev is not None:
dev.setdefault("region", 1)
@classmethod @classmethod
def deserialize_v1(cls, gz_file): def deserialize_v1(cls, gz_file):
json_len, = struct.unpack('!I', gz_file.read(4)) json_len, = struct.unpack('!I', gz_file.read(4))
@ -266,39 +270,52 @@ class Ring(object):
""" """
if time() > self._rtime: if time() > self._rtime:
self._reload() self._reload()
used = set(part2dev_id[part] primary_nodes = self._get_part_nodes(part)
for part2dev_id in self._replica2part2dev_id
if len(part2dev_id) > part) used = set(d['id'] for d in primary_nodes)
same_zones = set(self._devs[part2dev_id[part]]['zone'] same_regions = set(d['region'] for d in primary_nodes)
for part2dev_id in self._replica2part2dev_id same_zones = set((d['region'], d['zone']) for d in primary_nodes)
if len(part2dev_id) > part)
parts = len(self._replica2part2dev_id[0]) parts = len(self._replica2part2dev_id[0])
start = struct.unpack_from( start = struct.unpack_from(
'>I', md5(str(part)).digest())[0] >> self._part_shift '>I', md5(str(part)).digest())[0] >> self._part_shift
inc = int(parts / 65536) or 1 inc = int(parts / 65536) or 1
# Two loops for execution speed, second loop doesn't need the zone # Multiple loops for execution speed; the checks and bookkeeping get
# check. # simpler as you go along
for handoff_part in chain(xrange(start, parts, inc), for handoff_part in chain(xrange(start, parts, inc),
xrange(inc - ((parts - start) % inc), xrange(inc - ((parts - start) % inc),
start, inc)): start, inc)):
for part2dev_id in self._replica2part2dev_id: for part2dev_id in self._replica2part2dev_id:
try: if handoff_part < len(part2dev_id):
dev_id = part2dev_id[handoff_part] dev_id = part2dev_id[handoff_part]
dev = self._devs[dev_id] dev = self._devs[dev_id]
if dev_id not in used and dev['zone'] not in same_zones: region = dev['region']
zone = (dev['region'], dev['zone'])
if dev_id not in used and region not in same_regions:
yield dev yield dev
used.add(dev_id) used.add(dev_id)
same_zones.add(dev['zone']) same_regions.add(region)
except IndexError: # Happens with partial replicas same_zones.add(zone)
pass
for handoff_part in chain(xrange(start, parts, inc), for handoff_part in chain(xrange(start, parts, inc),
xrange(inc - ((parts - start) % inc), xrange(inc - ((parts - start) % inc),
start, inc)): start, inc)):
for part2dev_id in self._replica2part2dev_id: for part2dev_id in self._replica2part2dev_id:
try: if handoff_part < len(part2dev_id):
dev_id = part2dev_id[handoff_part]
dev = self._devs[dev_id]
zone = (dev['region'], dev['zone'])
if dev_id not in used and zone not in same_zones:
yield dev
used.add(dev_id)
same_zones.add(zone)
for handoff_part in chain(xrange(start, parts, inc),
xrange(inc - ((parts - start) % inc),
start, inc)):
for part2dev_id in self._replica2part2dev_id:
if handoff_part < len(part2dev_id):
dev_id = part2dev_id[handoff_part] dev_id = part2dev_id[handoff_part]
if dev_id not in used: if dev_id not in used:
yield self._devs[dev_id] yield self._devs[dev_id]
used.add(dev_id) used.add(dev_id)
except IndexError: # Happens with partial replicas
pass

106
swift/common/ring/utils.py
View File

@ -8,13 +8,15 @@ def tiers_for_dev(dev):
:returns: tuple of tiers :returns: tuple of tiers
""" """
t1 = dev['zone'] t1 = dev['region']
t2 = "{ip}:{port}".format(ip=dev.get('ip'), port=dev.get('port')) t2 = dev['zone']
t3 = dev['id'] t3 = "{ip}:{port}".format(ip=dev.get('ip'), port=dev.get('port'))
t4 = dev['id']
return ((t1,), return ((t1,),
(t1, t2), (t1, t2),
(t1, t2, t3)) (t1, t2, t3),
(t1, t2, t3, t4))
def build_tier_tree(devices): def build_tier_tree(devices):
@ -27,52 +29,72 @@ def build_tier_tree(devices):
Example: Example:
zone 1 -+---- 192.168.1.1:6000 -+---- device id 0 region 1 -+---- zone 1 -+---- 192.168.101.1:6000 -+---- device id 0
| | | | |
| +---- device id 1 | | +---- device id 1
| | | | |
| +---- device id 2 | | +---- device id 2
| | |
+---- 192.168.1.2:6000 -+---- device id 3 | +---- 192.168.101.2:6000 -+---- device id 3
| | |
+---- device id 4 | +---- device id 4
| | |
+---- device id 5 | +---- device id 5
|
+---- zone 2 -+---- 192.168.102.1:6000 -+---- device id 6
| |
| +---- device id 7
| |
| +---- device id 8
|
+---- 192.168.102.2:6000 -+---- device id 9
|
+---- device id 10
zone 2 -+---- 192.168.2.1:6000 -+---- device id 6 region 2 -+---- zone 1 -+---- 192.168.201.1:6000 -+---- device id 12
| | | |
| +---- device id 7 | +---- device id 13
| | | |
| +---- device id 8 | +---- device id 14
| |
+---- 192.168.2.2:6000 -+---- device id 9 +---- 192.168.201.2:6000 -+---- device id 15
| |
+---- device id 10 +---- device id 16
| |
+---- device id 11 +---- device id 17
The tier tree would look like: The tier tree would look like:
{ {
(): [(1,), (2,)], (): [(1,), (2,)],
(1,): [(1, 192.168.1.1:6000), (1,): [(1, 1), (1, 2)],
(1, 192.168.1.2:6000)], (2,): [(2, 1)],
(2,): [(2, 192.168.2.1:6000),
(2, 192.168.2.2:6000)],
(1, 192.168.1.1:6000): [(1, 192.168.1.1:6000, 0), (1, 1): [(1, 1, 192.168.101.1:6000),
(1, 192.168.1.1:6000, 1), (1, 1, 192.168.101.2:6000)],
(1, 192.168.1.1:6000, 2)], (1, 2): [(1, 2, 192.168.102.1:6000),
(1, 192.168.1.2:6000): [(1, 192.168.1.2:6000, 3), (1, 2, 192.168.102.2:6000)],
(1, 192.168.1.2:6000, 4), (2, 1): [(2, 1, 192.168.201.1:6000),
(1, 192.168.1.2:6000, 5)], (2, 1, 192.168.201.2:6000)],
(2, 192.168.2.1:6000): [(2, 192.168.2.1:6000, 6),
(2, 192.168.2.1:6000, 7), (1, 1, 192.168.101.1:6000): [(1, 1, 192.168.101.1:6000, 0),
(2, 192.168.2.1:6000, 8)], (1, 1, 192.168.101.1:6000, 1),
(2, 192.168.2.2:6000): [(2, 192.168.2.2:6000, 9), (1, 1, 192.168.101.1:6000, 2)],
(2, 192.168.2.2:6000, 10), (1, 1, 192.168.101.2:6000): [(1, 1, 192.168.101.2:6000, 3),
(2, 192.168.2.2:6000, 11)], (1, 1, 192.168.101.2:6000, 4),
(1, 1, 192.168.101.2:6000, 5)],
(1, 2, 192.168.102.1:6000): [(1, 2, 192.168.102.1:6000, 6),
(1, 2, 192.168.102.1:6000, 7),
(1, 2, 192.168.102.1:6000, 8)],
(1, 2, 192.168.102.2:6000): [(1, 2, 192.168.102.2:6000, 9),
(1, 2, 192.168.102.2:6000, 10)],
(2, 1, 192.168.201.1:6000): [(2, 1, 192.168.201.1:6000, 12),
(2, 1, 192.168.201.1:6000, 13),
(2, 1, 192.168.201.1:6000, 14)],
(2, 1, 192.168.201.2:6000): [(2, 1, 192.168.201.2:6000, 15),
(2, 1, 192.168.201.2:6000, 16),
(2, 1, 192.168.201.2:6000, 17)],
} }
:devices: device dicts from which to generate the tree :devices: device dicts from which to generate the tree

433
test/unit/common/ring/test_builder.py
View File

@ -49,14 +49,14 @@ class TestRingBuilder(unittest.TestCase):
def test_get_ring(self): def test_get_ring(self):
rb = ring.RingBuilder(8, 3, 1) rb = ring.RingBuilder(8, 3, 1)
rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1,
'port': 10000, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'})
rb.add_dev({'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1,
'port': 10001, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'})
rb.add_dev({'id': 2, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 2, 'region': 0, 'zone': 2, 'weight': 1,
'port': 10002, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10002, 'device': 'sda1'})
rb.add_dev({'id': 3, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 3, 'region': 0, 'zone': 1, 'weight': 1,
'port': 10004, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10004, 'device': 'sda1'})
rb.remove_dev(1) rb.remove_dev(1)
rb.rebalance() rb.rebalance()
r = rb.get_ring() r = rb.get_ring()
@ -75,7 +75,7 @@ class TestRingBuilder(unittest.TestCase):
for n in range(3): for n in range(3):
rb = ring.RingBuilder(8, 3, 1) rb = ring.RingBuilder(8, 3, 1)
for idx, (zone, port) in enumerate(devs): for idx, (zone, port) in enumerate(devs):
rb.add_dev({'id': idx, 'zone': zone, 'weight': 1, rb.add_dev({'id': idx, 'region': 0, 'zone': zone, 'weight': 1,
'ip': '127.0.0.1', 'port': port, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': port, 'device': 'sda1'})
ring_builders.append(rb) ring_builders.append(rb)
@ -110,28 +110,30 @@ class TestRingBuilder(unittest.TestCase):
def test_add_dev(self): def test_add_dev(self):
rb = ring.RingBuilder(8, 3, 1) rb = ring.RingBuilder(8, 3, 1)
dev = \ dev = {'id': 0, 'region': 0, 'zone': 0, 'weight': 1,
{'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000} 'ip': '127.0.0.1', 'port': 10000}
rb.add_dev(dev) rb.add_dev(dev)
self.assertRaises(exceptions.DuplicateDeviceError, rb.add_dev, dev) self.assertRaises(exceptions.DuplicateDeviceError, rb.add_dev, dev)
rb = ring.RingBuilder(8, 3, 1) rb = ring.RingBuilder(8, 3, 1)
#test add new dev with no id #test add new dev with no id
rb.add_dev({'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 6000}) rb.add_dev({'zone': 0, 'region': 1, 'weight': 1,
'ip': '127.0.0.1', 'port': 6000})
self.assertEquals(rb.devs[0]['id'], 0) self.assertEquals(rb.devs[0]['id'], 0)
#test add another dev with no id #test add another dev with no id
rb.add_dev({'zone': 3, 'weight': 1, 'ip': '127.0.0.1', 'port': 6000}) rb.add_dev({'zone': 3, 'region': 2, 'weight': 1,
'ip': '127.0.0.1', 'port': 6000})
self.assertEquals(rb.devs[1]['id'], 1) self.assertEquals(rb.devs[1]['id'], 1)
def test_set_dev_weight(self): def test_set_dev_weight(self):
rb = ring.RingBuilder(8, 3, 1) rb = ring.RingBuilder(8, 3, 1)
rb.add_dev({'id': 0, 'zone': 0, 'weight': 0.5, 'ip': '127.0.0.1', rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 0.5,
'port': 10000, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'})
rb.add_dev({'id': 1, 'zone': 0, 'weight': 0.5, 'ip': '127.0.0.1', rb.add_dev({'id': 1, 'region': 0, 'zone': 0, 'weight': 0.5,
'port': 10001, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'})
rb.add_dev({'id': 2, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 2, 'region': 0, 'zone': 1, 'weight': 1,
'port': 10002, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10002, 'device': 'sda1'})
rb.add_dev({'id': 3, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 3, 'region': 0, 'zone': 2, 'weight': 1,
'port': 10003, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10003, 'device': 'sda1'})
rb.rebalance() rb.rebalance()
r = rb.get_ring() r = rb.get_ring()
counts = {} counts = {}
@ -152,14 +154,14 @@ class TestRingBuilder(unittest.TestCase):
def test_remove_dev(self): def test_remove_dev(self):
rb = ring.RingBuilder(8, 3, 1) rb = ring.RingBuilder(8, 3, 1)
rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1,
'port': 10000, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'})
rb.add_dev({'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1,
'port': 10001, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'})
rb.add_dev({'id': 2, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 2, 'region': 0, 'zone': 2, 'weight': 1,
'port': 10002, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10002, 'device': 'sda1'})
rb.add_dev({'id': 3, 'zone': 3, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 3, 'region': 0, 'zone': 3, 'weight': 1,
'port': 10003, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10003, 'device': 'sda1'})
rb.rebalance() rb.rebalance()
r = rb.get_ring() r = rb.get_ring()
counts = {} counts = {}
@ -180,17 +182,17 @@ class TestRingBuilder(unittest.TestCase):
def test_remove_a_lot(self): def test_remove_a_lot(self):
rb = ring.RingBuilder(3, 3, 1) rb = ring.RingBuilder(3, 3, 1)
rb.add_dev({'id': 0, 'device': 'd0', 'ip': '10.0.0.1', rb.add_dev({'id': 0, 'device': 'd0', 'ip': '10.0.0.1',
'port': 6002, 'weight': 1000.0, 'zone': 1}) 'port': 6002, 'weight': 1000.0, 'region': 0, 'zone': 1})
rb.add_dev({'id': 1, 'device': 'd1', 'ip': '10.0.0.2', rb.add_dev({'id': 1, 'device': 'd1', 'ip': '10.0.0.2',
'port': 6002, 'weight': 1000.0, 'zone': 2}) 'port': 6002, 'weight': 1000.0, 'region': 0, 'zone': 2})
rb.add_dev({'id': 2, 'device': 'd2', 'ip': '10.0.0.3', rb.add_dev({'id': 2, 'device': 'd2', 'ip': '10.0.0.3',
'port': 6002, 'weight': 1000.0, 'zone': 3}) 'port': 6002, 'weight': 1000.0, 'region': 0, 'zone': 3})
rb.add_dev({'id': 3, 'device': 'd3', 'ip': '10.0.0.1', rb.add_dev({'id': 3, 'device': 'd3', 'ip': '10.0.0.1',
'port': 6002, 'weight': 1000.0, 'zone': 1}) 'port': 6002, 'weight': 1000.0, 'region': 0, 'zone': 1})
rb.add_dev({'id': 4, 'device': 'd4', 'ip': '10.0.0.2', rb.add_dev({'id': 4, 'device': 'd4', 'ip': '10.0.0.2',
'port': 6002, 'weight': 1000.0, 'zone': 2}) 'port': 6002, 'weight': 1000.0, 'region': 0, 'zone': 2})
rb.add_dev({'id': 5, 'device': 'd5', 'ip': '10.0.0.3', rb.add_dev({'id': 5, 'device': 'd5', 'ip': '10.0.0.3',
'port': 6002, 'weight': 1000.0, 'zone': 3}) 'port': 6002, 'weight': 1000.0, 'region': 0, 'zone': 3})
rb.rebalance() rb.rebalance()
rb.validate() rb.validate()
@ -216,15 +218,15 @@ class TestRingBuilder(unittest.TestCase):
def _shuffled_gather_helper(self): def _shuffled_gather_helper(self):
rb = ring.RingBuilder(8, 3, 1) rb = ring.RingBuilder(8, 3, 1)
rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1,
'port': 10000, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'})
rb.add_dev({'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1,
'port': 10001, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'})
rb.add_dev({'id': 2, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 2, 'region': 0, 'zone': 2, 'weight': 1,
'port': 10002, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10002, 'device': 'sda1'})
rb.rebalance() rb.rebalance()
rb.add_dev({'id': 3, 'zone': 3, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 3, 'region': 0, 'zone': 3, 'weight': 1,
'port': 10003, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10003, 'device': 'sda1'})
rb.pretend_min_part_hours_passed() rb.pretend_min_part_hours_passed()
parts = rb._gather_reassign_parts() parts = rb._gather_reassign_parts()
max_run = 0 max_run = 0
@ -243,34 +245,64 @@ class TestRingBuilder(unittest.TestCase):
return max_run > len(parts) / 2 return max_run > len(parts) / 2
def test_multitier_partial(self): def test_multitier_partial(self):
# Multitier test, zones full, nodes not full # Multitier test, nothing full
rb = ring.RingBuilder(8, 6, 1) rb = ring.RingBuilder(8, 3, 1)
rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1,
'port': 10000, 'device': 'sda'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda'})
rb.add_dev({'id': 1, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 1, 'region': 1, 'zone': 1, 'weight': 1,
'port': 10000, 'device': 'sdb'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdb'})
rb.add_dev({'id': 2, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 2, 'region': 2, 'zone': 2, 'weight': 1,
'port': 10000, 'device': 'sdc'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdc'})
rb.add_dev({'id': 3, 'region': 3, 'zone': 3, 'weight': 1,
rb.add_dev({'id': 3, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdd'})
'port': 10001, 'device': 'sdd'})
rb.add_dev({'id': 4, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1',
'port': 10001, 'device': 'sde'})
rb.add_dev({'id': 5, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1',
'port': 10001, 'device': 'sdf'})
rb.add_dev({'id': 6, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1',
'port': 10002, 'device': 'sdg'})
rb.add_dev({'id': 7, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1',
'port': 10002, 'device': 'sdh'})
rb.add_dev({'id': 8, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1',
'port': 10002, 'device': 'sdi'})
rb.rebalance() rb.rebalance()
rb.validate() rb.validate()
for part in xrange(rb.parts): for part in xrange(rb.parts):
counts = defaultdict(lambda: defaultdict(lambda: 0)) counts = defaultdict(lambda: defaultdict(int))
for replica in xrange(rb.replicas):
dev = rb.devs[rb._replica2part2dev[replica][part]]
counts['region'][dev['region']] += 1
counts['zone'][dev['zone']] += 1
if any(c > 1 for c in counts['region'].values()):
raise AssertionError(
"Partition %d not evenly region-distributed (got %r)" %
(part, counts['region']))
if any(c > 1 for c in counts['zone'].values()):
raise AssertionError(
"Partition %d not evenly zone-distributed (got %r)" %
(part, counts['zone']))
# Multitier test, zones full, nodes not full
rb = ring.RingBuilder(8, 6, 1)
rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1,
'ip': '127.0.0.1', 'port': 10000, 'device': 'sda'})
rb.add_dev({'id': 1, 'region': 0, 'zone': 0, 'weight': 1,
'ip': '127.0.0.1', 'port': 10000, 'device': 'sdb'})
rb.add_dev({'id': 2, 'region': 0, 'zone': 0, 'weight': 1,
'ip': '127.0.0.1', 'port': 10000, 'device': 'sdc'})
rb.add_dev({'id': 3, 'region': 0, 'zone': 1, 'weight': 1,
'ip': '127.0.0.1', 'port': 10001, 'device': 'sdd'})
rb.add_dev({'id': 4, 'region': 0, 'zone': 1, 'weight': 1,
'ip': '127.0.0.1', 'port': 10001, 'device': 'sde'})
rb.add_dev({'id': 5, 'region': 0, 'zone': 1, 'weight': 1,
'ip': '127.0.0.1', 'port': 10001, 'device': 'sdf'})
rb.add_dev({'id': 6, 'region': 0, 'zone': 2, 'weight': 1,
'ip': '127.0.0.1', 'port': 10002, 'device': 'sdg'})
rb.add_dev({'id': 7, 'region': 0, 'zone': 2, 'weight': 1,
'ip': '127.0.0.1', 'port': 10002, 'device': 'sdh'})
rb.add_dev({'id': 8, 'region': 0, 'zone': 2, 'weight': 1,
'ip': '127.0.0.1', 'port': 10002, 'device': 'sdi'})
rb.rebalance()
rb.validate()
for part in xrange(rb.parts):
counts = defaultdict(lambda: defaultdict(int))
for replica in xrange(rb.replicas): for replica in xrange(rb.replicas):
dev = rb.devs[rb._replica2part2dev[replica][part]] dev = rb.devs[rb._replica2part2dev[replica][part]]
counts['zone'][dev['zone']] += 1 counts['zone'][dev['zone']] += 1
@ -288,26 +320,26 @@ class TestRingBuilder(unittest.TestCase):
def test_multitier_full(self): def test_multitier_full(self):
# Multitier test, #replicas == #devs # Multitier test, #replicas == #devs
rb = ring.RingBuilder(8, 6, 1) rb = ring.RingBuilder(8, 6, 1)
rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1,
'port': 10000, 'device': 'sda'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda'})
rb.add_dev({'id': 1, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 1, 'region': 0, 'zone': 0, 'weight': 1,
'port': 10000, 'device': 'sdb'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdb'})
rb.add_dev({'id': 2, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 2, 'region': 0, 'zone': 1, 'weight': 1,
'port': 10000, 'device': 'sdc'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdc'})
rb.add_dev({'id': 3, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 3, 'region': 0, 'zone': 1, 'weight': 1,
'port': 10001, 'device': 'sdd'}) 'ip': '127.0.0.1', 'port': 10001, 'device': 'sdd'})
rb.add_dev({'id': 4, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 4, 'region': 0, 'zone': 2, 'weight': 1,
'port': 10001, 'device': 'sde'}) 'ip': '127.0.0.1', 'port': 10001, 'device': 'sde'})
rb.add_dev({'id': 5, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 5, 'region': 0, 'zone': 2, 'weight': 1,
'port': 10001, 'device': 'sdf'}) 'ip': '127.0.0.1', 'port': 10001, 'device': 'sdf'})
rb.rebalance() rb.rebalance()
rb.validate() rb.validate()
for part in xrange(rb.parts): for part in xrange(rb.parts):
counts = defaultdict(lambda: defaultdict(lambda: 0)) counts = defaultdict(lambda: defaultdict(int))
for replica in xrange(rb.replicas): for replica in xrange(rb.replicas):
dev = rb.devs[rb._replica2part2dev[replica][part]] dev = rb.devs[rb._replica2part2dev[replica][part]]
counts['zone'][dev['zone']] += 1 counts['zone'][dev['zone']] += 1
@ -325,26 +357,26 @@ class TestRingBuilder(unittest.TestCase):
def test_multitier_overfull(self): def test_multitier_overfull(self):
# Multitier test, #replicas > #devs + 2 (to prove even distribution) # Multitier test, #replicas > #devs + 2 (to prove even distribution)
rb = ring.RingBuilder(8, 8, 1) rb = ring.RingBuilder(8, 8, 1)
rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1,
'port': 10000, 'device': 'sda'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda'})
rb.add_dev({'id': 1, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 1, 'region': 0, 'zone': 0, 'weight': 1,
'port': 10000, 'device': 'sdb'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdb'})
rb.add_dev({'id': 2, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 2, 'region': 0, 'zone': 1, 'weight': 1,
'port': 10000, 'device': 'sdc'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdc'})
rb.add_dev({'id': 3, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 3, 'region': 0, 'zone': 1, 'weight': 1,
'port': 10001, 'device': 'sdd'}) 'ip': '127.0.0.1', 'port': 10001, 'device': 'sdd'})
rb.add_dev({'id': 4, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 4, 'region': 0, 'zone': 2, 'weight': 1,
'port': 10001, 'device': 'sde'}) 'ip': '127.0.0.1', 'port': 10001, 'device': 'sde'})
rb.add_dev({'id': 5, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 5, 'region': 0, 'zone': 2, 'weight': 1,
'port': 10001, 'device': 'sdf'}) 'ip': '127.0.0.1', 'port': 10001, 'device': 'sdf'})
rb.rebalance() rb.rebalance()
rb.validate() rb.validate()
for part in xrange(rb.parts): for part in xrange(rb.parts):
counts = defaultdict(lambda: defaultdict(lambda: 0)) counts = defaultdict(lambda: defaultdict(int))
for replica in xrange(rb.replicas): for replica in xrange(rb.replicas):
dev = rb.devs[rb._replica2part2dev[replica][part]] dev = rb.devs[rb._replica2part2dev[replica][part]]
counts['zone'][dev['zone']] += 1 counts['zone'][dev['zone']] += 1
@ -365,22 +397,22 @@ class TestRingBuilder(unittest.TestCase):
def test_multitier_expansion_more_devices(self): def test_multitier_expansion_more_devices(self):
rb = ring.RingBuilder(8, 6, 1) rb = ring.RingBuilder(8, 6, 1)
rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1,
'port': 10000, 'device': 'sda'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda'})
rb.add_dev({'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1,
'port': 10000, 'device': 'sdb'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdb'})
rb.add_dev({'id': 2, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 2, 'region': 0, 'zone': 2, 'weight': 1,
'port': 10000, 'device': 'sdc'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdc'})
rb.rebalance() rb.rebalance()
rb.validate() rb.validate()
rb.add_dev({'id': 3, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 3, 'region': 0, 'zone': 0, 'weight': 1,
'port': 10000, 'device': 'sdd'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdd'})
rb.add_dev({'id': 4, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 4, 'region': 0, 'zone': 1, 'weight': 1,
'port': 10000, 'device': 'sde'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sde'})
rb.add_dev({'id': 5, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 5, 'region': 0, 'zone': 2, 'weight': 1,
'port': 10000, 'device': 'sdf'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdf'})
for _ in xrange(5): for _ in xrange(5):
rb.pretend_min_part_hours_passed() rb.pretend_min_part_hours_passed()
@ -388,8 +420,8 @@ class TestRingBuilder(unittest.TestCase):
rb.validate() rb.validate()
for part in xrange(rb.parts): for part in xrange(rb.parts):
counts = dict(zone=defaultdict(lambda: 0), counts = dict(zone=defaultdict(int),
dev_id=defaultdict(lambda: 0)) dev_id=defaultdict(int))
for replica in xrange(rb.replicas): for replica in xrange(rb.replicas):
dev = rb.devs[rb._replica2part2dev[replica][part]] dev = rb.devs[rb._replica2part2dev[replica][part]]
counts['zone'][dev['zone']] += 1 counts['zone'][dev['zone']] += 1
@ -401,17 +433,17 @@ class TestRingBuilder(unittest.TestCase):
def test_multitier_part_moves_with_0_min_part_hours(self): def test_multitier_part_moves_with_0_min_part_hours(self):
rb = ring.RingBuilder(8, 3, 0) rb = ring.RingBuilder(8, 3, 0)
rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1,
'port': 10000, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'})
rb.rebalance() rb.rebalance()
rb.validate() rb.validate()
# min_part_hours is 0, so we're clear to move 2 replicas to # min_part_hours is 0, so we're clear to move 2 replicas to
# new devs # new devs
rb.add_dev({'id': 1, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 1, 'region': 0, 'zone': 0, 'weight': 1,
'port': 10000, 'device': 'sdb1'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdb1'})
rb.add_dev({'id': 2, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 2, 'region': 0, 'zone': 0, 'weight': 1,
'port': 10000, 'device': 'sdc1'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdc1'})
rb.rebalance() rb.rebalance()
rb.validate() rb.validate()
@ -426,17 +458,17 @@ class TestRingBuilder(unittest.TestCase):
def test_multitier_part_moves_with_positive_min_part_hours(self): def test_multitier_part_moves_with_positive_min_part_hours(self):
rb = ring.RingBuilder(8, 3, 99) rb = ring.RingBuilder(8, 3, 99)
rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1,
'port': 10000, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'})
rb.rebalance() rb.rebalance()
rb.validate() rb.validate()
# min_part_hours is >0, so we'll only be able to move 1 # min_part_hours is >0, so we'll only be able to move 1
# replica to a new home # replica to a new home
rb.add_dev({'id': 1, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 1, 'region': 0, 'zone': 0, 'weight': 1,
'port': 10000, 'device': 'sdb1'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdb1'})
rb.add_dev({'id': 2, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 2, 'region': 0, 'zone': 0, 'weight': 1,
'port': 10000, 'device': 'sdc1'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdc1'})
rb.pretend_min_part_hours_passed() rb.pretend_min_part_hours_passed()
rb.rebalance() rb.rebalance()
rb.validate() rb.validate()
@ -453,20 +485,20 @@ class TestRingBuilder(unittest.TestCase):
def test_multitier_dont_move_too_many_replicas(self): def test_multitier_dont_move_too_many_replicas(self):
rb = ring.RingBuilder(8, 3, 0) rb = ring.RingBuilder(8, 3, 0)
# there'll be at least one replica in z0 and z1 # there'll be at least one replica in z0 and z1
rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1,
'port': 10000, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'})
rb.add_dev({'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1,
'port': 10000, 'device': 'sdb1'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdb1'})
rb.rebalance() rb.rebalance()
rb.validate() rb.validate()
# only 1 replica should move # only 1 replica should move
rb.add_dev({'id': 2, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 2, 'region': 0, 'zone': 2, 'weight': 1,
'port': 10000, 'device': 'sdd1'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdd1'})
rb.add_dev({'id': 3, 'zone': 3, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 3, 'region': 0, 'zone': 3, 'weight': 1,
'port': 10000, 'device': 'sde1'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sde1'})
rb.add_dev({'id': 4, 'zone': 4, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 4, 'region': 0, 'zone': 4, 'weight': 1,
'port': 10000, 'device': 'sdf1'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdf1'})
rb.rebalance() rb.rebalance()
rb.validate() rb.validate()
@ -485,12 +517,12 @@ class TestRingBuilder(unittest.TestCase):
def test_rerebalance(self): def test_rerebalance(self):
rb = ring.RingBuilder(8, 3, 1) rb = ring.RingBuilder(8, 3, 1)
rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1,
'port': 10000, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'})
rb.add_dev({'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1,
'port': 10001, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'})
rb.add_dev({'id': 2, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 2, 'region': 0, 'zone': 2, 'weight': 1,
'port': 10002, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10002, 'device': 'sda1'})
rb.rebalance() rb.rebalance()
r = rb.get_ring() r = rb.get_ring()
counts = {} counts = {}
@ -498,8 +530,8 @@ class TestRingBuilder(unittest.TestCase):
for dev_id in part2dev_id: for dev_id in part2dev_id:
counts[dev_id] = counts.get(dev_id, 0) + 1 counts[dev_id] = counts.get(dev_id, 0) + 1
self.assertEquals(counts, {0: 256, 1: 256, 2: 256}) self.assertEquals(counts, {0: 256, 1: 256, 2: 256})
rb.add_dev({'id': 3, 'zone': 3, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 3, 'region': 0, 'zone': 3, 'weight': 1,
'port': 10003, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10003, 'device': 'sda1'})
rb.pretend_min_part_hours_passed() rb.pretend_min_part_hours_passed()
rb.rebalance() rb.rebalance()
r = rb.get_ring() r = rb.get_ring()
@ -521,21 +553,21 @@ class TestRingBuilder(unittest.TestCase):
""" Test for https://bugs.launchpad.net/swift/+bug/845952 """ """ Test for https://bugs.launchpad.net/swift/+bug/845952 """
# min_part of 0 to allow for rapid rebalancing # min_part of 0 to allow for rapid rebalancing
rb = ring.RingBuilder(8, 3, 0) rb = ring.RingBuilder(8, 3, 0)
rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1,
'port': 10000, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'})
rb.add_dev({'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1,
'port': 10001, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'})
rb.add_dev({'id': 2, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 2, 'region': 0, 'zone': 2, 'weight': 1,
'port': 10002, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10002, 'device': 'sda1'})
rb.rebalance() rb.rebalance()
rb.add_dev({'id': 3, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 3, 'region': 0, 'zone': 0, 'weight': 1,
'port': 10003, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10003, 'device': 'sda1'})
rb.add_dev({'id': 4, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 4, 'region': 0, 'zone': 1, 'weight': 1,
'port': 10004, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10004, 'device': 'sda1'})
rb.add_dev({'id': 5, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 5, 'region': 0, 'zone': 2, 'weight': 1,
'port': 10005, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10005, 'device': 'sda1'})
rb.rebalance() rb.rebalance()
@ -545,10 +577,10 @@ class TestRingBuilder(unittest.TestCase):
def test_set_replicas_increase(self): def test_set_replicas_increase(self):
rb = ring.RingBuilder(8, 2, 0) rb = ring.RingBuilder(8, 2, 0)
rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1,
'port': 10000, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'})
rb.add_dev({'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1,
'port': 10001, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'})
rb.rebalance() rb.rebalance()
rb.validate() rb.validate()
@ -567,10 +599,10 @@ class TestRingBuilder(unittest.TestCase):
def test_set_replicas_decrease(self): def test_set_replicas_decrease(self):
rb = ring.RingBuilder(4, 5, 0) rb = ring.RingBuilder(4, 5, 0)
rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1,
'port': 10000, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'})
rb.add_dev({'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1,
'port': 10001, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'})
rb.rebalance() rb.rebalance()
rb.validate() rb.validate()
@ -593,10 +625,10 @@ class TestRingBuilder(unittest.TestCase):
def test_fractional_replicas_rebalance(self): def test_fractional_replicas_rebalance(self):
rb = ring.RingBuilder(8, 2.5, 0) rb = ring.RingBuilder(8, 2.5, 0)
rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 0, 'region': 0, 'region': 0, 'zone': 0, 'weight': 1,
'port': 10000, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'})
rb.add_dev({'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 1, 'region': 0, 'region': 0, 'zone': 1, 'weight': 1,
'port': 10001, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'})
rb.rebalance() # passes by not crashing rb.rebalance() # passes by not crashing
rb.validate() # also passes by not crashing rb.validate() # also passes by not crashing
self.assertEqual([len(p2d) for p2d in rb._replica2part2dev], self.assertEqual([len(p2d) for p2d in rb._replica2part2dev],
@ -604,14 +636,17 @@ class TestRingBuilder(unittest.TestCase):
def test_load(self): def test_load(self):
rb = ring.RingBuilder(8, 3, 1) rb = ring.RingBuilder(8, 3, 1)
devs = [{'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.0', devs = [{'id': 0, 'region': 0, 'zone': 0, 'weight': 1,
'port': 10000, 'device': 'sda1', 'meta': 'meta0'}, 'ip': '127.0.0.0', 'port': 10000, 'device': 'sda1',
{'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'meta': 'meta0'},
'port': 10001, 'device': 'sdb1', 'meta': 'meta1'}, {'id': 1, 'region': 0, 'zone': 1, 'weight': 1,
{'id': 2, 'zone': 2, 'weight': 2, 'ip': '127.0.0.2', 'ip': '127.0.0.1', 'port': 10001, 'device': 'sdb1',
'port': 10002, 'device': 'sdc1', 'meta': 'meta2'}, 'meta': 'meta1'},
{'id': 3, 'zone': 3, 'weight': 2, 'ip': '127.0.0.3', {'id': 2, 'region': 0, 'zone': 2, 'weight': 2,
'port': 10003, 'device': 'sdd1'}] 'ip': '127.0.0.2', 'port': 10002, 'device': 'sdc1',
'meta': 'meta2'},
{'id': 3, 'region': 0, 'zone': 3, 'weight': 2,
'ip': '127.0.0.3', 'port': 10003, 'device': 'sdd1'}]
for d in devs: for d in devs:
rb.add_dev(d) rb.add_dev(d)
rb.rebalance() rb.rebalance()
@ -653,17 +688,27 @@ class TestRingBuilder(unittest.TestCase):
def test_search_devs(self): def test_search_devs(self):
rb = ring.RingBuilder(8, 3, 1) rb = ring.RingBuilder(8, 3, 1)
devs = [{'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.0', devs = [{'id': 0, 'region': 0, 'zone': 0, 'weight': 1,
'port': 10000, 'device': 'sda1', 'meta': 'meta0'}, 'ip': '127.0.0.0', 'port': 10000, 'device': 'sda1',
{'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'meta': 'meta0'},
'port': 10001, 'device': 'sdb1', 'meta': 'meta1'}, {'id': 1, 'region': 0, 'zone': 1, 'weight': 1,
{'id': 2, 'zone': 2, 'weight': 2, 'ip': '127.0.0.2', 'ip': '127.0.0.1', 'port': 10001, 'device': 'sdb1',
'port': 10002, 'device': 'sdc1', 'meta': 'meta2'}, 'meta': 'meta1'},
{'id': 3, 'zone': 3, 'weight': 2, 'ip': '127.0.0.3', {'id': 2, 'region': 1, 'zone': 2, 'weight': 2,
'port': 10003, 'device': 'sdd1', 'meta': 'meta3'}] 'ip': '127.0.0.2', 'port': 10002, 'device': 'sdc1',
'meta': 'meta2'},
{'id': 3, 'region': 1, 'zone': 3, 'weight': 2,
'ip': '127.0.0.3', 'port': 10003, 'device': 'sdffd1',
'meta': 'meta3'}]
for d in devs: for d in devs:
rb.add_dev(d) rb.add_dev(d)
rb.rebalance() rb.rebalance()
res = rb.search_devs('r0')
self.assertEquals(res, [devs[0], devs[1]])
res = rb.search_devs('r1')
self.assertEquals(res, [devs[2], devs[3]])
res = rb.search_devs('r1z2')
self.assertEquals(res, [devs[2]])
res = rb.search_devs('d1') res = rb.search_devs('d1')
self.assertEquals(res, [devs[1]]) self.assertEquals(res, [devs[1]])
res = rb.search_devs('z1') res = rb.search_devs('z1')
@ -681,14 +726,14 @@ class TestRingBuilder(unittest.TestCase):
def test_validate(self): def test_validate(self):
rb = ring.RingBuilder(8, 3, 1) rb = ring.RingBuilder(8, 3, 1)
rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1,
'port': 10000, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'})
rb.add_dev({'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1,
'port': 10001, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'})
rb.add_dev({'id': 2, 'zone': 2, 'weight': 2, 'ip': '127.0.0.1', rb.add_dev({'id': 2, 'region': 0, 'zone': 2, 'weight': 2,
'port': 10002, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10002, 'device': 'sda1'})
rb.add_dev({'id': 3, 'zone': 3, 'weight': 2, 'ip': '127.0.0.1', rb.add_dev({'id': 3, 'region': 0, 'zone': 3, 'weight': 2,
'port': 10003, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10003, 'device': 'sda1'})
rb.rebalance() rb.rebalance()
r = rb.get_ring() r = rb.get_ring()
counts = {} counts = {}
@ -744,18 +789,18 @@ class TestRingBuilder(unittest.TestCase):
# Validate that zero weight devices with no partitions don't count on # Validate that zero weight devices with no partitions don't count on
# the 'worst' value. # the 'worst' value.
self.assertNotEquals(rb.validate(stats=True)[1], 999.99) self.assertNotEquals(rb.validate(stats=True)[1], 999.99)
rb.add_dev({'id': 4, 'zone': 0, 'weight': 0, 'ip': '127.0.0.1', rb.add_dev({'id': 4, 'region': 0, 'zone': 0, 'weight': 0,
'port': 10004, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10004, 'device': 'sda1'})
rb.pretend_min_part_hours_passed() rb.pretend_min_part_hours_passed()
rb.rebalance() rb.rebalance()
self.assertNotEquals(rb.validate(stats=True)[1], 999.99) self.assertNotEquals(rb.validate(stats=True)[1], 999.99)
def test_get_part_devices(self): def test_get_part_devices(self):
rb = ring.RingBuilder(8, 3, 1) rb = ring.RingBuilder(8, 3, 1)
rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1,
'port': 10000, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'})
rb.add_dev({'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1,
'port': 10001, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'})
rb.rebalance() rb.rebalance()
part_devs = sorted(rb.get_part_devices(0), part_devs = sorted(rb.get_part_devices(0),
@ -764,10 +809,10 @@ class TestRingBuilder(unittest.TestCase):
def test_get_part_devices_partial_replicas(self): def test_get_part_devices_partial_replicas(self):
rb = ring.RingBuilder(8, 2.5, 1) rb = ring.RingBuilder(8, 2.5, 1)
rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1,
'port': 10000, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'})
rb.add_dev({'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1,
'port': 10001, 'device': 'sda1'}) 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'})
rb.rebalance() rb.rebalance()
# note: partition 255 will only have 2 replicas # note: partition 255 will only have 2 replicas

75
test/unit/common/ring/test_ring.py
View File

@ -43,7 +43,8 @@ class TestRingData(unittest.TestCase):
def test_attrs(self): def test_attrs(self):
r2p2d = [[0, 1, 0, 1], [0, 1, 0, 1]] r2p2d = [[0, 1, 0, 1], [0, 1, 0, 1]]
d = [{'id': 0, 'zone': 0}, {'id': 1, 'zone': 1}] d = [{'id': 0, 'zone': 0, 'region': 0},
{'id': 1, 'zone': 1, 'region': 1}]
s = 30 s = 30
rd = ring.RingData(r2p2d, d, s) rd = ring.RingData(r2p2d, d, s)
self.assertEquals(rd._replica2part2dev_id, r2p2d) self.assertEquals(rd._replica2part2dev_id, r2p2d)
@ -104,14 +105,14 @@ class TestRing(unittest.TestCase):
array.array('H', [0, 1, 0, 1]), array.array('H', [0, 1, 0, 1]),
array.array('H', [0, 1, 0, 1]), array.array('H', [0, 1, 0, 1]),
array.array('H', [3, 4, 3, 4])] array.array('H', [3, 4, 3, 4])]
self.intended_devs = [{'id': 0, 'zone': 0, 'weight': 1.0, self.intended_devs = [{'id': 0, 'region': 0, 'zone': 0, 'weight': 1.0,
'ip': '10.1.1.1', 'port': 6000}, 'ip': '10.1.1.1', 'port': 6000},
{'id': 1, 'zone': 0, 'weight': 1.0, {'id': 1, 'region': 0, 'zone': 0, 'weight': 1.0,
'ip': '10.1.1.1', 'port': 6000}, 'ip': '10.1.1.1', 'port': 6000},
None, None,
{'id': 3, 'zone': 2, 'weight': 1.0, {'id': 3, 'region': 0, 'zone': 2, 'weight': 1.0,
'ip': '10.1.2.1', 'port': 6000}, 'ip': '10.1.2.1', 'port': 6000},
{'id': 4, 'zone': 2, 'weight': 1.0, {'id': 4, 'region': 0, 'zone': 2, 'weight': 1.0,
'ip': '10.1.2.2', 'port': 6000}] 'ip': '10.1.2.2', 'port': 6000}]
self.intended_part_shift = 30 self.intended_part_shift = 30
self.intended_reload_time = 15 self.intended_reload_time = 15
@ -149,7 +150,7 @@ class TestRing(unittest.TestCase):
ring_name='whatever') ring_name='whatever')
orig_mtime = self.ring._mtime orig_mtime = self.ring._mtime
self.assertEquals(len(self.ring.devs), 5) self.assertEquals(len(self.ring.devs), 5)
self.intended_devs.append({'id': 3, 'zone': 3, 'weight': 1.0}) self.intended_devs.append({'id': 3, 'region': 0, 'zone': 3, 'weight': 1.0})
ring.RingData(self.intended_replica2part2dev_id, ring.RingData(self.intended_replica2part2dev_id,
self.intended_devs, self.intended_part_shift).save(self.testgz) self.intended_devs, self.intended_part_shift).save(self.testgz)
sleep(0.1) sleep(0.1)
@ -162,7 +163,7 @@ class TestRing(unittest.TestCase):
ring_name='whatever') ring_name='whatever')
orig_mtime = self.ring._mtime orig_mtime = self.ring._mtime
self.assertEquals(len(self.ring.devs), 6) self.assertEquals(len(self.ring.devs), 6)
self.intended_devs.append({'id': 5, 'zone': 4, 'weight': 1.0}) self.intended_devs.append({'id': 5, 'region': 0, 'zone': 4, 'weight': 1.0})
ring.RingData(self.intended_replica2part2dev_id, ring.RingData(self.intended_replica2part2dev_id,
self.intended_devs, self.intended_part_shift).save(self.testgz) self.intended_devs, self.intended_part_shift).save(self.testgz)
sleep(0.1) sleep(0.1)
@ -176,7 +177,7 @@ class TestRing(unittest.TestCase):
orig_mtime = self.ring._mtime orig_mtime = self.ring._mtime
part, nodes = self.ring.get_nodes('a') part, nodes = self.ring.get_nodes('a')
self.assertEquals(len(self.ring.devs), 7) self.assertEquals(len(self.ring.devs), 7)
self.intended_devs.append({'id': 6, 'zone': 5, 'weight': 1.0}) self.intended_devs.append({'id': 6, 'region': 0, 'zone': 5, 'weight': 1.0})
ring.RingData(self.intended_replica2part2dev_id, ring.RingData(self.intended_replica2part2dev_id,
self.intended_devs, self.intended_part_shift).save(self.testgz) self.intended_devs, self.intended_part_shift).save(self.testgz)
sleep(0.1) sleep(0.1)
@ -189,7 +190,7 @@ class TestRing(unittest.TestCase):
ring_name='whatever') ring_name='whatever')
orig_mtime = self.ring._mtime orig_mtime = self.ring._mtime
self.assertEquals(len(self.ring.devs), 8) self.assertEquals(len(self.ring.devs), 8)
self.intended_devs.append({'id': 5, 'zone': 4, 'weight': 1.0}) self.intended_devs.append({'id': 5, 'region': 0, 'zone': 4, 'weight': 1.0})
ring.RingData(self.intended_replica2part2dev_id, ring.RingData(self.intended_replica2part2dev_id,
self.intended_devs, self.intended_part_shift).save(self.testgz) self.intended_devs, self.intended_part_shift).save(self.testgz)
sleep(0.1) sleep(0.1)
@ -312,7 +313,8 @@ class TestRing(unittest.TestCase):
for device in xrange(1, 4): for device in xrange(1, 4):
rb.add_dev({'id': next_dev_id, rb.add_dev({'id': next_dev_id,
'ip': '1.2.%d.%d' % (zone, server), 'ip': '1.2.%d.%d' % (zone, server),
'port': 1234, 'zone': zone, 'weight': 1.0}) 'port': 1234, 'zone': zone, 'region': 0,
'weight': 1.0})
next_dev_id += 1 next_dev_id += 1
rb.rebalance(seed=1) rb.rebalance(seed=1)
rb.get_ring().save(self.testgz) rb.get_ring().save(self.testgz)
@ -343,7 +345,7 @@ class TestRing(unittest.TestCase):
server = 0 server = 0
rb.add_dev({'id': next_dev_id, rb.add_dev({'id': next_dev_id,
'ip': '1.2.%d.%d' % (zone, server), 'ip': '1.2.%d.%d' % (zone, server),
'port': 1234, 'zone': zone, 'weight': 1.0}) 'port': 1234, 'zone': zone, 'region': 0, 'weight': 1.0})
next_dev_id += 1 next_dev_id += 1
rb.rebalance(seed=1) rb.rebalance(seed=1)
rb.get_ring().save(self.testgz) rb.get_ring().save(self.testgz)
@ -542,6 +544,57 @@ class TestRing(unittest.TestCase):
seen_zones.update(d['zone'] for d in devs2[:6]) seen_zones.update(d['zone'] for d in devs2[:6])
self.assertEquals(seen_zones, set(range(1, 10))) self.assertEquals(seen_zones, set(range(1, 10)))
# Test distribution across regions
rb.set_replicas(3)
for region in xrange(1, 5):
rb.add_dev({'id': next_dev_id,
'ip': '1.%d.1.%d' % (region, server), 'port': 1234,
'zone': 1, 'region': region, 'weight': 1.0})
next_dev_id += 1
rb.pretend_min_part_hours_passed()
rb.rebalance(seed=1)
rb.pretend_min_part_hours_passed()
rb.rebalance(seed=1)
rb.get_ring().save(self.testgz)
r = ring.Ring(self.testdir, ring_name='whatever')
# There's 5 regions now, so the primary nodes + first 2 handoffs
# should span all 5 regions
part, devs = r.get_nodes('a1', 'c1', 'o1')
primary_regions = set(d['region'] for d in devs)
primary_zones = set((d['region'], d['zone']) for d in devs)
more_devs = list(r.get_more_nodes(part))
seen_regions = set(primary_regions)
seen_regions.update(d['region'] for d in more_devs[:2])
self.assertEquals(seen_regions, set(range(0, 5)))
# There are 13 zones now, so the first 13 nodes should all have
# distinct zones (that's r0z0, r0z1, ..., r0z8, r1z1, r2z1, r3z1, and
# r4z1).
seen_zones = set(primary_zones)
seen_zones.update((d['region'], d['zone']) for d in more_devs[:10])
self.assertEquals(13, len(seen_zones))
# Here's a brittle canary-in-the-coalmine test to make sure the region
# handoff computation didn't change accidentally
exp_handoffs = [111, 112, 74, 54, 93, 31, 2, 43, 100, 22, 71, 32, 92,
35, 9, 50, 41, 76, 80, 84, 88, 17, 94, 101, 1, 10, 96,
44, 73, 6, 75, 102, 37, 21, 97, 29, 105, 5, 28, 47,
106, 30, 16, 39, 77, 42, 72, 20, 13, 34, 99, 108, 14,
66, 61, 81, 90, 4, 40, 3, 45, 62, 7, 15, 87, 12, 83,
89, 53, 33, 98, 49, 65, 25, 107, 56, 58, 86, 48, 57,
24, 11, 23, 26, 46, 64, 69, 38, 36, 79, 63, 104, 51,
70, 82, 67, 68, 8, 95, 91, 55, 59, 85]
dev_ids = [d['id'] for d in more_devs]
self.assertEquals(len(dev_ids), len(exp_handoffs))
for index, dev_id in enumerate(dev_ids):
self.assertEquals(
dev_id, exp_handoffs[index],
'handoff differs at position %d\n%s\n%s' % (
index, dev_ids[index:], exp_handoffs[index:]))
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()

92
test/unit/common/ring/test_utils.py
View File

@ -21,22 +21,34 @@ from swift.common.ring.utils import build_tier_tree, tiers_for_dev
class TestUtils(unittest.TestCase): class TestUtils(unittest.TestCase):
def setUp(self): def setUp(self):
self.test_dev = {'zone': 1, 'ip': '192.168.1.1', self.test_dev = {'region': 1, 'zone': 1, 'ip': '192.168.1.1',
'port': '6000', 'id': 0} 'port': '6000', 'id': 0}
def get_test_devs(): def get_test_devs():
dev0 = {'zone': 1, 'ip': '192.168.1.1', 'port': '6000', 'id': 0} dev0 = {'region': 1, 'zone': 1, 'ip': '192.168.1.1',
dev1 = {'zone': 1, 'ip': '192.168.1.1', 'port': '6000', 'id': 1} 'port': '6000', 'id': 0}
dev2 = {'zone': 1, 'ip': '192.168.1.1', 'port': '6000', 'id': 2} dev1 = {'region': 1, 'zone': 1, 'ip': '192.168.1.1',
dev3 = {'zone': 1, 'ip': '192.168.1.2', 'port': '6000', 'id': 3} 'port': '6000', 'id': 1}
dev4 = {'zone': 1, 'ip': '192.168.1.2', 'port': '6000', 'id': 4} dev2 = {'region': 1, 'zone': 1, 'ip': '192.168.1.1',
dev5 = {'zone': 1, 'ip': '192.168.1.2', 'port': '6000', 'id': 5} 'port': '6000', 'id': 2}
dev6 = {'zone': 2, 'ip': '192.168.2.1', 'port': '6000', 'id': 6} dev3 = {'region': 1, 'zone': 1, 'ip': '192.168.1.2',
dev7 = {'zone': 2, 'ip': '192.168.2.1', 'port': '6000', 'id': 7} 'port': '6000', 'id': 3}
dev8 = {'zone': 2, 'ip': '192.168.2.1', 'port': '6000', 'id': 8} dev4 = {'region': 1, 'zone': 1, 'ip': '192.168.1.2',
dev9 = {'zone': 2, 'ip': '192.168.2.2', 'port': '6000', 'id': 9} 'port': '6000', 'id': 4}
dev10 = {'zone': 2, 'ip': '192.168.2.2', 'port': '6000', 'id': 10} dev5 = {'region': 1, 'zone': 1, 'ip': '192.168.1.2',
dev11 = {'zone': 2, 'ip': '192.168.2.2', 'port': '6000', 'id': 11} 'port': '6000', 'id': 5}
dev6 = {'region': 1, 'zone': 2, 'ip': '192.168.2.1',
'port': '6000', 'id': 6}
dev7 = {'region': 1, 'zone': 2, 'ip': '192.168.2.1',
'port': '6000', 'id': 7}
dev8 = {'region': 1, 'zone': 2, 'ip': '192.168.2.1',
'port': '6000', 'id': 8}
dev9 = {'region': 1, 'zone': 2, 'ip': '192.168.2.2',
'port': '6000', 'id': 9}
dev10 = {'region': 1, 'zone': 2, 'ip': '192.168.2.2',
'port': '6000', 'id': 10}
dev11 = {'region': 1, 'zone': 2, 'ip': '192.168.2.2',
'port': '6000', 'id': 11}
return [dev0, dev1, dev2, dev3, dev4, dev5, return [dev0, dev1, dev2, dev3, dev4, dev5,
dev6, dev7, dev8, dev9, dev10, dev11] dev6, dev7, dev8, dev9, dev10, dev11]
@ -44,34 +56,38 @@ class TestUtils(unittest.TestCase):
def test_tiers_for_dev(self): def test_tiers_for_dev(self):
self.assertEqual(tiers_for_dev(self.test_dev), self.assertEqual(tiers_for_dev(self.test_dev),
((1,), (1, '192.168.1.1:6000'), (1, '192.168.1.1:6000', 0))) ((1,),
(1, 1),
(1, 1, '192.168.1.1:6000'),
(1, 1, '192.168.1.1:6000', 0)))
def test_build_tier_tree(self): def test_build_tier_tree(self):
ret = build_tier_tree(self.test_devs) ret = build_tier_tree(self.test_devs)
self.assertEqual(len(ret), 7) self.assertEqual(len(ret), 8)
self.assertEqual(ret[()], set([(2,), (1,)])) self.assertEqual(ret[()], set([(1,)]))
self.assertEqual(ret[(1,)], self.assertEqual(ret[(1,)], set([(1, 1), (1, 2)]))
set([(1, '192.168.1.2:6000'), self.assertEqual(ret[(1, 1)],
(1, '192.168.1.1:6000')])) set([(1, 1, '192.168.1.2:6000'),
self.assertEqual(ret[(2,)], (1, 1, '192.168.1.1:6000')]))
set([(2, '192.168.2.2:6000'), self.assertEqual(ret[(1, 2)],
(2, '192.168.2.1:6000')])) set([(1, 2, '192.168.2.2:6000'),
self.assertEqual(ret[(1, '192.168.1.1:6000')], (1, 2, '192.168.2.1:6000')]))
set([(1, '192.168.1.1:6000', 0), self.assertEqual(ret[(1, 1, '192.168.1.1:6000')],
(1, '192.168.1.1:6000', 1), set([(1, 1, '192.168.1.1:6000', 0),
(1, '192.168.1.1:6000', 2)])) (1, 1, '192.168.1.1:6000', 1),
self.assertEqual(ret[(1, '192.168.1.2:6000')], (1, 1, '192.168.1.1:6000', 2)]))
set([(1, '192.168.1.2:6000', 3), self.assertEqual(ret[(1, 1, '192.168.1.2:6000')],
(1, '192.168.1.2:6000', 4), set([(1, 1, '192.168.1.2:6000', 3),
(1, '192.168.1.2:6000', 5)])) (1, 1, '192.168.1.2:6000', 4),
self.assertEqual(ret[(2, '192.168.2.1:6000')], (1, 1, '192.168.1.2:6000', 5)]))
set([(2, '192.168.2.1:6000', 6), self.assertEqual(ret[(1, 2, '192.168.2.1:6000')],
(2, '192.168.2.1:6000', 7), set([(1, 2, '192.168.2.1:6000', 6),
(2, '192.168.2.1:6000', 8)])) (1, 2, '192.168.2.1:6000', 7),
self.assertEqual(ret[(2, '192.168.2.2:6000')], (1, 2, '192.168.2.1:6000', 8)]))
set([(2, '192.168.2.2:6000', 9), self.assertEqual(ret[(1, 2, '192.168.2.2:6000')],
(2, '192.168.2.2:6000', 10), set([(1, 2, '192.168.2.2:6000', 9),
(2, '192.168.2.2:6000', 11)])) (1, 2, '192.168.2.2:6000', 10),
(1, 2, '192.168.2.2:6000', 11)]))
if __name__ == '__main__': if __name__ == '__main__':