openstack/swift
Code Issues Proposed changes

Merge "Make ring class interface slightly more abstracted from implementation."

Browse Source
This commit is contained in:
Jenkins 2012-03-22 20:25:06 +00:00 committed by Gerrit Code Review
commit 6682138b0a
18 changed files with 52 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
bin/swift-account-audit
View File

@ -55,9 +55,9 @@ class Auditor(object):
def __init__(self, swift_dir='/etc/swift', concurrency=50, deep=False,
error_file=None):
self.pool = GreenPool(concurrency)
self.object_ring = Ring(os.path.join(swift_dir, 'object.ring.gz'))
self.container_ring = Ring(os.path.join(swift_dir, 'container.ring.gz'))
self.account_ring = Ring(os.path.join(swift_dir, 'account.ring.gz'))
self.object_ring = Ring(os.path.join(swift_dir, ring_name='object'))
self.container_ring = Ring(os.path.join(swift_dir, ring_name='container'))
self.account_ring = Ring(os.path.join(swift_dir, ring_name='account'))
self.deep = deep
self.error_file = error_file
# zero out stats

4
bin/swift-dispersion-populate
View File

@ -104,7 +104,7 @@ if __name__ == '__main__':
retries=retries,
preauthurl=url, preauthtoken=token)
container_ring = Ring(os.path.join(swift_dir, 'container.ring.gz'))
container_ring = Ring(swift_dir, ring_name='container')
parts_left = dict((x, x) for x in xrange(container_ring.partition_count))
item_type = 'containers'
created = 0
@ -130,7 +130,7 @@ if __name__ == '__main__':
container = 'dispersion_objects'
put_container(connpool, container, None)
object_ring = Ring(os.path.join(swift_dir, 'object.ring.gz'))
object_ring = Ring(swift_dir, ring_name='object'))
parts_left = dict((x, x) for x in xrange(object_ring.partition_count))
item_type = 'objects'
created = 0

4
bin/swift-dispersion-report
View File

@ -289,8 +289,8 @@ Usage: %prog [options] [conf_file]
retries=retries,
preauthurl=url, preauthtoken=token)
container_ring = Ring(os.path.join(swift_dir, 'container.ring.gz'))
object_ring = Ring(os.path.join(swift_dir, 'object.ring.gz'))
container_ring = Ring(swift_dir, ring_name='container')
object_ring = Ring(swift_dir, ring_name='object'))
container_result = container_dispersion_report(coropool, connpool,
account, container_ring,

2
bin/swift-object-info
View File

@ -28,7 +28,7 @@ if __name__ == '__main__':
print "Usage: %s OBJECT_FILE" % sys.argv[0]
sys.exit(1)
try:
ring = Ring('/etc/swift/object.ring.gz')
ring = Ring('/etc/swift/', ring_name='object')
except Exception:
ring = None
datafile = sys.argv[1]

11
bin/swift-recon
View File

@ -88,15 +88,16 @@ class SwiftRecon(object):
self.pool_size = 30
self.pool = eventlet.GreenPool(self.pool_size)
def get_devices(self, zone_filter, ring_file):
def get_devices(self, zone_filter, swift_dir, ring_name):
"""
Get a list of hosts in the ring
:param zone_filter: Only list zones matching given filter
:param ring_file: Ring file to obtain hosts from
:param swift_dir: Directory of swift config, usually /etc/swift
:param ring_name: Name of the ring, such as 'object'
:returns: a set of tuples containing the ip and port of hosts
"""
ring_data = Ring(ring_file)
ring_data = Ring(swift_dir, ring_name=ring_name)
if zone_filter:
ips = set((n['ip'], n['port']) for n in ring_data.devs if n \
if n['zone'] == zone_filter)
@ -430,9 +431,9 @@ class SwiftRecon(object):
self.timeout = options.timeout
if options.zone:
hosts = self.get_devices(options.zone, obj_ring)
hosts = self.get_devices(options.zone, swift_dir, 'object')
else:
hosts = self.get_devices(None, obj_ring)
hosts = self.get_devices(None, swift_dir, 'object')
if options.all:
self.async_check(hosts)

17
swift/account/reaper.py
View File

@ -58,10 +58,7 @@ class AccountReaper(Daemon):
self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y')
self.interval = int(conf.get('interval', 3600))
swift_dir = conf.get('swift_dir', '/etc/swift')
self.account_ring_path = os.path.join(swift_dir, 'account.ring.gz')
self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz')
self.object_ring_path = os.path.join(swift_dir, 'object.ring.gz')
self.swift_dir = conf.get('swift_dir', '/etc/swift')
self.account_ring = None
self.container_ring = None
self.object_ring = None
@ -76,25 +73,19 @@ class AccountReaper(Daemon):
def get_account_ring(self):
""" The account :class:`swift.common.ring.Ring` for the cluster. """
if not self.account_ring:
self.logger.debug(
_('Loading account ring from %s'), self.account_ring_path)
self.account_ring = Ring(self.account_ring_path)
self.account_ring = Ring(self.swift_dir, ring_name='account')
return self.account_ring
def get_container_ring(self):
""" The container :class:`swift.common.ring.Ring` for the cluster. """
if not self.container_ring:
self.logger.debug(
_('Loading container ring from %s'), self.container_ring_path)
self.container_ring = Ring(self.container_ring_path)
self.container_ring = Ring(self.swift_dir, ring_name='container')
return self.container_ring
def get_object_ring(self):
""" The object :class:`swift.common.ring.Ring` for the cluster. """
if not self.object_ring:
self.logger.debug(
_('Loading object ring from %s'), self.object_ring_path)
self.object_ring = Ring(self.object_ring_path)
self.object_ring = Ring(self.swift_dir, ring_name='object')
return self.object_ring
def run_forever(self, *args, **kwargs):

1
swift/account/replicator.py
View File

@ -19,7 +19,6 @@ from swift.common import db, db_replicator
class AccountReplicator(db_replicator.Replicator):
server_type = 'account'
ring_file = 'account.ring.gz'
brokerclass = db.AccountBroker
datadir = account_server.DATADIR
default_port = 6002

2
swift/common/db_replicator.py
View File

@ -110,7 +110,7 @@ class Replicator(Daemon):
concurrency = int(conf.get('concurrency', 8))
self.cpool = GreenPool(size=concurrency)
swift_dir = conf.get('swift_dir', '/etc/swift')
self.ring = ring.Ring(os.path.join(swift_dir, self.ring_file))
self.ring = ring.Ring(swift_dir, ring_name=self.server_type)
self.per_diff = int(conf.get('per_diff', 1000))
self.max_diffs = int(conf.get('max_diffs') or 100)
self.interval = int(conf.get('interval') or

10
swift/common/ring/ring.py
View File

@ -18,6 +18,8 @@ from gzip import GzipFile
from os.path import getmtime
from struct import unpack_from
from time import time
import os
from swift.common.utils import hash_path, validate_configuration
@ -43,10 +45,14 @@ class Ring(object):
:param reload_time: time interval in seconds to check for a ring change
"""
def __init__(self, pickle_gz_path, reload_time=15):
def __init__(self, pickle_gz_path, reload_time=15, ring_name=None):
# can't use the ring unless HASH_PATH_SUFFIX is set
validate_configuration()
self.pickle_gz_path = pickle_gz_path
if ring_name:
self.pickle_gz_path = os.path.join(pickle_gz_path,
ring_name + '.ring.gz')
else:
self.pickle_gz_path = os.path.join(pickle_gz_path)
self.reload_time = reload_time
self._reload(force=True)

1
swift/container/replicator.py
View File

@ -19,7 +19,6 @@ from swift.common import db, db_replicator
class ContainerReplicator(db_replicator.Replicator):
server_type = 'container'
ring_file = 'container.ring.gz'
brokerclass = db.ContainerBroker
datadir = container_server.DATADIR
default_port = 6001

7
swift/container/sync.py
View File

@ -172,11 +172,10 @@ class ContainerSync(Daemon):
self.reported = time()
swift_dir = conf.get('swift_dir', '/etc/swift')
#: swift.common.ring.Ring for locating containers.
self.container_ring = container_ring or \
Ring(os.path.join(swift_dir, 'container.ring.gz'))
self.container_ring = container_ring or Ring(swift_dir,
ring_name='container')
#: swift.common.ring.Ring for locating objects.
self.object_ring = object_ring or \
Ring(os.path.join(swift_dir, 'object.ring.gz'))
self.object_ring = object_ring or Ring(swift_dir, ring_name='object')
self._myips = whataremyips()
self._myport = int(conf.get('bind_port', 6001))

7
swift/container/updater.py
View File

@ -41,9 +41,8 @@ class ContainerUpdater(Daemon):
self.devices = conf.get('devices', '/srv/node')
self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y')
swift_dir = conf.get('swift_dir', '/etc/swift')
self.swift_dir = conf.get('swift_dir', '/etc/swift')
self.interval = int(conf.get('interval', 300))
self.account_ring_path = os.path.join(swift_dir, 'account.ring.gz')
self.account_ring = None
self.concurrency = int(conf.get('concurrency', 4))
self.slowdown = float(conf.get('slowdown', 0.01))
@ -60,9 +59,7 @@ class ContainerUpdater(Daemon):
def get_account_ring(self):
"""Get the account ring. Load it if it hasn't been yet."""
if not self.account_ring:
self.logger.debug(
_('Loading account ring from %s'), self.account_ring_path)
self.account_ring = Ring(self.account_ring_path)
self.account_ring = Ring(self.swift_dir, ring_name='account')
return self.account_ring
def get_paths(self):

2
swift/obj/replicator.py
View File

@ -236,7 +236,7 @@ class ObjectReplicator(Daemon):
self.port = int(conf.get('bind_port', 6000))
self.concurrency = int(conf.get('concurrency', 1))
self.stats_interval = int(conf.get('stats_interval', '300'))
self.object_ring = Ring(join(self.swift_dir, 'object.ring.gz'))
self.object_ring = Ring(self.swift_dir, ring_name='object')
self.ring_check_interval = int(conf.get('ring_check_interval', 15))
self.next_check = time.time() + self.ring_check_interval
self.reclaim_age = int(conf.get('reclaim_age', 86400 * 7))

7
swift/obj/updater.py
View File

@ -39,9 +39,8 @@ class ObjectUpdater(Daemon):
self.devices = conf.get('devices', '/srv/node')
self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y')
swift_dir = conf.get('swift_dir', '/etc/swift')
self.swift_dir = conf.get('swift_dir', '/etc/swift')
self.interval = int(conf.get('interval', 300))
self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz')
self.container_ring = None
self.concurrency = int(conf.get('concurrency', 1))
self.slowdown = float(conf.get('slowdown', 0.01))
@ -53,9 +52,7 @@ class ObjectUpdater(Daemon):
def get_container_ring(self):
"""Get the container ring. Load it, if it hasn't been yet."""
if not self.container_ring:
self.logger.debug(
_('Loading container ring from %s'), self.container_ring_path)
self.container_ring = Ring(self.container_ring_path)
self.container_ring = Ring(self.swift_dir, ring_name='container')
return self.container_ring
def run_forever(self, *args, **kwargs):

11
swift/proxy/server.py
View File

@ -1745,12 +1745,11 @@ class BaseApplication(object):
conf.get('object_post_as_copy', 'true').lower() in TRUE_VALUES
self.resellers_conf = ConfigParser()
self.resellers_conf.read(os.path.join(swift_dir, 'resellers.conf'))
self.object_ring = object_ring or \
Ring(os.path.join(swift_dir, 'object.ring.gz'))
self.container_ring = container_ring or \
Ring(os.path.join(swift_dir, 'container.ring.gz'))
self.account_ring = account_ring or \
Ring(os.path.join(swift_dir, 'account.ring.gz'))
self.object_ring = object_ring or Ring(swift_dir, ring_name='object')
self.container_ring = container_ring or Ring(swift_dir,
ring_name='container')
self.account_ring = account_ring or Ring(swift_dir,
ring_name='account')
self.memcache = memcache
mimetypes.init(mimetypes.knownfiles +
[os.path.join(swift_dir, 'mime.types')])

18
test/unit/common/ring/test_ring.py
View File

@ -48,7 +48,7 @@ class TestRing(unittest.TestCase):
self.testdir = os.path.join(os.path.dirname(__file__), 'ring')
rmtree(self.testdir, ignore_errors=1)
os.mkdir(self.testdir)
self.testgz = os.path.join(self.testdir, 'ring.gz')
self.testgz = os.path.join(self.testdir, 'whatever.ring.gz')
self.intended_replica2part2dev_id = [[0, 2, 0, 2], [2, 0, 2, 0]]
self.intended_devs = [{'id': 0, 'zone': 0, 'weight': 1.0}, None,
{'id': 2, 'zone': 2, 'weight': 1.0}]
@ -57,8 +57,8 @@ class TestRing(unittest.TestCase):
pickle.dump(ring.RingData(self.intended_replica2part2dev_id,
self.intended_devs, self.intended_part_shift),
GzipFile(self.testgz, 'wb'))
self.ring = \
ring.Ring(self.testgz, reload_time=self.intended_reload_time)
self.ring = ring.Ring(self.testdir,
reload_time=self.intended_reload_time, ring_name='whatever')
def tearDown(self):
rmtree(self.testdir, ignore_errors=1)
@ -74,7 +74,7 @@ class TestRing(unittest.TestCase):
_orig_hash_path_suffix = utils.HASH_PATH_SUFFIX
try:
utils.HASH_PATH_SUFFIX = ''
self.assertRaises(SystemExit, ring.Ring, self.testgz)
self.assertRaises(SystemExit, ring.Ring, self.testdir, 'whatever')
finally:
utils.HASH_PATH_SUFFIX = _orig_hash_path_suffix
@ -85,7 +85,8 @@ class TestRing(unittest.TestCase):
def test_reload(self):
os.utime(self.testgz, (time() - 300, time() - 300))
self.ring = ring.Ring(self.testgz, reload_time=0.001)
self.ring = ring.Ring(self.testdir, reload_time=0.001,
ring_name='whatever')
orig_mtime = self.ring._mtime
self.assertEquals(len(self.ring.devs), 3)
self.intended_devs.append({'id': 3, 'zone': 3, 'weight': 1.0})
@ -98,7 +99,8 @@ class TestRing(unittest.TestCase):
self.assertNotEquals(self.ring._mtime, orig_mtime)
os.utime(self.testgz, (time() - 300, time() - 300))
self.ring = ring.Ring(self.testgz, reload_time=0.001)
self.ring = ring.Ring(self.testdir, reload_time=0.001,
ring_name='whatever')
orig_mtime = self.ring._mtime
self.assertEquals(len(self.ring.devs), 4)
self.intended_devs.append({'id': 4, 'zone': 4, 'weight': 1.0})
@ -111,8 +113,8 @@ class TestRing(unittest.TestCase):
self.assertNotEquals(self.ring._mtime, orig_mtime)
os.utime(self.testgz, (time() - 300, time() - 300))
self.ring = \
ring.Ring(self.testgz, reload_time=0.001)
self.ring = ring.Ring(self.testdir, reload_time=0.001,
ring_name='whatever')
orig_mtime = self.ring._mtime
part, nodes = self.ring.get_nodes('a')
self.assertEquals(len(self.ring.devs), 5)

2
test/unit/common/test_db_replicator.py
View File

@ -36,7 +36,7 @@ def lock_parent_directory(filename):
class FakeRing:
class Ring:
devs = []
def __init__(self, path):
def __init__(self, path, reload_time=15, ring_name=None):
pass
def get_part_nodes(self, part):
return []

2
test/unit/obj/test_replicator.py
View File

@ -126,7 +126,7 @@ def _create_test_ring(path):
pickle.dump(ring.RingData(intended_replica2part2dev_id,
intended_devs, intended_part_shift),
GzipFile(testgz, 'wb'))
return ring.Ring(testgz, reload_time=intended_reload_time)
return ring.Ring(path, ring_name='object', reload_time=intended_reload_time)
class TestObjectReplicator(unittest.TestCase):