From a979c8007bf7376541c932c3e91d5ba1a95f4481 Mon Sep 17 00:00:00 2001
From: David Hadas <davidh@il.ibm.com>
Date: Wed, 20 Mar 2013 01:35:41 +0200
Subject: [PATCH] Add support for Hash Prefix
A new configuration parameter is added to /etc/swift/swift.conf
[swift-hash]
swift_hash_path_prefix = 'random unique string'
New installations are advised to set this parameter to a random secret,
which would not be disclosed ouside the organization.
The same secret needs to be used by all swift servers of the same cluster.
Existing installations should set this parameter to an empty string
(the default)
DocImpact
Fixes: Bug #1157454
Change-Id: I63b10d0b7d6dd3f74e0f10bb41b5f240fa03578a
---
doc/source/development_saio.rst | 3 +-
doc/source/howto_installmultinode.rst | 3 +-
swift/common/utils.py | 17 +++++++---
.../common/middleware/test_list_endpoints.py | 1 +
test/unit/common/ring/test_ring.py | 4 +++
test/unit/common/test_daemon.py | 1 +
test/unit/common/test_utils.py | 33 ++++++++++++-------
test/unit/container/test_sync.py | 1 +
test/unit/container/test_updater.py | 1 +
test/unit/obj/test_replicator.py | 1 +
test/unit/obj/test_server.py | 7 ++++
test/unit/obj/test_updater.py | 1 +
12 files changed, 55 insertions(+), 18 deletions(-)
diff --git a/doc/source/development_saio.rst b/doc/source/development_saio.rst
index 9d71fbd389..4cc661e7a0 100644
--- a/doc/source/development_saio.rst
+++ b/doc/source/development_saio.rst
@@ -318,7 +318,8 @@ Sample configuration files are provided with all defaults in line-by-line commen
#. Create `/etc/swift/swift.conf`::
[swift-hash]
- # random unique string that can never change (DO NOT LOSE)
+ # random unique strings that can never change (DO NOT LOSE)
+ swift_hash_path_prefix = changeme
swift_hash_path_suffix = changeme
#. Create `/etc/swift/account-server/1.conf`::
diff --git a/doc/source/howto_installmultinode.rst b/doc/source/howto_installmultinode.rst
index 0a6681725c..f4174cceb4 100644
--- a/doc/source/howto_installmultinode.rst
+++ b/doc/source/howto_installmultinode.rst
@@ -73,7 +73,8 @@ General OS configuration and partitioning for each node
cat >/etc/swift/swift.conf <<EOF
[swift-hash]
- # random unique string that can never change (DO NOT LOSE)
+ # random unique strings that can never change (DO NOT LOSE)
+ swift_hash_path_prefix = `od -t x8 -N 8 -A n </dev/random`
swift_hash_path_suffix = `od -t x8 -N 8 -A n </dev/random`
EOF
diff --git a/swift/common/utils.py b/swift/common/utils.py
index 74b7872488..c35569bdfd 100644
--- a/swift/common/utils.py
+++ b/swift/common/utils.py
@@ -76,12 +76,18 @@ FALLOCATE_RESERVE = 0
# will end up with would also require knowing this suffix.
hash_conf = ConfigParser()
HASH_PATH_SUFFIX = ''
+HASH_PATH_PREFIX = ''
if hash_conf.read('/etc/swift/swift.conf'):
try:
HASH_PATH_SUFFIX = hash_conf.get('swift-hash',
'swift_hash_path_suffix')
except (NoSectionError, NoOptionError):
pass
+ try:
+ HASH_PATH_PREFIX = hash_conf.get('swift-hash',
+ 'swift_hash_path_prefix')
+ except (NoSectionError, NoOptionError):
+ pass
def backward(f, blocksize=4096):
@@ -134,8 +140,9 @@ def noop_libc_function(*args):
def validate_configuration():
- if HASH_PATH_SUFFIX == '':
- sys.exit("Error: [swift-hash]: swift_hash_path_suffix missing "
+ if not HASH_PATH_SUFFIX and not HASH_PATH_PREFIX:
+ sys.exit("Error: [swift-hash]: both swift_hash_path_suffix "
+ "and swift_hash_path_prefix are missing "
"from /etc/swift/swift.conf")
@@ -991,9 +998,11 @@ def hash_path(account, container=None, object=None, raw_digest=False):
if object:
paths.append(object)
if raw_digest:
- return md5('/' + '/'.join(paths) + HASH_PATH_SUFFIX).digest()
+ return md5(HASH_PATH_PREFIX + '/' + '/'.join(paths)
+ + HASH_PATH_SUFFIX).digest()
else:
- return md5('/' + '/'.join(paths) + HASH_PATH_SUFFIX).hexdigest()
+ return md5(HASH_PATH_PREFIX + '/' + '/'.join(paths)
+ + HASH_PATH_SUFFIX).hexdigest()
@contextmanager
diff --git a/test/unit/common/middleware/test_list_endpoints.py b/test/unit/common/middleware/test_list_endpoints.py
index 642c01f9d1..eaafd0397c 100644
--- a/test/unit/common/middleware/test_list_endpoints.py
+++ b/test/unit/common/middleware/test_list_endpoints.py
@@ -36,6 +36,7 @@ def start_response(*args):
class TestListEndpoints(unittest.TestCase):
def setUp(self):
utils.HASH_PATH_SUFFIX = 'endcap'
+ utils.HASH_PATH_PREFIX = ''
self.testdir = os.path.join(os.path.dirname(__file__), 'ring')
rmtree(self.testdir, ignore_errors=1)
os.mkdir(self.testdir)
diff --git a/test/unit/common/ring/test_ring.py b/test/unit/common/ring/test_ring.py
index 00606d5666..3646eda832 100644
--- a/test/unit/common/ring/test_ring.py
+++ b/test/unit/common/ring/test_ring.py
@@ -97,6 +97,7 @@ class TestRing(unittest.TestCase):
def setUp(self):
utils.HASH_PATH_SUFFIX = 'endcap'
+ utils.HASH_PATH_PREFIX = ''
self.testdir = os.path.join(os.path.dirname(__file__), 'ring')
rmtree(self.testdir, ignore_errors=1)
os.mkdir(self.testdir)
@@ -133,11 +134,14 @@ class TestRing(unittest.TestCase):
self.assertEquals(self.ring.serialized_path, self.testgz)
# test invalid endcap
_orig_hash_path_suffix = utils.HASH_PATH_SUFFIX
+ _orig_hash_path_prefix = utils.HASH_PATH_PREFIX
try:
utils.HASH_PATH_SUFFIX = ''
+ utils.HASH_PATH_PREFIX = ''
self.assertRaises(SystemExit, ring.Ring, self.testdir, 'whatever')
finally:
utils.HASH_PATH_SUFFIX = _orig_hash_path_suffix
+ utils.HASH_PATH_PREFIX = _orig_hash_path_prefix
def test_has_changed(self):
self.assertEquals(self.ring.has_changed(), False)
diff --git a/test/unit/common/test_daemon.py b/test/unit/common/test_daemon.py
index f2da86965b..267d2cecad 100644
--- a/test/unit/common/test_daemon.py
+++ b/test/unit/common/test_daemon.py
@@ -62,6 +62,7 @@ class TestRunDaemon(unittest.TestCase):
def setUp(self):
utils.HASH_PATH_SUFFIX = 'endcap'
+ utils.HASH_PATH_PREFIX = 'startcap'
utils.drop_privileges = lambda *args: None
utils.capture_stdio = lambda *args: None
diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py
index c817c725b1..094a7301a8 100644
--- a/test/unit/common/test_utils.py
+++ b/test/unit/common/test_utils.py
@@ -127,6 +127,7 @@ class TestUtils(unittest.TestCase):
def setUp(self):
utils.HASH_PATH_SUFFIX = 'endcap'
+ utils.HASH_PATH_PREFIX = 'startcap'
def test_normalize_timestamp(self):
""" Test swift.common.utils.normalize_timestamp """
@@ -626,20 +627,28 @@ class TestUtils(unittest.TestCase):
self.assert_('127.0.0.1' in myips)
def test_hash_path(self):
+ _prefix = utils.HASH_PATH_PREFIX
+ utils.HASH_PATH_PREFIX = ''
# Yes, these tests are deliberately very fragile. We want to make sure
# that if someones changes the results hash_path produces, they know it
- self.assertEquals(utils.hash_path('a'),
- '1c84525acb02107ea475dcd3d09c2c58')
- self.assertEquals(utils.hash_path('a', 'c'),
- '33379ecb053aa5c9e356c68997cbb59e')
- self.assertEquals(utils.hash_path('a', 'c', 'o'),
- '06fbf0b514e5199dfc4e00f42eb5ea83')
- self.assertEquals(utils.hash_path('a', 'c', 'o', raw_digest=False),
- '06fbf0b514e5199dfc4e00f42eb5ea83')
- self.assertEquals(utils.hash_path('a', 'c', 'o', raw_digest=True),
- '\x06\xfb\xf0\xb5\x14\xe5\x19\x9d\xfcN'
- '\x00\xf4.\xb5\xea\x83')
- self.assertRaises(ValueError, utils.hash_path, 'a', object='o')
+ try:
+ self.assertEquals(utils.hash_path('a'),
+ '1c84525acb02107ea475dcd3d09c2c58')
+ self.assertEquals(utils.hash_path('a', 'c'),
+ '33379ecb053aa5c9e356c68997cbb59e')
+ self.assertEquals(utils.hash_path('a', 'c', 'o'),
+ '06fbf0b514e5199dfc4e00f42eb5ea83')
+ self.assertEquals(utils.hash_path('a', 'c', 'o', raw_digest=False),
+ '06fbf0b514e5199dfc4e00f42eb5ea83')
+ self.assertEquals(utils.hash_path('a', 'c', 'o', raw_digest=True),
+ '\x06\xfb\xf0\xb5\x14\xe5\x19\x9d\xfcN'
+ '\x00\xf4.\xb5\xea\x83')
+ self.assertRaises(ValueError, utils.hash_path, 'a', object='o')
+ utils.HASH_PATH_PREFIX = 'abcdef'
+ self.assertEquals(utils.hash_path('a', 'c', 'o', raw_digest=False),
+ '363f9b535bfb7d17a43a46a358afca0e')
+ finally:
+ utils.HASH_PATH_PREFIX = _prefix
def test_load_libc_function(self):
self.assert_(callable(
diff --git a/test/unit/container/test_sync.py b/test/unit/container/test_sync.py
index 34d00c39c4..a3a36afecf 100644
--- a/test/unit/container/test_sync.py
+++ b/test/unit/container/test_sync.py
@@ -23,6 +23,7 @@ from swiftclient import ClientException
utils.HASH_PATH_SUFFIX = 'endcap'
+utils.HASH_PATH_PREFIX = 'endcap'
class FakeRing(object):
diff --git a/test/unit/container/test_updater.py b/test/unit/container/test_updater.py
index 6e1fe32bec..d21e4ed830 100644
--- a/test/unit/container/test_updater.py
+++ b/test/unit/container/test_updater.py
@@ -35,6 +35,7 @@ class TestContainerUpdater(unittest.TestCase):
def setUp(self):
utils.HASH_PATH_SUFFIX = 'endcap'
+ utils.HASH_PATH_PREFIX = 'startcap'
self.testdir = os.path.join(mkdtemp(), 'tmp_test_container_updater')
rmtree(self.testdir, ignore_errors=1)
os.mkdir(self.testdir)
diff --git a/test/unit/obj/test_replicator.py b/test/unit/obj/test_replicator.py
index 4924ad32a7..b5ca89e7c5 100644
--- a/test/unit/obj/test_replicator.py
+++ b/test/unit/obj/test_replicator.py
@@ -135,6 +135,7 @@ class TestObjectReplicator(unittest.TestCase):
def setUp(self):
utils.HASH_PATH_SUFFIX = 'endcap'
+ utils.HASH_PATH_PREFIX = ''
# Setup a test ring (stolen from common/test_ring.py)
self.testdir = tempfile.mkdtemp()
self.devices = os.path.join(self.testdir, 'node')
diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py
index 3f30991e2f..8ee266bf89 100755
--- a/test/unit/obj/test_server.py
+++ b/test/unit/obj/test_server.py
@@ -383,6 +383,7 @@ class TestObjectController(unittest.TestCase):
def setUp(self):
""" Set up for testing swift.object_server.ObjectController """
utils.HASH_PATH_SUFFIX = 'endcap'
+ utils.HASH_PATH_PREFIX = 'startcap'
self.testdir = \
os.path.join(mkdtemp(), 'tmp_test_object_server_ObjectController')
mkdirs(os.path.join(self.testdir, 'sda1', 'tmp'))
@@ -1808,6 +1809,8 @@ class TestObjectController(unittest.TestCase):
'x-trans-id': '-'}})
def test_async_update_saves_on_exception(self):
+ _prefix = utils.HASH_PATH_PREFIX
+ utils.HASH_PATH_PREFIX = ''
def fake_http_connect(*args):
raise Exception('test')
@@ -1820,6 +1823,7 @@ class TestObjectController(unittest.TestCase):
{'x-timestamp': '1', 'x-out': 'set'}, 'sda1')
finally:
object_server.http_connect = orig_http_connect
+ utils.HASH_PATH_PREFIX = _prefix
self.assertEquals(
pickle.load(open(os.path.join(self.testdir, 'sda1',
'async_pending', 'a83',
@@ -1828,6 +1832,8 @@ class TestObjectController(unittest.TestCase):
'container': 'c', 'obj': 'o', 'op': 'PUT'})
def test_async_update_saves_on_non_2xx(self):
+ _prefix = utils.HASH_PATH_PREFIX
+ utils.HASH_PATH_PREFIX = ''
def fake_http_connect(status):
@@ -1860,6 +1866,7 @@ class TestObjectController(unittest.TestCase):
'op': 'PUT'})
finally:
object_server.http_connect = orig_http_connect
+ utils.HASH_PATH_PREFIX = _prefix
def test_async_update_does_not_save_on_2xx(self):
diff --git a/test/unit/obj/test_updater.py b/test/unit/obj/test_updater.py
index 104325b6ac..27d9ef7d85 100644
--- a/test/unit/obj/test_updater.py
+++ b/test/unit/obj/test_updater.py
@@ -37,6 +37,7 @@ class TestObjectUpdater(unittest.TestCase):
def setUp(self):
utils.HASH_PATH_SUFFIX = 'endcap'
+ utils.HASH_PATH_PREFIX = ''
self.testdir = os.path.join(os.path.dirname(__file__),
'object_updater')
rmtree(self.testdir, ignore_errors=1)