From cc00bd40e0726b89ff460053e7bfb54e5055c038 Mon Sep 17 00:00:00 2001
From: gholt <gholt@rackspace.com>
Date: Mon, 24 Jan 2011 17:12:38 -0800
Subject: [PATCH 1/3] Fix tests to cleanup their /tmp dirs
---
test/unit/container/test_server.py | 2 +-
test/unit/container/test_updater.py | 2 +-
test/unit/obj/test_auditor.py | 30 +++++++++++++----------------
test/unit/obj/test_server.py | 2 +-
test/unit/proxy/test_server.py | 8 +++++---
5 files changed, 21 insertions(+), 23 deletions(-)
diff --git a/test/unit/container/test_server.py b/test/unit/container/test_server.py
index 62d62a1f87..194ffff83d 100644
--- a/test/unit/container/test_server.py
+++ b/test/unit/container/test_server.py
@@ -45,7 +45,7 @@ class TestContainerController(unittest.TestCase):
def tearDown(self):
""" Tear down for testing swift.object_server.ObjectController """
- rmtree(self.testdir, ignore_errors=1)
+ rmtree(os.path.dirname(self.testdir), ignore_errors=1)
def test_acl_container(self):
# Ensure no acl by default
diff --git a/test/unit/container/test_updater.py b/test/unit/container/test_updater.py
index b7dbe6dd6d..a7a2094824 100644
--- a/test/unit/container/test_updater.py
+++ b/test/unit/container/test_updater.py
@@ -51,7 +51,7 @@ class TestContainerUpdater(unittest.TestCase):
os.mkdir(self.sda1)
def tearDown(self):
- rmtree(self.testdir, ignore_errors=1)
+ rmtree(os.path.dirname(self.testdir), ignore_errors=1)
def test_creation(self):
cu = container_updater.ContainerUpdater({
diff --git a/test/unit/obj/test_auditor.py b/test/unit/obj/test_auditor.py
index 8c1d08e522..66540a3693 100644
--- a/test/unit/obj/test_auditor.py
+++ b/test/unit/obj/test_auditor.py
@@ -56,7 +56,7 @@ class TestAuditor(unittest.TestCase):
mount_check='false')
def tearDown(self):
- rmtree(self.testdir, ignore_errors=1)
+ rmtree(os.path.dirname(self.testdir), ignore_errors=1)
def test_object_audit_extra_data(self):
self.auditor = auditor.ObjectAuditor(self.conf)
@@ -123,25 +123,21 @@ class TestAuditor(unittest.TestCase):
self.assertEquals(self.auditor.quarantines, pre_quarantines + 1)
def test_object_audit_no_meta(self):
- self.auditor = auditor.ObjectAuditor(self.conf)
cur_part = '0'
disk_file = DiskFile(self.devices, 'sda', cur_part, 'a', 'c', 'o')
- data = '0' * 1024
- etag = md5()
+ timestamp = str(normalize_timestamp(time.time()))
+ path = os.path.join(disk_file.datadir, timestamp + '.data')
+ mkdirs(disk_file.datadir)
+ fp = open(path, 'w')
+ fp.write('0' * 1024)
+ fp.close()
+ invalidate_hash(os.path.dirname(disk_file.datadir))
+ self.auditor = auditor.ObjectAuditor(self.conf)
pre_quarantines = self.auditor.quarantines
- with disk_file.mkstemp() as (fd, tmppath):
- os.write(fd, data)
- etag.update(data)
- etag = etag.hexdigest()
- timestamp = str(normalize_timestamp(time.time()))
- os.fsync(fd)
- invalidate_hash(os.path.dirname(disk_file.datadir))
- renamer(tmppath, os.path.join(disk_file.datadir,
- timestamp + '.data'))
- self.auditor.object_audit(
- os.path.join(disk_file.datadir, timestamp + '.data'),
- 'sda', cur_part)
- self.assertEquals(self.auditor.quarantines, pre_quarantines + 1)
+ self.auditor.object_audit(
+ os.path.join(disk_file.datadir, timestamp + '.data'),
+ 'sda', cur_part)
+ self.assertEquals(self.auditor.quarantines, pre_quarantines + 1)
def test_object_audit_bad_args(self):
self.auditor = auditor.ObjectAuditor(self.conf)
diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py
index 735b944d2d..22d2fe20a4 100644
--- a/test/unit/obj/test_server.py
+++ b/test/unit/obj/test_server.py
@@ -53,7 +53,7 @@ class TestObjectController(unittest.TestCase):
def tearDown(self):
""" Tear down for testing swift.object_server.ObjectController """
- rmtree(self.testdir)
+ rmtree(os.path.dirname(self.testdir))
def test_POST_update_meta(self):
""" Test swift.object_server.ObjectController.POST """
diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py
index b327b99a46..1a3a442bea 100644
--- a/test/unit/proxy/test_server.py
+++ b/test/unit/proxy/test_server.py
@@ -142,7 +142,7 @@ def teardown():
for server in _test_coros:
server.kill()
proxy_server.CONTAINER_LISTING_LIMIT = _orig_container_listing_limit
- rmtree(_testdir)
+ rmtree(os.path.dirname(_testdir))
def fake_http_connect(*code_iter, **kwargs):
@@ -3425,5 +3425,7 @@ class TestSegmentedIterable(unittest.TestCase):
if __name__ == '__main__':
setup()
- unittest.main()
- teardown()
+ try:
+ unittest.main()
+ finally:
+ teardown()
From f57250ae11314424b393c99e1bd9e9d0049f6434 Mon Sep 17 00:00:00 2001
From: gholt <gholt@rackspace.com>
Date: Mon, 24 Jan 2011 21:39:00 -0800
Subject: [PATCH 2/3] pep8 and i18n fixes
---
swift/auth/server.py | 60 +++++++++++++++----------
swift/common/middleware/auth.py | 10 +++--
swift/common/middleware/domain_remap.py | 3 +-
swift/obj/replicator.py | 3 +-
swift/stats/account_stats.py | 7 +--
swift/stats/log_processor.py | 3 +-
6 files changed, 51 insertions(+), 35 deletions(-)
diff --git a/swift/auth/server.py b/swift/auth/server.py
index 413c03cce1..8f1483ab27 100644
--- a/swift/auth/server.py
+++ b/swift/auth/server.py
@@ -149,31 +149,32 @@ class AuthController(object):
previous_prefix = ''
if '_' in row[0]:
previous_prefix = row[0].split('_', 1)[0]
- msg = _(('''
+ msg = (_('''
THERE ARE ACCOUNTS IN YOUR auth.db THAT DO NOT BEGIN WITH YOUR NEW RESELLER
-PREFIX OF "%s".
+PREFIX OF "%(reseller)s".
YOU HAVE A FEW OPTIONS:
- 1) RUN "swift-auth-update-reseller-prefixes %s %s",
+ 1. RUN "swift-auth-update-reseller-prefixes %(db_file)s %(reseller)s",
"swift-init auth-server restart", AND
"swift-auth-recreate-accounts -K ..." TO CREATE FRESH ACCOUNTS.
OR
- 2) REMOVE %s, RUN "swift-init auth-server restart", AND RUN
+ 2. REMOVE %(db_file)s, RUN "swift-init auth-server restart", AND RUN
"swift-auth-add-user ..." TO CREATE BRAND NEW ACCOUNTS THAT WAY.
OR
- 3) ADD "reseller_prefix = %s" (WITHOUT THE QUOTES) TO YOUR
+ 3. ADD "reseller_prefix = %(previous)s" (WITHOUT THE QUOTES) TO YOUR
proxy-server.conf IN THE [filter:auth] SECTION AND TO YOUR
auth-server.conf IN THE [app:auth-server] SECTION AND RUN
"swift-init proxy-server restart" AND "swift-init auth-server restart"
TO REVERT BACK TO YOUR PREVIOUS RESELLER PREFIX.
- %s
- ''') % (self.reseller_prefix.rstrip('_'), self.db_file,
- self.reseller_prefix.rstrip('_'), self.db_file,
- previous_prefix, previous_prefix and ' ' or _('''
+ %(note)s
+ ''') % {'reseller': self.reseller_prefix.rstrip('_'),
+ 'db_file': self.db_file,
+ 'previous': previous_prefix,
+ 'note': previous_prefix and ' ' or _('''
SINCE YOUR PREVIOUS RESELLER PREFIX WAS AN EMPTY STRING, IT IS NOT
RECOMMENDED TO PERFORM OPTION 3 AS THAT WOULD MAKE SUPPORTING MULTIPLE
RESELLERS MORE DIFFICULT.
- ''').strip())).strip()
+ ''').strip()}).strip()
self.logger.critical(_('CRITICAL: ') + ' '.join(msg.split()))
raise Exception('\n' + msg)
@@ -243,7 +244,8 @@ YOU HAVE A FEW OPTIONS:
raise err
def validate_s3_sign(self, request, token):
- account, user, sign = request.headers['Authorization'].split(' ')[-1].split(':')
+ account, user, sign = \
+ request.headers['Authorization'].split(' ')[-1].split(':')
msg = base64.urlsafe_b64decode(unquote(token))
rv = False
with self.get_conn() as conn:
@@ -253,7 +255,8 @@ YOU HAVE A FEW OPTIONS:
(account, user)).fetchone()
rv = (84000, account, user, row[1])
if rv:
- s = base64.encodestring(hmac.new(row[0], msg, sha1).digest()).strip()
+ s = base64.encodestring(hmac.new(row[0], msg,
+ sha1).digest()).strip()
self.logger.info("orig %s, calc %s" % (sign, s))
if sign != s:
rv = False
@@ -340,10 +343,14 @@ YOU HAVE A FEW OPTIONS:
'SELECT url FROM account WHERE account = ? AND user = ?',
(account, user)).fetchone()
if row:
- self.logger.info(
- _('ALREADY EXISTS create_user(%s, %s, _, %s, %s) [%.02f]') %
- (repr(account), repr(user), repr(admin),
- repr(reseller_admin), time() - begin))
+ self.logger.info(_('ALREADY EXISTS create_user(%(account)s, '
+ '%(user)s, _, %(admin)s, %(reseller_admin)s) '
+ '[%(elapsed).02f]') %
+ {'account': repr(account),
+ 'user': repr(user),
+ 'admin': repr(admin),
+ 'reseller_admin': repr(reseller_admin),
+ 'elapsed': time() - begin})
return 'already exists'
row = conn.execute(
'SELECT url, cfaccount FROM account WHERE account = ?',
@@ -354,10 +361,14 @@ YOU HAVE A FEW OPTIONS:
else:
account_hash = self.add_storage_account()
if not account_hash:
- self.logger.info(
- _('FAILED create_user(%s, %s, _, %s, %s) [%.02f]') %
- (repr(account), repr(user), repr(admin),
- repr(reseller_admin), time() - begin))
+ self.logger.info(_('FAILED create_user(%(account)s, '
+ '%(user)s, _, %(admin)s, %(reseller_admin)s) '
+ '[%(elapsed).02f]') %
+ {'account': repr(account),
+ 'user': repr(user),
+ 'admin': repr(admin),
+ 'reseller_admin': repr(reseller_admin),
+ 'elapsed': time() - begin})
return False
url = self.default_cluster_url.rstrip('/') + '/' + account_hash
conn.execute('''INSERT INTO account
@@ -367,10 +378,11 @@ YOU HAVE A FEW OPTIONS:
(account, url, account_hash, user, password,
admin and 't' or '', reseller_admin and 't' or ''))
conn.commit()
- self.logger.info(
- _('SUCCESS create_user(%s, %s, _, %s, %s) = %s [%.02f]') %
- (repr(account), repr(user), repr(admin), repr(reseller_admin),
- repr(url), time() - begin))
+ self.logger.info(_('SUCCESS create_user(%(account)s, %(user)s, _, '
+ '%(admin)s, %(reseller_admin)s) = %(url)s [%(elapsed).02f]') %
+ {'account': repr(account), 'user': repr(user),
+ 'admin': repr(admin), 'reseller_admin': repr(reseller_admin),
+ 'url': repr(url), 'elapsed': time() - begin})
return url
def recreate_accounts(self):
diff --git a/swift/common/middleware/auth.py b/swift/common/middleware/auth.py
index a2c71a3070..a51788f7b7 100644
--- a/swift/common/middleware/auth.py
+++ b/swift/common/middleware/auth.py
@@ -59,8 +59,8 @@ class DevAuth(object):
if s3 or (token and token.startswith(self.reseller_prefix)):
# Note: Empty reseller_prefix will match all tokens.
# Attempt to auth my token with my auth server
- groups = \
- self.get_groups(env, token, memcache_client=cache_from_env(env))
+ groups = self.get_groups(env, token,
+ memcache_client=cache_from_env(env))
if groups:
env['REMOTE_USER'] = groups
user = groups and groups.split(',', 1)[0] or ''
@@ -154,10 +154,12 @@ class DevAuth(object):
timeout=expiration)
if env.get('HTTP_AUTHORIZATION'):
- account, user, sign = env['HTTP_AUTHORIZATION'].split(' ')[-1].split(':')
+ account, user, sign = \
+ env['HTTP_AUTHORIZATION'].split(' ')[-1].split(':')
cfaccount = resp.getheader('x-auth-account-suffix')
path = env['PATH_INFO']
- env['PATH_INFO'] = path.replace("%s:%s" % (account, user), cfaccount, 1)
+ env['PATH_INFO'] = \
+ path.replace("%s:%s" % (account, user), cfaccount, 1)
return groups
diff --git a/swift/common/middleware/domain_remap.py b/swift/common/middleware/domain_remap.py
index e959276733..a6ed943bb2 100644
--- a/swift/common/middleware/domain_remap.py
+++ b/swift/common/middleware/domain_remap.py
@@ -35,7 +35,7 @@ class DomainRemapMiddleware(object):
instead of the found reseller prefix. The reseller_prefixes list is
exclusive. If defined, any request with an account prefix not in that list
will be ignored by this middleware. reseller_prefixes defaults to 'AUTH'.
-
+
Note that this middleware requires that container names and account names
(except as described above) must be DNS-compatible. This means that the
account name created in the system and the containers created by users
@@ -111,4 +111,3 @@ def filter_factory(global_conf, **local_conf):
def domain_filter(app):
return DomainRemapMiddleware(app, conf)
return domain_filter
-
diff --git a/swift/obj/replicator.py b/swift/obj/replicator.py
index ec76fb384c..fc7e7cbd07 100644
--- a/swift/obj/replicator.py
+++ b/swift/obj/replicator.py
@@ -407,7 +407,8 @@ class ObjectReplicator(Daemon):
conn.getresponse().read()
self.suffix_sync += len(suffixes)
except (Exception, Timeout):
- self.logger.exception(_("Error syncing with node: %s") % node)
+ self.logger.exception(_("Error syncing with node: %s") %
+ node)
self.suffix_count += len(local_hash)
except (Exception, Timeout):
self.logger.exception(_("Error syncing partition"))
diff --git a/swift/stats/account_stats.py b/swift/stats/account_stats.py
index 859151d238..6a9688831f 100644
--- a/swift/stats/account_stats.py
+++ b/swift/stats/account_stats.py
@@ -55,7 +55,8 @@ class AccountStat(Daemon):
self.logger.info(_("Gathering account stats"))
start = time.time()
self.find_and_process()
- self.logger.info(_("Gathering account stats complete (%0.2f minutes)") %
+ self.logger.info(
+ _("Gathering account stats complete (%0.2f minutes)") %
((time.time() - start) / 60))
def find_and_process(self):
@@ -70,8 +71,8 @@ class AccountStat(Daemon):
# Account Name, Container Count, Object Count, Bytes Used
for device in os.listdir(self.devices):
if self.mount_check and not check_mount(self.devices, device):
- self.logger.error(_("Device %s is not mounted, skipping.") %
- device)
+ self.logger.error(
+ _("Device %s is not mounted, skipping.") % device)
continue
accounts = os.path.join(self.devices,
device,
diff --git a/swift/stats/log_processor.py b/swift/stats/log_processor.py
index 8174fb265e..531ec3e0ac 100644
--- a/swift/stats/log_processor.py
+++ b/swift/stats/log_processor.py
@@ -280,7 +280,8 @@ class LogProcessorDaemon(Daemon):
logs_to_process = self.log_processor.get_data_list(lookback_start,
lookback_end,
already_processed_files)
- self.logger.info(_('loaded %d files to process') % len(logs_to_process))
+ self.logger.info(_('loaded %d files to process') %
+ len(logs_to_process))
if not logs_to_process:
self.logger.info(_("Log processing done (%0.2f minutes)") %
((time.time() - start) / 60))
From 1eaae6a074d934299cfa691105e4b974733397c5 Mon Sep 17 00:00:00 2001
From: Chuck Thier <cthier@gmail.com>
Date: Tue, 25 Jan 2011 16:19:38 +0000
Subject: [PATCH 3/3] Bumped version to 1.2 to get ready for release, and
updated the AUTHORS file
---
AUTHORS | 4 ++++
swift/__init__.py | 2 +-
2 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/AUTHORS b/AUTHORS
index e7bc59b72e..bf834db788 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -24,9 +24,13 @@ Paul Jimenez
Brian K. Jones
Ed Leafe
Stephen Milton
+Russ Nelson
+Colin Nicholson
Andrew Clay Shafer
Monty Taylor
Caleb Tennis
+FUJITA Tomonori
Kapil Thangavelu
Conrad Weidenkeller
+Chris Wedgwood
Cory Wright
diff --git a/swift/__init__.py b/swift/__init__.py
index 0bd0062056..899047889e 100644
--- a/swift/__init__.py
+++ b/swift/__init__.py
@@ -1,5 +1,5 @@
import gettext
-__version__ = '1.1.0'
+__version__ = '1.2.0'
gettext.install('swift')