From 92618c97638f66f6588e5729264637e46a592e7f Mon Sep 17 00:00:00 2001 From: gholt Date: Mon, 1 Aug 2011 16:08:55 +0000 Subject: [PATCH 1/5] More quarantine catching --- swift/common/db.py | 23 +++++++++++++++++++++ swift/obj/replicator.py | 45 ++++++++++++++++++++++++++++++++++++++--- swift/obj/server.py | 30 ++------------------------- 3 files changed, 67 insertions(+), 31 deletions(-) diff --git a/swift/common/db.py b/swift/common/db.py index 4a4c029cf7..9dbd0f9e14 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -270,6 +270,29 @@ class DatabaseBroker(object): yield conn conn.rollback() self.conn = conn + except sqlite3.DatabaseError, err: + try: + conn.close() + except: + pass + if 'database disk image is malformed' not in str(err): + raise + prefix_path = os.path.dirname(self.db_dir) + partition_path = os.path.dirname(prefix_path) + dbs_path = os.path.dirname(partition_path) + device_path = os.path.dirname(dbs_path) + quar_path = os.path.join(device_path, 'quarantined', self.db_type, + os.path.basename(self.db_dir)) + try: + renamer(self.db_dir, quar_path) + except OSError, e: + if e.errno not in (errno.EEXIST, errno.ENOTEMPTY): + raise + quar_path = "%s-%s" % (quar_path, uuid4().hex) + renamer(self.db_dir, quar_path) + self.logger(_('Quarantined %s to %s due to malformed database') % + (self.db_dir, quar_path)) + raise err except Exception: conn.close() raise diff --git a/swift/obj/replicator.py b/swift/obj/replicator.py index c65751e747..9d072bf9d4 100644 --- a/swift/obj/replicator.py +++ b/swift/obj/replicator.py @@ -14,7 +14,7 @@ # limitations under the License. import os -from os.path import isdir, join +from os.path import basename, dirname, isdir, join import random import shutil import time @@ -22,6 +22,8 @@ import logging import hashlib import itertools import cPickle as pickle +import errno +import uuid import eventlet from eventlet import GreenPool, tpool, Timeout, sleep, hubs @@ -30,7 +32,7 @@ from eventlet.support.greenlets import GreenletExit from swift.common.ring import Ring from swift.common.utils import whataremyips, unlink_older_than, lock_path, \ - compute_eta, get_logger, write_pickle + compute_eta, get_logger, write_pickle, renamer from swift.common.bufferedhttp import http_connect from swift.common.daemon import Daemon @@ -41,6 +43,31 @@ ONE_WEEK = 604800 HASH_FILE = 'hashes.pkl' +def quarantine_renamer(device_path, corrupted_file_path): + """ + In the case that a file is corrupted, move it to a quarantined + area to allow replication to fix it. + + :params device_path: The path to the device the corrupted file is on. + :params corrupted_file_path: The path to the file you want quarantined. + + :returns: path (str) of directory the file was moved to + :raises OSError: re-raises non errno.EEXIST / errno.ENOTEMPTY + exceptions from rename + """ + from_dir = dirname(corrupted_file_path) + to_dir = join(device_path, 'quarantined', 'objects', basename(from_dir)) + invalidate_hash(dirname(from_dir)) + try: + renamer(from_dir, to_dir) + except OSError, e: + if e.errno not in (errno.EEXIST, errno.ENOTEMPTY): + raise + to_dir = "%s-%s" % (to_dir, uuid.uuid4().hex) + renamer(from_dir, to_dir) + return to_dir + + def hash_suffix(path, reclaim_age): """ Performs reclamation and returns an md5 of all (remaining) files. @@ -50,7 +77,19 @@ def hash_suffix(path, reclaim_age): md5 = hashlib.md5() for hsh in sorted(os.listdir(path)): hsh_path = join(path, hsh) - files = os.listdir(hsh_path) + try: + files = os.listdir(hsh_path) + except OSError, err: + if err.ernno == errno.ENOTDIR: + partition_path = dirname(path) + objects_path = dirname(partition_path) + device_path = dirname(objects_path) + quar_path = quarantine_renamer(device_path, hsh_path) + logging.exception( + _('Quarantined %s to %s because it is not a directory') % + (hsh_path, quar_path)) + continue + raise if len(files) == 1: if files[0].endswith('.ts'): # remove tombstones older than reclaim_age diff --git a/swift/obj/server.py b/swift/obj/server.py index 470ef4e8cc..6b2e79319e 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -21,7 +21,6 @@ import errno import os import time import traceback -import uuid from datetime import datetime from hashlib import md5 from tempfile import mkstemp @@ -44,7 +43,8 @@ from swift.common.constraints import check_object_creation, check_mount, \ check_float, check_utf8 from swift.common.exceptions import ConnectionTimeout, DiskFileError, \ DiskFileNotExist -from swift.obj.replicator import tpooled_get_hashes, invalidate_hash +from swift.obj.replicator import tpooled_get_hashes, invalidate_hash, \ + quarantine_renamer DATADIR = 'objects' @@ -91,32 +91,6 @@ def write_metadata(fd, metadata): key += 1 -def quarantine_renamer(device_path, corrupted_file_path): - """ - In the case that a file is corrupted, move it to a quarantined - area to allow replication to fix it. - - :params device_path: The path to the device the corrupted file is on. - :params corrupted_file_path: The path to the file you want quarantined. - - :returns: path (str) of directory the file was moved to - :raises OSError: re-raises non errno.EEXIST / errno.ENOTEMPTY - exceptions from rename - """ - from_dir = os.path.dirname(corrupted_file_path) - to_dir = os.path.join(device_path, 'quarantined', - 'objects', os.path.basename(from_dir)) - invalidate_hash(os.path.dirname(from_dir)) - try: - renamer(from_dir, to_dir) - except OSError, e: - if e.errno not in (errno.EEXIST, errno.ENOTEMPTY): - raise - to_dir = "%s-%s" % (to_dir, uuid.uuid4().hex) - renamer(from_dir, to_dir) - return to_dir - - class DiskFile(object): """ Manage object files on disk. From 671117e1d999875d5e7dfe79504d0aaf471890f0 Mon Sep 17 00:00:00 2001 From: gholt Date: Mon, 1 Aug 2011 20:46:30 +0000 Subject: [PATCH 2/5] Tests for db quarantining --- swift/common/db.py | 50 ++-- test/unit/common/test_db.py | 480 ++++++++++++++++++++++++++++++++++++ 2 files changed, 511 insertions(+), 19 deletions(-) diff --git a/swift/common/db.py b/swift/common/db.py index 9dbd0f9e14..2f939fbce4 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -22,6 +22,7 @@ import logging import operator import os from uuid import uuid4 +import sys import time import cPickle as pickle import errno @@ -256,12 +257,40 @@ class DatabaseBroker(object): self._delete_db(conn, timestamp) conn.commit() + def possibly_quarantine(self, exc_type, exc_value, exc_traceback): + if 'database disk image is malformed' in str(exc_value): + exc_hint = 'malformed' + elif 'file is encrypted or is not a database' in str(exc_value): + exc_hint = 'corrupted' + else: + raise exc_type, exc_value, exc_traceback + prefix_path = os.path.dirname(self.db_dir) + partition_path = os.path.dirname(prefix_path) + dbs_path = os.path.dirname(partition_path) + device_path = os.path.dirname(dbs_path) + quar_path = os.path.join(device_path, 'quarantined', self.db_type, + os.path.basename(self.db_dir)) + try: + renamer(self.db_dir, quar_path) + except OSError, e: + if e.errno not in (errno.EEXIST, errno.ENOTEMPTY): + raise + quar_path = "%s-%s" % (quar_path, uuid4().hex) + renamer(self.db_dir, quar_path) + detail = _('Quarantined %s to %s due to %s database') % \ + (self.db_dir, quar_path, exc_hint) + self.logger.error(detail) + raise sqlite3.DatabaseError(detail) + @contextmanager def get(self): """Use with the "with" statement; returns a database connection.""" if not self.conn: if self.db_file != ':memory:' and os.path.exists(self.db_file): - self.conn = get_db_connection(self.db_file, self.timeout) + try: + self.conn = get_db_connection(self.db_file, self.timeout) + except (sqlite3.DatabaseError, DatabaseConnectionError): + self.possibly_quarantine(*sys.exc_info()) else: raise DatabaseConnectionError(self.db_file, "DB doesn't exist") conn = self.conn @@ -275,24 +304,7 @@ class DatabaseBroker(object): conn.close() except: pass - if 'database disk image is malformed' not in str(err): - raise - prefix_path = os.path.dirname(self.db_dir) - partition_path = os.path.dirname(prefix_path) - dbs_path = os.path.dirname(partition_path) - device_path = os.path.dirname(dbs_path) - quar_path = os.path.join(device_path, 'quarantined', self.db_type, - os.path.basename(self.db_dir)) - try: - renamer(self.db_dir, quar_path) - except OSError, e: - if e.errno not in (errno.EEXIST, errno.ENOTEMPTY): - raise - quar_path = "%s-%s" % (quar_path, uuid4().hex) - renamer(self.db_dir, quar_path) - self.logger(_('Quarantined %s to %s due to malformed database') % - (self.db_dir, quar_path)) - raise err + self.possibly_quarantine(*sys.exc_info()) except Exception: conn.close() raise diff --git a/test/unit/common/test_db.py b/test/unit/common/test_db.py index 1c5bf2e65a..a5bfabe01c 100644 --- a/test/unit/common/test_db.py +++ b/test/unit/common/test_db.py @@ -27,6 +27,7 @@ from uuid import uuid4 import simplejson import sqlite3 +import swift.common.db from swift.common.db import AccountBroker, chexor, ContainerBroker, \ DatabaseBroker, DatabaseConnectionError, dict_factory, get_db_connection from swift.common.utils import normalize_timestamp @@ -199,6 +200,41 @@ class TestDatabaseBroker(unittest.TestCase): with broker.get() as conn: self.assertEquals( [r[0] for r in conn.execute('SELECT * FROM test')], ['1']) + orig_renamer = swift.common.db.renamer + try: + swift.common.db.renamer = lambda a, b: b + # Test malformed database + fp = open(os.path.join(self.testdir, '1.db'), 'wb') + fp.write(EXAMPLE_MALFORMED_DB.decode('hex')) + fp.close() + broker = DatabaseBroker(os.path.join(self.testdir, '1.db')) + broker.db_type = 'test' + exc = None + try: + with broker.get() as conn: + conn.execute('SELECT * FROM test') + except Exception, err: + exc = err + self.assertTrue(str(exc).startswith( + 'Quarantined test/unit/common/db to quarantined/test/db')) + self.assertTrue(str(exc).endswith(' due to malformed database')) + # Test corrupted database + fp = open(os.path.join(self.testdir, '1.db'), 'wb') + fp.write(EXAMPLE_CORRUPTED_DB.decode('hex')) + fp.close() + broker = DatabaseBroker(os.path.join(self.testdir, '1.db')) + broker.db_type = 'test' + exc = None + try: + with broker.get() as conn: + conn.execute('SELECT * FROM test') + except Exception, err: + exc = err + self.assertTrue(str(exc).startswith( + 'Quarantined test/unit/common/db to quarantined/test/db')) + self.assertTrue(str(exc).endswith(' due to corrupted database')) + finally: + swift.common.db.renamer = orig_renamer def test_lock(self): broker = DatabaseBroker(os.path.join(self.testdir, '1.db'), timeout=.1) @@ -1970,5 +2006,449 @@ class TestAccountBrokerBeforeMetadata(TestAccountBroker): conn.execute('SELECT metadata FROM account_stat') +EXAMPLE_MALFORMED_DB = \ +'53514c69746520666f726d6174203300040001016a402020000000080000000000000000000' \ +'000000000000700000001000000000000000000000001000000000000000000000000000000' \ +'00000000000000000000000000000000000000000000000000050000000103fb00000000070' \ +'3fb000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000006050d00000000040000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'00000000000000000000000000000000000000000000000a000000000400000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'0000000000000000000000000000000000000000000000000000000000000000000000d0000' \ +'000004000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000a00000000040000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'00000000000000000000000000000000000000000d0000000500dd0000dd01a601e102aa02e' \ +'500000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000008146010717272701824b7461626c656f7574676f696' \ +'e675f73796e636f7574676f696e675f73796e6302435245415445205441424c45206f757467' \ +'6f696e675f73796e6320280a2020202020202020202020202020202072656d6f74655f69642' \ +'05445585420554e495155452c0a2020202020202020202020202020202073796e635f706f69' \ +'6e7420494e54454745522c0a20202020202020202020202020202020757064617465645f617' \ +'420544558542044454641554c5420300a20202020202020202020202029390206174d270100' \ +'696e64657873716c6974655f6175746f696e6465785f6f7574676f696e675f73796e635f316' \ +'f7574676f696e675f73796e63038146030717272701824b7461626c65696e636f6d696e675f' \ +'73796e63696e636f6d696e675f73796e6304435245415445205441424c4520696e636f6d696' \ +'e675f73796e6320280a2020202020202020202020202020202072656d6f74655f6964205445' \ +'585420554e495155452c0a2020202020202020202020202020202073796e635f706f696e742' \ +'0494e54454745522c0a20202020202020202020202020202020757064617465645f61742054' \ +'4558542044454641554c5420300a20202020202020202020202029390406174d270100696e6' \ +'4657873716c6974655f6175746f696e6465785f696e636f6d696e675f73796e635f31696e63' \ +'6f6d696e675f73796e6305821805071b352701835d747269676765726f7574676f696e675f7' \ +'3796e635f696e736572746f7574676f696e675f73796e630043524541544520545249474745' \ +'52206f7574676f696e675f73796e635f696e7365727420414654455220494e53455254204f4' \ +'e206f7574676f696e675f73796e630a202020202020202020202020424547494e0a20202020' \ +'202020202020202020202020555044415445206f7574676f696e675f73796e630a202020202' \ +'0202020202020202020202053455420757064617465645f6174203d205354524654494d4528' \ +'272573272c20274e4f5727290a20202020202020202020202020202020574845524520524f5' \ +'74944203d206e65772e524f5749443b0a202020202020202020202020454e440d0000000400' \ +'7d0002e501ca00af007d0000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000030090617151501457461626c65746573747465737408435245415445205441' \ +'424c45207465737420286f6e65205445585429821808071b352701835d74726967676572696' \ +'e636f6d696e675f73796e635f757064617465696e636f6d696e675f73796e63004352454154' \ +'45205452494747455220696e636f6d696e675f73796e635f757064617465204146544552205' \ +'55044415445204f4e20696e636f6d696e675f73796e630a2020202020202020202020204245' \ +'47494e0a2020202020202020202020202020202055504441544520696e636f6d696e675f737' \ +'96e630a2020202020202020202020202020202053455420757064617465645f6174203d2053' \ +'54524654494d4528272573272c20274e4f5727290a202020202020202020202020202020205' \ +'74845524520524f574944203d206e65772e524f5749443b0a20202020202020202020202045' \ +'4e44821807071b352701835d74726967676572696e636f6d696e675f73796e635f696e73657' \ +'274696e636f6d696e675f73796e6300435245415445205452494747455220696e636f6d696e' \ +'675f73796e635f696e7365727420414654455220494e53455254204f4e20696e636f6d696e6' \ +'75f73796e630a202020202020202020202020424547494e0a20202020202020202020202020' \ +'20202055504441544520696e636f6d696e675f73796e630a202020202020202020202020202' \ +'0202053455420757064617465645f6174203d205354524654494d4528272573272c20274e4f' \ +'5727290a20202020202020202020202020202020574845524520524f574944203d206e65772' \ +'e524f5749443b0a202020202020202020202020454e44821806071b352701835d7472696767' \ +'65726f7574676f696e675f73796e635f7570646174656f7574676f696e675f73796e6300435' \ +'2454154452054524947474552206f7574676f696e675f73796e635f75706461746520414654' \ +'455220555044415445204f4e206f7574676f696e675f73796e630a202020202020202020202' \ +'020424547494e0a20202020202020202020202020202020555044415445206f7574676f696e' \ +'675f73796e630a2020202020202020202020202020202053455420757064617465645f61742' \ +'03d205354524654494d4528272573272c20274e4f5727290a20202020202020202020202020' \ +'202020574845524520524f574944203d206e65772e524f5749443b0a2020202020202020202' \ +'02020454e440d0000000103fb0003fb00000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'0000000000000000000000000301020f31' + + +EXAMPLE_CORRUPTED_DB = \ +'6a756e6b746520666f726d61742033000400010100402020000000080000000000000000000' \ +'000000000000700000001000000000000000000000001000000000000000000000000000000' \ +'00000000000000000000000000000000000000000000000000050000000103fb00000000070' \ +'3fb000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000006050d00000000040000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'00000000000000000000000000000000000000000000000a000000000400000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'0000000000000000000000000000000000000000000000000000000000000000000000d0000' \ +'000004000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000a00000000040000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'00000000000000000000000000000000000000000d0000000500dd0000dd01a601e102aa02e' \ +'500000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000008146010717272701824b7461626c656f7574676f696' \ +'e675f73796e636f7574676f696e675f73796e6302435245415445205441424c45206f757467' \ +'6f696e675f73796e6320280a2020202020202020202020202020202072656d6f74655f69642' \ +'05445585420554e495155452c0a2020202020202020202020202020202073796e635f706f69' \ +'6e7420494e54454745522c0a20202020202020202020202020202020757064617465645f617' \ +'420544558542044454641554c5420300a20202020202020202020202029390206174d270100' \ +'696e64657873716c6974655f6175746f696e6465785f6f7574676f696e675f73796e635f316' \ +'f7574676f696e675f73796e63038146030717272701824b7461626c65696e636f6d696e675f' \ +'73796e63696e636f6d696e675f73796e6304435245415445205441424c4520696e636f6d696' \ +'e675f73796e6320280a2020202020202020202020202020202072656d6f74655f6964205445' \ +'585420554e495155452c0a2020202020202020202020202020202073796e635f706f696e742' \ +'0494e54454745522c0a20202020202020202020202020202020757064617465645f61742054' \ +'4558542044454641554c5420300a20202020202020202020202029390406174d270100696e6' \ +'4657873716c6974655f6175746f696e6465785f696e636f6d696e675f73796e635f31696e63' \ +'6f6d696e675f73796e6305821805071b352701835d747269676765726f7574676f696e675f7' \ +'3796e635f696e736572746f7574676f696e675f73796e630043524541544520545249474745' \ +'52206f7574676f696e675f73796e635f696e7365727420414654455220494e53455254204f4' \ +'e206f7574676f696e675f73796e630a202020202020202020202020424547494e0a20202020' \ +'202020202020202020202020555044415445206f7574676f696e675f73796e630a202020202' \ +'0202020202020202020202053455420757064617465645f6174203d205354524654494d4528' \ +'272573272c20274e4f5727290a20202020202020202020202020202020574845524520524f5' \ +'74944203d206e65772e524f5749443b0a202020202020202020202020454e440d0000000400' \ +'7d0002e501ca00af007d0000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000030090617151501457461626c65746573747465737408435245415445205441' \ +'424c45207465737420286f6e65205445585429821808071b352701835d74726967676572696' \ +'e636f6d696e675f73796e635f757064617465696e636f6d696e675f73796e63004352454154' \ +'45205452494747455220696e636f6d696e675f73796e635f757064617465204146544552205' \ +'55044415445204f4e20696e636f6d696e675f73796e630a2020202020202020202020204245' \ +'47494e0a2020202020202020202020202020202055504441544520696e636f6d696e675f737' \ +'96e630a2020202020202020202020202020202053455420757064617465645f6174203d2053' \ +'54524654494d4528272573272c20274e4f5727290a202020202020202020202020202020205' \ +'74845524520524f574944203d206e65772e524f5749443b0a20202020202020202020202045' \ +'4e44821807071b352701835d74726967676572696e636f6d696e675f73796e635f696e73657' \ +'274696e636f6d696e675f73796e6300435245415445205452494747455220696e636f6d696e' \ +'675f73796e635f696e7365727420414654455220494e53455254204f4e20696e636f6d696e6' \ +'75f73796e630a202020202020202020202020424547494e0a20202020202020202020202020' \ +'20202055504441544520696e636f6d696e675f73796e630a202020202020202020202020202' \ +'0202053455420757064617465645f6174203d205354524654494d4528272573272c20274e4f' \ +'5727290a20202020202020202020202020202020574845524520524f574944203d206e65772' \ +'e524f5749443b0a202020202020202020202020454e44821806071b352701835d7472696767' \ +'65726f7574676f696e675f73796e635f7570646174656f7574676f696e675f73796e6300435' \ +'2454154452054524947474552206f7574676f696e675f73796e635f75706461746520414654' \ +'455220555044415445204f4e206f7574676f696e675f73796e630a202020202020202020202' \ +'020424547494e0a20202020202020202020202020202020555044415445206f7574676f696e' \ +'675f73796e630a2020202020202020202020202020202053455420757064617465645f61742' \ +'03d205354524654494d4528272573272c20274e4f5727290a20202020202020202020202020' \ +'202020574845524520524f574944203d206e65772e524f5749443b0a2020202020202020202' \ +'02020454e440d0000000103fb0003fb00000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'000000000000000000000000000000000000000000000000000000000000000000000000000' \ +'0000000000000000000000000301020f31' + + if __name__ == '__main__': unittest.main() From a68e5d883c2c827bc8ef70b0bef1495325ad7840 Mon Sep 17 00:00:00 2001 From: gholt Date: Mon, 1 Aug 2011 21:14:41 +0000 Subject: [PATCH 3/5] Fixed test based on testdir --- test/unit/common/test_db.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/test/unit/common/test_db.py b/test/unit/common/test_db.py index a5bfabe01c..58c979ca11 100644 --- a/test/unit/common/test_db.py +++ b/test/unit/common/test_db.py @@ -203,6 +203,12 @@ class TestDatabaseBroker(unittest.TestCase): orig_renamer = swift.common.db.renamer try: swift.common.db.renamer = lambda a, b: b + qpath = os.path.dirname(os.path.dirname(os.path.dirname( + os.path.dirname(self.testdir)))) + if qpath: + qpath += '/quarantined/test/db' + else: + qpath = 'quarantined/test/db' # Test malformed database fp = open(os.path.join(self.testdir, '1.db'), 'wb') fp.write(EXAMPLE_MALFORMED_DB.decode('hex')) @@ -215,9 +221,9 @@ class TestDatabaseBroker(unittest.TestCase): conn.execute('SELECT * FROM test') except Exception, err: exc = err - self.assertTrue(str(exc).startswith( - 'Quarantined test/unit/common/db to quarantined/test/db')) - self.assertTrue(str(exc).endswith(' due to malformed database')) + self.assertEquals(str(exc), + 'Quarantined %s to %s due to malformed database' % + (self.testdir, qpath)) # Test corrupted database fp = open(os.path.join(self.testdir, '1.db'), 'wb') fp.write(EXAMPLE_CORRUPTED_DB.decode('hex')) @@ -230,9 +236,9 @@ class TestDatabaseBroker(unittest.TestCase): conn.execute('SELECT * FROM test') except Exception, err: exc = err - self.assertTrue(str(exc).startswith( - 'Quarantined test/unit/common/db to quarantined/test/db')) - self.assertTrue(str(exc).endswith(' due to corrupted database')) + self.assertEquals(str(exc), + 'Quarantined %s to %s due to corrupted database' % + (self.testdir, qpath)) finally: swift.common.db.renamer = orig_renamer From dfd61697c17eb5a44587635542293f4c1f9e50f3 Mon Sep 17 00:00:00 2001 From: gholt Date: Tue, 2 Aug 2011 17:46:17 +0000 Subject: [PATCH 4/5] Fix bug; added test for quarantined a hash dir that becomes a file --- swift/common/db.py | 6 ++++++ swift/obj/replicator.py | 2 +- test/unit/obj/test_replicator.py | 21 +++++++++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/swift/common/db.py b/swift/common/db.py index 2f939fbce4..1ca1151e57 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -258,6 +258,12 @@ class DatabaseBroker(object): conn.commit() def possibly_quarantine(self, exc_type, exc_value, exc_traceback): + """ + Checks the exception info to see if it indicates a quarantine situation + (malformed or corrupted database). If not, the original exception will + be reraised. If so, the database will be quarantined and a new + sqlite3.DatabaseError will be raised indicating the action taken. + """ if 'database disk image is malformed' in str(exc_value): exc_hint = 'malformed' elif 'file is encrypted or is not a database' in str(exc_value): diff --git a/swift/obj/replicator.py b/swift/obj/replicator.py index 9d072bf9d4..f4823776d6 100644 --- a/swift/obj/replicator.py +++ b/swift/obj/replicator.py @@ -80,7 +80,7 @@ def hash_suffix(path, reclaim_age): try: files = os.listdir(hsh_path) except OSError, err: - if err.ernno == errno.ENOTDIR: + if err.errno == errno.ENOTDIR: partition_path = dirname(path) objects_path = dirname(partition_path) device_path = dirname(objects_path) diff --git a/test/unit/obj/test_replicator.py b/test/unit/obj/test_replicator.py index fc4a80524b..a9f5497226 100644 --- a/test/unit/obj/test_replicator.py +++ b/test/unit/obj/test_replicator.py @@ -205,6 +205,27 @@ class TestObjectReplicator(unittest.TestCase): self.assertEquals(hashed, 1) self.assert_('a83' in hashes) + def test_hash_suffix_hash_dir_is_file_quarantine(self): + df = DiskFile(self.devices, 'sda', '0', 'a', 'c', 'o', FakeLogger()) + mkdirs(os.path.dirname(df.datadir)) + open(df.datadir, 'wb').close() + ohash = hash_path('a', 'c', 'o') + data_dir = ohash[-3:] + whole_path_from = os.path.join(self.objects, '0', data_dir) + orig_quarantine_renamer = object_replicator.quarantine_renamer + called = [False] + + def wrapped(*args, **kwargs): + called[0] = True + return orig_quarantine_renamer(*args, **kwargs) + + try: + object_replicator.quarantine_renamer = wrapped + object_replicator.hash_suffix(whole_path_from, 101) + finally: + object_replicator.quarantine_renamer = orig_quarantine_renamer + self.assertTrue(called[0]) + def test_hash_suffix_one_file(self): df = DiskFile(self.devices, 'sda', '0', 'a', 'c', 'o', FakeLogger()) mkdirs(df.datadir) From 3628c08b3e882670a4ff113107a3b3fcdb0f93fc Mon Sep 17 00:00:00 2001 From: gholt Date: Tue, 2 Aug 2011 18:21:25 +0000 Subject: [PATCH 5/5] Real files for bad databases. --- test/unit/common/corrupted_example.db | Bin 0 -> 8192 bytes test/unit/common/malformed_example.db | Bin 0 -> 8192 bytes test/unit/common/test_db.py | 458 +------------------------- 3 files changed, 7 insertions(+), 451 deletions(-) create mode 100644 test/unit/common/corrupted_example.db create mode 100644 test/unit/common/malformed_example.db diff --git a/test/unit/common/corrupted_example.db b/test/unit/common/corrupted_example.db new file mode 100644 index 0000000000000000000000000000000000000000..24e2085655da52d9856c3559f23ec18d8a1cfee7 GIT binary patch literal 8192 zcmeI1&59F25P-WUzb=TUB>^vWkx5{Y4dOv$!K~@T!QF{YI>d_%(X?UK&16kFs0R-l z-^GI$-@|vS>s;lx~PIKBY^ILBe79U9+BQXRFfwT$abGNn2Y}$T}KSRI}*aZT6 zItbJMyO0VKgCUSQ0dxPSj@&pj1a^jix&L=26($fvAaw%z{?CysLayjH`pf!mU8f$x z%{jbm(n86$>HN9uzd0A;TJ@2YFP++y z1mQq@ntnJBl>7cns6 z{4r8wZbSEfmV6=BHT_9`s7adR@K(OGzfW1zol;Dteu^vfQ7>ui=Dl2+KYYfQA@I)$WT^vWkx5{Y4dOv$!K~@T!R*AHbch!jqG`h{o5`AV*!AFH z@s-|J4ecBA9fRCf`uqWXONfS!X zUH}k6^6HLWIW?gXL3A7u$A=khnEI_3a*Ln1j*%DwhCtE;vYGqZWjblU#-AZz2*J-X`+jRC?_THb1(NqpbK{)UypTiGmV!iZOl{4Vg<0b>N1f@M#yT)QTlEM%A zz}Z_49oISSu!ESI_TpbChY}o@vj+3xI@61OPl~>;-o4KEst)8uf7b}K)#vV{|y zk|6AhtI4Nx^~%=oO{Lmf&3zQFUqREb$AxqC<{jN^Qm=QYA;w=3bpUC#+;Yn%DN+(zf>2yO zo{yg+MdsFY|EI|pV%^f8x08Z0xp%!kS(QbNeKW^%KcPWK&Wa#qZ3L$fGy6ybI>3ximZqaFSm zG+B%WwcN4@kZc_dTHI7MNa^p