openstack/swift
Code Issues Proposed changes

Merge "Remove :memory: from DatabaseBrokers and unittests"

Browse Source
This commit is contained in:
Zuul 2023-02-09 08:46:13 +00:00 committed by Gerrit Code Review
commit 6994200026
8 changed files with 345 additions and 194 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
swift/common/db.py
View File

@ -205,7 +205,7 @@ def get_db_connection(path, timeout=30, logger=None, okay_to_create=False):
factory=GreenDBConnection, timeout=timeout)
if QUERY_LOGGING and logger and not six.PY2:
conn.set_trace_callback(logger.debug)
if path != ':memory:' and not okay_to_create:
if not okay_to_create:
# attempt to detect and fail when connect creates the db file
stat = os.stat(path)
if stat.st_size == 0 and stat.st_ctime >= connect_time:
@ -359,17 +359,13 @@ class DatabaseBroker(object):
:param put_timestamp: internalized timestamp of initial PUT request
:param storage_policy_index: only required for containers
"""
if self._db_file == ':memory:':
tmp_db_file = None
conn = get_db_connection(self._db_file, self.timeout, self.logger)
else:
mkdirs(self.db_dir)
fd, tmp_db_file = mkstemp(suffix='.tmp', dir=self.db_dir)
os.close(fd)
conn = sqlite3.connect(tmp_db_file, check_same_thread=False,
factory=GreenDBConnection, timeout=0)
if QUERY_LOGGING and not six.PY2:
conn.set_trace_callback(self.logger.debug)
mkdirs(self.db_dir)
fd, tmp_db_file = mkstemp(suffix='.tmp', dir=self.db_dir)
os.close(fd)
conn = sqlite3.connect(tmp_db_file, check_same_thread=False,
factory=GreenDBConnection, timeout=0)
if QUERY_LOGGING and not six.PY2:
conn.set_trace_callback(self.logger.debug)
# creating dbs implicitly does a lot of transactions, so we
# pick fast, unsafe options here and do a big fsync at the end.
with closing(conn.cursor()) as cur:
@ -541,7 +537,7 @@ class DatabaseBroker(object):
def get(self):
"""Use with the "with" statement; returns a database connection."""
if not self.conn:
if self.db_file != ':memory:' and os.path.exists(self.db_file):
if os.path.exists(self.db_file):
try:
self.conn = get_db_connection(self.db_file, self.timeout,
self.logger)
@ -569,7 +565,7 @@ class DatabaseBroker(object):
def lock(self):
"""Use with the "with" statement; locks a database."""
if not self.conn:
if self.db_file != ':memory:' and os.path.exists(self.db_file):
if os.path.exists(self.db_file):
self.conn = get_db_connection(self.db_file, self.timeout,
self.logger)
else:
@ -636,7 +632,7 @@ class DatabaseBroker(object):
:returns: True if the DB is considered to be deleted, False otherwise
"""
if self.db_file != ':memory:' and not os.path.exists(self.db_file):
if not os.path.exists(self.db_file):
return True
self._commit_puts_stale_ok()
with self.get() as conn:
@ -769,8 +765,8 @@ class DatabaseBroker(object):
"""
Put a record into the DB. If the DB has an associated pending file with
space then the record is appended to that file and a commit to the DB
is deferred. If the DB is in-memory or its pending file is full then
the record will be committed immediately.
is deferred. If its pending file is full then the record will be
committed immediately.
:param record: a record to be added to the DB.
:raises DatabaseConnectionError: if the DB file does not exist or if
@ -778,9 +774,6 @@ class DatabaseBroker(object):
:raises LockTimeout: if a timeout occurs while waiting to take a lock
to write to the pending file.
"""
if self._db_file == ':memory:':
self.merge_items([record])
return
if not os.path.exists(self.db_file):
raise DatabaseConnectionError(self.db_file, "DB doesn't exist")
if self.skip_commits:
@ -806,8 +799,7 @@ class DatabaseBroker(object):
fp.flush()
def _skip_commit_puts(self):
return (self._db_file == ':memory:' or self.skip_commits or not
os.path.exists(self.pending_file))
return self.skip_commits or not os.path.exists(self.pending_file)
def _commit_puts(self, item_list=None):
"""
@ -921,7 +913,7 @@ class DatabaseBroker(object):
within 512k of a boundary, it allocates to the next boundary.
Boundaries are 2m, 5m, 10m, 25m, 50m, then every 50m after.
"""
if not DB_PREALLOCATION or self._db_file == ':memory:':
if not DB_PREALLOCATION:
return
MB = (1024 * 1024)

11
swift/container/backend.py
View File

@ -387,10 +387,7 @@ class ContainerBroker(DatabaseBroker):
stale_reads_ok=False, skip_commits=False,
force_db_file=False):
self._init_db_file = db_file
if db_file == ':memory:':
base_db_file = db_file
else:
base_db_file = make_db_file_path(db_file, None)
base_db_file = make_db_file_path(db_file, None)
super(ContainerBroker, self).__init__(
base_db_file, timeout, logger, account, container, pending_timeout,
stale_reads_ok, skip_commits=skip_commits)
@ -440,8 +437,6 @@ class ContainerBroker(DatabaseBroker):
"""
Returns the current state of on disk db files.
"""
if self._db_file == ':memory:':
return UNSHARDED
if not self.db_files:
return NOTFOUND
if len(self.db_files) > 1:
@ -482,8 +477,6 @@ class ContainerBroker(DatabaseBroker):
"""
Reloads the cached list of valid on disk db files for this broker.
"""
if self._db_file == ':memory:':
return
# reset connection so the next access will use the correct DB file
self.conn = None
self._db_files = get_db_files(self._init_db_file)
@ -890,7 +883,7 @@ class ContainerBroker(DatabaseBroker):
:returns: a tuple, in the form (info, is_deleted) info is a dict as
returned by get_info and is_deleted is a boolean.
"""
if self.db_file != ':memory:' and not os.path.exists(self.db_file):
if not os.path.exists(self.db_file):
return {}, True
info = self.get_info()
return info, self._is_deleted_info(**info)

7
test/unit/__init__.py
View File

@ -42,6 +42,7 @@ import random
import errno
import xattr
from io import BytesIO
from uuid import uuid4
import six
import six.moves.cPickle as pickle
@ -1405,3 +1406,9 @@ def group_by_byte(contents):
return [
(char, sum(1 for _ in grp))
for char, grp in itertools.groupby(byte_iter)]
def generate_db_path(tempdir, server_type):
return os.path.join(
tempdir, '%ss' % server_type, 'part', 'suffix', 'hash',
'%s-%s.db' % (server_type, uuid4()))

7
test/unit/account/test_auditor.py
View File

@ -53,14 +53,15 @@ class TestAuditorRealBrokerMigration(
self.assertUnmigrated(self.broker)
# run auditor, and validate migration
conf = {'devices': self.tempdir, 'mount_check': False,
'recon_cache_path': self.tempdir}
conf = {'devices': self.testdir, 'mount_check': False,
'recon_cache_path': self.testdir}
test_auditor = auditor.AccountAuditor(conf, logger=debug_logger())
test_auditor.run_once()
self.restore_account_broker()
broker = auditor.AccountBroker(self.db_path)
broker = auditor.AccountBroker(self.db_path, account='a')
broker.initialize(Timestamp('1').internal, 0)
# go after rows directly to avoid unintentional migration
with broker.get() as conn:
rows = conn.execute('''

109
test/unit/account/test_backend.py
View File

@ -17,13 +17,10 @@
from collections import defaultdict
import json
import unittest
import pickle
import os
from time import sleep, time
from uuid import uuid4
from tempfile import mkdtemp
from shutil import rmtree
import sqlite3
import itertools
from contextlib import contextmanager
@ -46,23 +43,25 @@ from test.unit.common import test_db
@patch_policies
class TestAccountBroker(unittest.TestCase):
class TestAccountBroker(test_db.TestDbBase):
"""Tests for AccountBroker"""
def setUp(self):
super(TestAccountBroker, self).setUp()
# tests seem to assume x-timestamp was set by the proxy before "now"
self.ts = make_timestamp_iter(offset=-1)
def test_creation(self):
# Test AccountBroker.__init__
broker = AccountBroker(':memory:', account='a')
self.assertEqual(broker.db_file, ':memory:')
db_file = self.get_db_path()
broker = AccountBroker(db_file, account='a')
self.assertEqual(broker.db_file, db_file)
try:
with broker.get() as conn:
pass
except DatabaseConnectionError as e:
self.assertTrue(hasattr(e, 'path'))
self.assertEqual(e.path, ':memory:')
self.assertEqual(e.path, db_file)
self.assertTrue(hasattr(e, 'msg'))
self.assertEqual(e.msg, "DB doesn't exist")
except Exception as e:
@ -76,7 +75,7 @@ class TestAccountBroker(unittest.TestCase):
self.assertEqual(curs.fetchall()[0][0], 1)
def test_initialize_fail(self):
broker = AccountBroker(':memory:')
broker = AccountBroker(self.get_db_path())
with self.assertRaises(ValueError) as cm:
broker.initialize(Timestamp('1').internal)
self.assertEqual(str(cm.exception), 'Attempting to create a new'
@ -85,7 +84,7 @@ class TestAccountBroker(unittest.TestCase):
def test_exception(self):
# Test AccountBroker throwing a conn away after exception
first_conn = None
broker = AccountBroker(':memory:', account='a')
broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
with broker.get() as conn:
first_conn = conn
@ -99,7 +98,7 @@ class TestAccountBroker(unittest.TestCase):
def test_empty(self):
# Test AccountBroker.empty
broker = AccountBroker(':memory:', account='a')
broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
self.assertTrue(broker.empty())
broker.put_container('o', Timestamp.now().internal, 0, 0, 0,
@ -112,12 +111,12 @@ class TestAccountBroker(unittest.TestCase):
def test_is_status_deleted(self):
# Test AccountBroker.is_status_deleted
broker1 = AccountBroker(':memory:', account='a')
broker1 = AccountBroker(self.get_db_path(), account='a')
broker1.initialize(Timestamp.now().internal)
self.assertFalse(broker1.is_status_deleted())
broker1.delete_db(Timestamp.now().internal)
self.assertTrue(broker1.is_status_deleted())
broker2 = AccountBroker(':memory:', account='a')
broker2 = AccountBroker(self.get_db_path(), account='a')
broker2.initialize(Timestamp.now().internal)
# Set delete_timestamp greater than put_timestamp
broker2.merge_timestamps(
@ -126,10 +125,12 @@ class TestAccountBroker(unittest.TestCase):
self.assertTrue(broker2.is_status_deleted())
def test_reclaim(self):
broker = AccountBroker(':memory:', account='test_account')
broker = AccountBroker(self.get_db_path(), account='test_account')
broker.initialize(Timestamp('1').internal)
broker.put_container('c', Timestamp.now().internal, 0, 0, 0,
POLICIES.default.idx)
# commit pending file into db
broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM container "
@ -148,6 +149,7 @@ class TestAccountBroker(unittest.TestCase):
sleep(.00001)
broker.put_container('c', 0, Timestamp.now().internal, 0, 0,
POLICIES.default.idx)
broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM container "
@ -176,6 +178,7 @@ class TestAccountBroker(unittest.TestCase):
broker.put_container('x', 0, 0, 0, 0, POLICIES.default.idx)
broker.put_container('y', 0, 0, 0, 0, POLICIES.default.idx)
broker.put_container('z', 0, 0, 0, 0, POLICIES.default.idx)
broker._commit_puts()
broker.reclaim(Timestamp.now().internal, time())
# Now delete the account
broker.delete_db(Timestamp.now().internal)
@ -193,7 +196,7 @@ class TestAccountBroker(unittest.TestCase):
random.seed(now)
random.shuffle(container_specs)
policy_indexes = list(p.idx for p in POLICIES)
broker = AccountBroker(':memory:', account='test_account')
broker = AccountBroker(self.get_db_path(), account='test_account')
broker.initialize(Timestamp('1').internal)
for i, container_spec in enumerate(container_specs):
# with container12 before container2 and shuffled ts.internal we
@ -206,6 +209,9 @@ class TestAccountBroker(unittest.TestCase):
else:
broker.put_container(name, ts.internal, 0, 0, 0, pidx)
# commit pending file into db
broker._commit_puts()
def count_reclaimable(conn, reclaim_age):
return conn.execute(
"SELECT count(*) FROM container "
@ -251,7 +257,7 @@ class TestAccountBroker(unittest.TestCase):
def test_delete_db_status(self):
start = next(self.ts)
broker = AccountBroker(':memory:', account='a')
broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(start.internal)
info = broker.get_info()
self.assertEqual(info['put_timestamp'], start.internal)
@ -273,10 +279,12 @@ class TestAccountBroker(unittest.TestCase):
def test_delete_container(self):
# Test AccountBroker.delete_container
broker = AccountBroker(':memory:', account='a')
broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
broker.put_container('o', Timestamp.now().internal, 0, 0, 0,
POLICIES.default.idx)
# commit pending file into db
broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM container "
@ -287,6 +295,7 @@ class TestAccountBroker(unittest.TestCase):
sleep(.00001)
broker.put_container('o', 0, Timestamp.now().internal, 0, 0,
POLICIES.default.idx)
broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM container "
@ -297,13 +306,15 @@ class TestAccountBroker(unittest.TestCase):
def test_put_container(self):
# Test AccountBroker.put_container
broker = AccountBroker(':memory:', account='a')
broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
# Create initial container
timestamp = Timestamp.now().internal
broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0,
POLICIES.default.idx)
# commit pending file into db
broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0],
@ -317,6 +328,7 @@ class TestAccountBroker(unittest.TestCase):
# Reput same event
broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0,
POLICIES.default.idx)
broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0],
@ -332,6 +344,7 @@ class TestAccountBroker(unittest.TestCase):
timestamp = Timestamp.now().internal
broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0,
POLICIES.default.idx)
broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0],
@ -346,6 +359,7 @@ class TestAccountBroker(unittest.TestCase):
otimestamp = Timestamp(float(Timestamp(timestamp)) - 1).internal
broker.put_container('"{<container \'&\' name>}"', otimestamp, 0, 0, 0,
POLICIES.default.idx)
broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0],
@ -360,6 +374,7 @@ class TestAccountBroker(unittest.TestCase):
dtimestamp = Timestamp(float(Timestamp(timestamp)) - 1).internal
broker.put_container('"{<container \'&\' name>}"', 0, dtimestamp, 0, 0,
POLICIES.default.idx)
broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0],
@ -378,6 +393,7 @@ class TestAccountBroker(unittest.TestCase):
timestamp = Timestamp.now().internal
broker.put_container('"{<container \'&\' name>}"', 0, timestamp, 0, 0,
POLICIES.default.idx)
broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0],
@ -393,6 +409,7 @@ class TestAccountBroker(unittest.TestCase):
timestamp = Timestamp.now().internal
broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0,
POLICIES.default.idx)
broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0],
@ -405,7 +422,7 @@ class TestAccountBroker(unittest.TestCase):
def test_get_info(self):
# Test AccountBroker.get_info
broker = AccountBroker(':memory:', account='test1')
broker = AccountBroker(self.get_db_path(), account='test1')
broker.initialize(Timestamp('1').internal)
info = broker.get_info()
@ -452,7 +469,7 @@ class TestAccountBroker(unittest.TestCase):
def test_list_containers_iter(self):
# Test AccountBroker.list_containers_iter
broker = AccountBroker(':memory:', account='a')
broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
for cont1 in range(4):
for cont2 in range(125):
@ -582,7 +599,7 @@ class TestAccountBroker(unittest.TestCase):
def test_list_objects_iter_order_and_reverse(self):
# Test ContainerBroker.list_objects_iter
broker = AccountBroker(':memory:', account='a')
broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal, 0)
broker.put_container(
@ -617,7 +634,7 @@ class TestAccountBroker(unittest.TestCase):
def test_list_container_iter_with_reserved_name(self):
# Test ContainerBroker.list_objects_iter
broker = AccountBroker(':memory:', account='a')
broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(next(self.ts).internal, 0)
broker.put_container(
@ -754,11 +771,13 @@ class TestAccountBroker(unittest.TestCase):
}
failures = []
for expected in expectations:
broker = AccountBroker(':memory:', account='a')
broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(next(self.ts).internal, 0)
for name in expected['containers']:
broker.put_container(name, next(self.ts).internal, 0, 0, 0,
POLICIES.default.idx)
# commit pending file into db
broker._commit_puts()
params = default_listing_params.copy()
params.update(expected['params'])
listing = list(c[0] for c in broker.list_containers_iter(**params))
@ -773,7 +792,7 @@ class TestAccountBroker(unittest.TestCase):
def test_double_check_trailing_delimiter(self):
# Test AccountBroker.list_containers_iter for an
# account that has an odd container with a trailing delimiter
broker = AccountBroker(':memory:', account='a')
broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
broker.put_container('a', Timestamp.now().internal, 0, 0, 0,
POLICIES.default.idx)
@ -814,7 +833,7 @@ class TestAccountBroker(unittest.TestCase):
self.assertEqual([row[0] for row in listing], ['b-a', 'b-b'])
def test_chexor(self):
broker = AccountBroker(':memory:', account='a')
broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
broker.put_container('a', Timestamp(1).internal,
Timestamp(0).internal, 0, 0,
@ -842,9 +861,9 @@ class TestAccountBroker(unittest.TestCase):
self.assertEqual(broker.get_info()['hash'], hashc)
def test_merge_items(self):
broker1 = AccountBroker(':memory:', account='a')
broker1 = AccountBroker(self.get_db_path(), account='a')
broker1.initialize(Timestamp('1').internal)
broker2 = AccountBroker(':memory:', account='a')
broker2 = AccountBroker(self.get_db_path(), account='a')
broker2.initialize(Timestamp('1').internal)
broker1.put_container('a', Timestamp(1).internal, 0, 0, 0,
POLICIES.default.idx)
@ -869,10 +888,10 @@ class TestAccountBroker(unittest.TestCase):
snowman = u'\N{SNOWMAN}'
if six.PY2:
snowman = snowman.encode('utf-8')
broker1 = AccountBroker(':memory:', account='a')
broker1 = AccountBroker(self.get_db_path(), account='a')
broker1.initialize(Timestamp('1').internal, 0)
id1 = broker1.get_info()['id']
broker2 = AccountBroker(':memory:', account='a')
broker2 = AccountBroker(self.get_db_path(), account='a')
broker2.initialize(Timestamp('1').internal, 0)
broker1.put_container(snowman, Timestamp(2).internal, 0, 1, 100,
POLICIES.default.idx)
@ -974,7 +993,7 @@ class TestAccountBroker(unittest.TestCase):
StoragePolicy(2, 'two', False),
StoragePolicy(3, 'three', False)])
def test_get_policy_stats(self):
broker = AccountBroker(':memory:', account='a')
broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(next(self.ts).internal)
# check empty policy_stats
self.assertTrue(broker.empty())
@ -1041,7 +1060,7 @@ class TestAccountBroker(unittest.TestCase):
@patch_policies([StoragePolicy(0, 'zero', False),
StoragePolicy(1, 'one', True)])
def test_policy_stats_tracking(self):
broker = AccountBroker(':memory:', account='a')
broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(next(self.ts).internal)
# policy 0
@ -1162,6 +1181,7 @@ def premetadata_create_account_stat_table(self, conn, put_timestamp):
class TestCommonAccountBroker(test_db.TestExampleBroker):
broker_class = AccountBroker
server_type = 'account'
def setUp(self):
super(TestCommonAccountBroker, self).setUp()
@ -1183,13 +1203,14 @@ class TestAccountBrokerBeforeMetadata(TestAccountBroker):
"""
def setUp(self):
super(TestAccountBroker, self).setUp()
# tests seem to assume x-timestamp was set by the proxy before "now"
self.ts = make_timestamp_iter(offset=-1)
self._imported_create_account_stat_table = \
AccountBroker.create_account_stat_table
AccountBroker.create_account_stat_table = \
premetadata_create_account_stat_table
broker = AccountBroker(':memory:', account='a')
broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
exc = None
with broker.get() as conn:
@ -1202,10 +1223,11 @@ class TestAccountBrokerBeforeMetadata(TestAccountBroker):
def tearDown(self):
AccountBroker.create_account_stat_table = \
self._imported_create_account_stat_table
broker = AccountBroker(':memory:', account='a')
broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
with broker.get() as conn:
conn.execute('SELECT metadata FROM account_stat')
super(TestAccountBrokerBeforeMetadata, self).tearDown()
def prespi_create_container_table(self, conn):
@ -1270,6 +1292,7 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker):
"""
def setUp(self):
super(TestAccountBrokerBeforeSPI, self).setUp()
# tests seem to assume x-timestamp was set by the proxy before "now"
self.ts = make_timestamp_iter(offset=-1)
self._imported_create_container_table = \
@ -1278,7 +1301,7 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker):
prespi_create_container_table
self._imported_initialize = AccountBroker._initialize
AccountBroker._initialize = prespi_AccountBroker_initialize
broker = AccountBroker(':memory:', account='a')
broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
exc = None
with broker.get() as conn:
@ -1299,10 +1322,11 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker):
AccountBroker.create_container_table = \
self._imported_create_container_table
AccountBroker._initialize = self._imported_initialize
broker = AccountBroker(':memory:', account='a')
broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
with broker.get() as conn:
conn.execute('SELECT storage_policy_index FROM container')
super(TestAccountBrokerBeforeSPI, self).tearDown()
@with_tempdir
def test_policy_table_migration(self, tempdir):
@ -1681,7 +1705,7 @@ def pre_track_containers_create_container_table(self, conn):
""" + OLD_POLICY_STAT_TRIGGER_SCRIPT)
class AccountBrokerPreTrackContainerCountSetup(object):
class AccountBrokerPreTrackContainerCountSetup(test_db.TestDbBase):
def assertUnmigrated(self, broker):
with broker.get() as conn:
try:
@ -1696,6 +1720,7 @@ class AccountBrokerPreTrackContainerCountSetup(object):
'trying to select container_count from policy_stat!')
def setUp(self):
super(AccountBrokerPreTrackContainerCountSetup, self).setUp()
# use old version of policy_stat
self._imported_create_policy_stat_table = \
AccountBroker.create_policy_stat_table
@ -1708,17 +1733,16 @@ class AccountBrokerPreTrackContainerCountSetup(object):
AccountBroker.create_container_table = \
pre_track_containers_create_container_table
broker = AccountBroker(':memory:', account='a')
broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
self.assertUnmigrated(broker)
self.tempdir = mkdtemp()
# tests seem to assume x-timestamp was set by the proxy before "now"
self.ts = make_timestamp_iter(offset=-1)
self.db_path = os.path.join(self.tempdir, 'sda', 'accounts',
self.db_path = os.path.join(self.testdir, 'sda', 'accounts',
'0', '0', '0', 'test.db')
self.broker = AccountBroker(self.db_path, account='a')
self.broker = AccountBroker(self.get_db_path(), account='a')
self.broker.initialize(next(self.ts).internal)
# Common sanity-check that our starting, pre-migration state correctly
@ -1726,14 +1750,13 @@ class AccountBrokerPreTrackContainerCountSetup(object):
self.assertUnmigrated(self.broker)
def tearDown(self):
rmtree(self.tempdir, ignore_errors=True)
self.restore_account_broker()
broker = AccountBroker(':memory:', account='a')
broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
with broker.get() as conn:
conn.execute('SELECT container_count FROM policy_stat')
super(AccountBrokerPreTrackContainerCountSetup, self).tearDown()
def restore_account_broker(self):
AccountBroker.create_policy_stat_table = \
@ -1925,7 +1948,7 @@ class TestAccountBrokerBeforePerPolicyContainerTrack(
self.assertEqual(policy_stat['container_count'], 1)
def test_migrate_add_storage_policy_index_fail(self):
broker = AccountBroker(':memory:', account='a')
broker = AccountBroker(self.db_path, account='a')
broker.initialize(Timestamp('1').internal)
with mock.patch.object(
broker, 'create_policy_stat_table',

13
test/unit/account/test_utils.py
View File

@ -26,6 +26,7 @@ from swift.common.header_key_dict import HeaderKeyDict
from swift.common.request_helpers import get_reserved_name
from test.unit import patch_policies, make_timestamp_iter
from test.unit.common.test_db import TestDbBase
class TestFakeAccountBroker(unittest.TestCase):
@ -58,9 +59,11 @@ class TestFakeAccountBroker(unittest.TestCase):
self.assertEqual(broker.get_policy_stats(), {})
class TestAccountUtils(unittest.TestCase):
class TestAccountUtils(TestDbBase):
server_type = 'account'
def setUp(self):
super(TestAccountUtils, self).setUp()
self.ts = make_timestamp_iter()
def test_get_response_headers_fake_broker(self):
@ -78,7 +81,7 @@ class TestAccountUtils(unittest.TestCase):
self.assertEqual(resp_headers, expected)
def test_get_response_headers_empty_memory_broker(self):
broker = backend.AccountBroker(':memory:', account='a')
broker = backend.AccountBroker(self.db_path, account='a')
now = time.time()
with mock.patch('time.time', new=lambda: now):
broker.initialize(Timestamp(now).internal)
@ -94,7 +97,7 @@ class TestAccountUtils(unittest.TestCase):
@patch_policies
def test_get_response_headers_with_data(self):
broker = backend.AccountBroker(':memory:', account='a')
broker = backend.AccountBroker(self.db_path, account='a')
now = time.time()
with mock.patch('time.time', new=lambda: now):
broker.initialize(Timestamp(now).internal)
@ -141,7 +144,7 @@ class TestAccountUtils(unittest.TestCase):
@patch_policies
def test_get_response_headers_with_legacy_data(self):
broker = backend.AccountBroker(':memory:', account='a')
broker = backend.AccountBroker(self.db_path, account='a')
now = time.time()
with mock.patch('time.time', new=lambda: now):
broker.initialize(Timestamp(now).internal)
@ -213,7 +216,7 @@ class TestAccountUtils(unittest.TestCase):
@patch_policies([StoragePolicy(0, 'zero', is_default=True)])
def test_account_listing_reserved_names(self):
broker = backend.AccountBroker(':memory:', account='a')
broker = backend.AccountBroker(self.db_path, account='a')
put_timestamp = next(self.ts)
now = time.time()
with mock.patch('time.time', new=lambda: now):

121
test/unit/common/test_db.py
View File

@ -46,7 +46,7 @@ from swift.common.utils import normalize_timestamp, mkdirs, Timestamp
from swift.common.exceptions import LockTimeout
from swift.common.swob import HTTPException
from test.unit import with_tempdir, make_timestamp_iter
from test.unit import make_timestamp_iter, generate_db_path
class TestHelperFunctions(unittest.TestCase):
@ -180,10 +180,35 @@ class TestGreenDBConnection(unittest.TestCase):
InterceptConnection.commit.call_count))
class TestGetDBConnection(unittest.TestCase):
class TestDbBase(unittest.TestCase):
server_type = 'container'
testdir = None
def setUp(self):
self.testdir = mkdtemp()
self.db_path = self.get_db_path()
def tearDown(self):
rmtree(self.testdir, ignore_errors=True)
def get_db_path(self):
return generate_db_path(self.testdir, self.server_type)
class TestGetDBConnection(TestDbBase):
def setUp(self):
super(TestGetDBConnection, self).setUp()
self.db_path = self.init_db_path()
def init_db_path(self):
# Test ContainerBroker.empty
db_path = self.get_db_path()
broker = ExampleBroker(db_path, account='a')
broker.initialize(Timestamp.now().internal, 0)
return db_path
def test_normal_case(self):
conn = get_db_connection(':memory:')
conn = get_db_connection(self.db_path)
self.assertTrue(hasattr(conn, 'execute'))
def test_invalid_path(self):
@ -201,8 +226,8 @@ class TestGetDBConnection(unittest.TestCase):
InterceptCursor.execute = mock_db_cmd
with patch('sqlite3.Cursor', new=InterceptCursor):
self.assertRaises(Timeout, get_db_connection, ':memory:',
timeout=0.1)
self.assertRaises(Timeout, get_db_connection,
self.db_path, timeout=0.1)
self.assertTrue(mock_db_cmd.called)
self.assertEqual(mock_db_cmd.call_args_list,
list((mock_db_cmd.call_args,) *
@ -319,7 +344,7 @@ class ExampleBroker(DatabaseBroker):
Timestamp(info['put_timestamp']))
class TestExampleBroker(unittest.TestCase):
class TestExampleBroker(TestDbBase):
"""
Tests that use the mostly Concrete enough ExampleBroker to exercise some
of the abstract methods on DatabaseBroker.
@ -327,19 +352,21 @@ class TestExampleBroker(unittest.TestCase):
broker_class = ExampleBroker
policy = 0
server_type = 'example'
def setUp(self):
super(TestExampleBroker, self).setUp()
self.ts = make_timestamp_iter()
def test_delete_db(self):
broker = self.broker_class(':memory:', account='a', container='c')
broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(next(self.ts).internal)
broker.delete_db(next(self.ts).internal)
self.assertTrue(broker.is_deleted())
def test_merge_timestamps_simple_delete(self):
put_timestamp = next(self.ts).internal
broker = self.broker_class(':memory:', account='a', container='c')
broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(put_timestamp)
created_at = broker.get_info()['created_at']
broker.merge_timestamps(created_at, put_timestamp, '0')
@ -366,7 +393,7 @@ class TestExampleBroker(unittest.TestCase):
def test_merge_timestamps_delete_with_objects(self):
put_timestamp = next(self.ts).internal
broker = self.broker_class(':memory:', account='a', container='c')
broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(put_timestamp, storage_policy_index=int(self.policy))
created_at = broker.get_info()['created_at']
broker.merge_timestamps(created_at, put_timestamp, '0')
@ -397,7 +424,7 @@ class TestExampleBroker(unittest.TestCase):
def test_merge_timestamps_simple_recreate(self):
put_timestamp = next(self.ts).internal
broker = self.broker_class(':memory:', account='a', container='c')
broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(put_timestamp, storage_policy_index=int(self.policy))
virgin_status_changed_at = broker.get_info()['status_changed_at']
created_at = broker.get_info()['created_at']
@ -426,7 +453,7 @@ class TestExampleBroker(unittest.TestCase):
def test_merge_timestamps_recreate_with_objects(self):
put_timestamp = next(self.ts).internal
broker = self.broker_class(':memory:', account='a', container='c')
broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(put_timestamp, storage_policy_index=int(self.policy))
created_at = broker.get_info()['created_at']
# delete
@ -460,7 +487,7 @@ class TestExampleBroker(unittest.TestCase):
def test_merge_timestamps_update_put_no_status_change(self):
put_timestamp = next(self.ts).internal
broker = self.broker_class(':memory:', account='a', container='c')
broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(put_timestamp, storage_policy_index=int(self.policy))
info = broker.get_info()
orig_status_changed_at = info['status_changed_at']
@ -473,7 +500,7 @@ class TestExampleBroker(unittest.TestCase):
def test_merge_timestamps_update_delete_no_status_change(self):
put_timestamp = next(self.ts).internal
broker = self.broker_class(':memory:', account='a', container='c')
broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(put_timestamp, storage_policy_index=int(self.policy))
created_at = broker.get_info()['created_at']
broker.merge_timestamps(created_at, put_timestamp,
@ -487,19 +514,23 @@ class TestExampleBroker(unittest.TestCase):
self.assertEqual(orig_status_changed_at, info['status_changed_at'])
def test_get_max_row(self):
broker = self.broker_class(':memory:', account='a', container='c')
broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(next(self.ts).internal,
storage_policy_index=int(self.policy))
self.assertEqual(-1, broker.get_max_row())
self.put_item(broker, next(self.ts).internal)
# commit pending file into db
broker._commit_puts()
self.assertEqual(1, broker.get_max_row())
self.delete_item(broker, next(self.ts).internal)
broker._commit_puts()
self.assertEqual(2, broker.get_max_row())
self.put_item(broker, next(self.ts).internal)
broker._commit_puts()
self.assertEqual(3, broker.get_max_row())
def test_get_info(self):
broker = self.broker_class(':memory:', account='test', container='c')
broker = self.broker_class(self.db_path, account='test', container='c')
created_at = time.time()
with patch('swift.common.db.time.time', new=lambda: created_at):
broker.initialize(Timestamp(1).internal,
@ -519,7 +550,7 @@ class TestExampleBroker(unittest.TestCase):
k, info[k], v))
def test_get_raw_metadata(self):
broker = self.broker_class(':memory:', account='test', container='c')
broker = self.broker_class(self.db_path, account='test', container='c')
broker.initialize(Timestamp(0).internal,
storage_policy_index=int(self.policy))
self.assertEqual(broker.metadata, {})
@ -544,7 +575,7 @@ class TestExampleBroker(unittest.TestCase):
json.dumps(metadata))
def test_put_timestamp(self):
broker = self.broker_class(':memory:', account='a', container='c')
broker = self.broker_class(self.db_path, account='a', container='c')
orig_put_timestamp = next(self.ts).internal
broker.initialize(orig_put_timestamp,
storage_policy_index=int(self.policy))
@ -565,7 +596,7 @@ class TestExampleBroker(unittest.TestCase):
newer_put_timestamp)
def test_status_changed_at(self):
broker = self.broker_class(':memory:', account='test', container='c')
broker = self.broker_class(self.db_path, account='test', container='c')
put_timestamp = next(self.ts).internal
created_at = time.time()
with patch('swift.common.db.time.time', new=lambda: created_at):
@ -591,7 +622,7 @@ class TestExampleBroker(unittest.TestCase):
status_changed_at)
def test_get_syncs(self):
broker = self.broker_class(':memory:', account='a', container='c')
broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(Timestamp.now().internal,
storage_policy_index=int(self.policy))
self.assertEqual([], broker.get_syncs())
@ -604,9 +635,8 @@ class TestExampleBroker(unittest.TestCase):
self.assertEqual([{'sync_point': 2, 'remote_id': 'remote2'}],
broker.get_syncs(incoming=False))
@with_tempdir
def test_commit_pending(self, tempdir):
broker = self.broker_class(os.path.join(tempdir, 'test.db'),
def test_commit_pending(self):
broker = self.broker_class(os.path.join(self.testdir, 'test.db'),
account='a', container='c')
broker.initialize(next(self.ts).internal,
storage_policy_index=int(self.policy))
@ -617,12 +647,12 @@ class TestExampleBroker(unittest.TestCase):
info = rows[0]
count_key = '%s_count' % broker.db_contains_type
self.assertEqual(0, info[count_key])
broker.get_info()
# commit pending file into db
broker._commit_puts()
self.assertEqual(1, broker.get_info()[count_key])
@with_tempdir
def test_maybe_get(self, tempdir):
broker = self.broker_class(os.path.join(tempdir, 'test.db'),
def test_maybe_get(self):
broker = self.broker_class(os.path.join(self.testdir, 'test.db'),
account='a', container='c')
broker.initialize(next(self.ts).internal,
storage_policy_index=int(self.policy))
@ -640,13 +670,7 @@ class TestExampleBroker(unittest.TestCase):
self.assertEqual(broker.conn, conn)
class TestDatabaseBroker(unittest.TestCase):
def setUp(self):
self.testdir = mkdtemp()
def tearDown(self):
rmtree(self.testdir, ignore_errors=1)
class TestDatabaseBroker(TestDbBase):
def test_DB_PREALLOCATION_setting(self):
u = uuid4().hex
@ -657,8 +681,8 @@ class TestDatabaseBroker(unittest.TestCase):
self.assertRaises(OSError, b._preallocate)
def test_memory_db_init(self):
broker = DatabaseBroker(':memory:')
self.assertEqual(broker.db_file, ':memory:')
broker = DatabaseBroker(self.db_path)
self.assertEqual(broker.db_file, self.db_path)
self.assertRaises(AttributeError, broker.initialize,
normalize_timestamp('0'))
@ -687,7 +711,7 @@ class TestDatabaseBroker(unittest.TestCase):
def test_initialize(self):
self.assertRaises(AttributeError,
DatabaseBroker(':memory:').initialize,
DatabaseBroker(self.db_path).initialize,
normalize_timestamp('1'))
stub_dict = {}
@ -695,7 +719,7 @@ class TestDatabaseBroker(unittest.TestCase):
stub_dict.clear()
stub_dict['args'] = args
stub_dict.update(kwargs)
broker = DatabaseBroker(':memory:')
broker = DatabaseBroker(self.db_path)
broker._initialize = stub
broker.initialize(normalize_timestamp('1'))
self.assertTrue(hasattr(stub_dict['args'][0], 'execute'))
@ -735,7 +759,7 @@ class TestDatabaseBroker(unittest.TestCase):
def do_test(expected_metadata, delete_meta_whitelist=None):
if not delete_meta_whitelist:
delete_meta_whitelist = []
broker = DatabaseBroker(':memory:')
broker = DatabaseBroker(self.get_db_path())
broker.delete_meta_whitelist = delete_meta_whitelist
broker.db_type = 'test'
broker._initialize = init_stub
@ -791,13 +815,13 @@ class TestDatabaseBroker(unittest.TestCase):
['x-container-meta-test', 'x-something-else'])
def test_get(self):
broker = DatabaseBroker(':memory:')
broker = DatabaseBroker(self.db_path)
with self.assertRaises(DatabaseConnectionError) as raised, \
broker.get() as conn:
conn.execute('SELECT 1')
self.assertEqual(
str(raised.exception),
"DB connection error (:memory:, 0):\nDB doesn't exist")
"DB connection error (%s, 0):\nDB doesn't exist" % self.db_path)
broker = DatabaseBroker(os.path.join(self.testdir, '1.db'))
with self.assertRaises(DatabaseConnectionError) as raised, \
@ -947,7 +971,7 @@ class TestDatabaseBroker(unittest.TestCase):
pass
def test_newid(self):
broker = DatabaseBroker(':memory:')
broker = DatabaseBroker(self.db_path)
broker.db_type = 'test'
broker.db_contains_type = 'test'
uuid1 = str(uuid4())
@ -998,7 +1022,7 @@ class TestDatabaseBroker(unittest.TestCase):
self.assertEqual(points[0][1], uuid2)
def test_get_items_since(self):
broker = DatabaseBroker(':memory:')
broker = DatabaseBroker(self.db_path)
broker.db_type = 'test'
broker.db_contains_type = 'test'
@ -1020,7 +1044,7 @@ class TestDatabaseBroker(unittest.TestCase):
self.assertEqual(broker.get_items_since(999, 2), [])
def test_get_sync(self):
broker = DatabaseBroker(':memory:')
broker = DatabaseBroker(self.db_path)
broker.db_type = 'test'
broker.db_contains_type = 'test'
uuid1 = str(uuid4())
@ -1060,7 +1084,7 @@ class TestDatabaseBroker(unittest.TestCase):
self.assertEqual(broker.get_sync(uuid3, incoming=False), 2)
def test_merge_syncs(self):
broker = DatabaseBroker(':memory:')
broker = DatabaseBroker(self.db_path)
def stub(*args, **kwargs):
pass
@ -1105,7 +1129,7 @@ class TestDatabaseBroker(unittest.TestCase):
self.get_replication_info_tester(metadata=True)
def get_replication_info_tester(self, metadata=False):
broker = DatabaseBroker(':memory:', account='a')
broker = DatabaseBroker(self.db_path, account='a')
broker.db_type = 'test'
broker.db_contains_type = 'test'
broker.db_reclaim_timestamp = 'created_at'
@ -1428,7 +1452,7 @@ class TestDatabaseBroker(unittest.TestCase):
(dbpath, qpath, hint))
def test_skip_commits(self):
broker = DatabaseBroker(':memory:')
broker = DatabaseBroker(self.db_path)
self.assertTrue(broker._skip_commit_puts())
broker._initialize = MagicMock()
broker.initialize(Timestamp.now())
@ -1585,7 +1609,7 @@ class TestDatabaseBroker(unittest.TestCase):
self.assertFalse(pending)
class TestTombstoneReclaimer(unittest.TestCase):
class TestTombstoneReclaimer(TestDbBase):
def _make_object(self, broker, obj_name, ts, deleted):
if deleted:
broker.delete_test(obj_name, ts.internal)
@ -1603,7 +1627,8 @@ class TestTombstoneReclaimer(unittest.TestCase):
return self._count_reclaimable(conn, reclaim_age)
def _setup_tombstones(self, reverse_names=True):
broker = ExampleBroker(':memory:', account='test_account',
broker = ExampleBroker(self.db_path,
account='test_account',
container='test_container')
broker.initialize(Timestamp('1').internal, 0)
now = time.time()

233
test/unit/container/test_backend.py
View File

@ -53,13 +53,15 @@ from test.unit import (patch_policies, with_tempdir, make_timestamp_iter,
from test.unit.common import test_db
class TestContainerBroker(unittest.TestCase):
class TestContainerBroker(test_db.TestDbBase):
"""Tests for ContainerBroker"""
expected_db_tables = {'outgoing_sync', 'incoming_sync', 'object',
'sqlite_sequence', 'policy_stat',
'container_info', 'shard_range'}
server_type = 'container'
def setUp(self):
super(TestContainerBroker, self).setUp()
self.ts = make_timestamp_iter()
def _assert_shard_ranges(self, broker, expected, include_own=False):
@ -70,8 +72,9 @@ class TestContainerBroker(unittest.TestCase):
def test_creation(self):
# Test ContainerBroker.__init__
broker = ContainerBroker(':memory:', account='a', container='c')
self.assertEqual(broker._db_file, ':memory:')
db_file = self.get_db_path()
broker = ContainerBroker(db_file, account='a', container='c')
self.assertEqual(broker._db_file, db_file)
broker.initialize(Timestamp('1').internal, 0)
with broker.get() as conn:
curs = conn.cursor()
@ -83,6 +86,8 @@ class TestContainerBroker(unittest.TestCase):
# check the update trigger
broker.put_object('blah', Timestamp.now().internal, 0, 'text/plain',
'etag', 0, 0)
# commit pending file into db
broker._commit_puts()
with broker.get() as conn:
with self.assertRaises(sqlite3.DatabaseError) as cm:
conn.execute('UPDATE object SET name="blah";')
@ -98,7 +103,7 @@ class TestContainerBroker(unittest.TestCase):
@patch_policies
def test_storage_policy_property(self):
for policy in POLICIES:
broker = ContainerBroker(':memory:', account='a',
broker = ContainerBroker(self.get_db_path(), account='a',
container='policy_%s' % policy.name)
broker.initialize(next(self.ts).internal, policy.idx)
with broker.get() as conn:
@ -121,7 +126,8 @@ class TestContainerBroker(unittest.TestCase):
# Test ContainerBroker throwing a conn away after
# unhandled exception
first_conn = None
broker = ContainerBroker(':memory:', account='a', container='c')
broker = ContainerBroker(self.get_db_path(),
account='a', container='c')
broker.initialize(Timestamp('1').internal, 0)
with broker.get() as conn:
first_conn = conn
@ -617,11 +623,14 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual('.shards_a/cc', broker.root_path)
def test_reclaim(self):
broker = ContainerBroker(':memory:', account='test_account',
broker = ContainerBroker(self.get_db_path(),
account='test_account',
container='test_container')
broker.initialize(Timestamp('1').internal, 0)
broker.put_object('o', Timestamp.now().internal, 0, 'text/plain',
'd41d8cd98f00b204e9800998ecf8427e')
# commit pending file into db
broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM object "
@ -639,6 +648,7 @@ class TestContainerBroker(unittest.TestCase):
"WHERE deleted = 1").fetchone()[0], 0)
sleep(.00001)
broker.delete_object('o', Timestamp.now().internal)
broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM object "
@ -672,6 +682,7 @@ class TestContainerBroker(unittest.TestCase):
'd41d8cd98f00b204e9800998ecf8427e')
broker.put_object('z', Timestamp.now().internal, 0, 'text/plain',
'd41d8cd98f00b204e9800998ecf8427e')
broker._commit_puts()
# Test before deletion
broker.reclaim(Timestamp.now().internal, time())
broker.delete_db(Timestamp.now().internal)
@ -688,7 +699,8 @@ class TestContainerBroker(unittest.TestCase):
random.seed(now)
random.shuffle(obj_specs)
policy_indexes = list(p.idx for p in POLICIES)
broker = ContainerBroker(':memory:', account='test_account',
broker = ContainerBroker(self.get_db_path(),
account='test_account',
container='test_container')
broker.initialize(Timestamp('1').internal, 0)
for i, obj_spec in enumerate(obj_specs):
@ -702,6 +714,8 @@ class TestContainerBroker(unittest.TestCase):
else:
broker.put_object(obj_name, ts.internal, 0, 'text/plain',
'etag', storage_policy_index=pidx)
# commit pending file into db
broker._commit_puts()
def count_reclaimable(conn, reclaim_age):
return conn.execute(
@ -748,7 +762,8 @@ class TestContainerBroker(unittest.TestCase):
trace[1][2] > trace[2][2])
def test_reclaim_with_duplicate_names(self):
broker = ContainerBroker(':memory:', account='test_account',
broker = ContainerBroker(self.get_db_path(),
account='test_account',
container='test_container')
broker.initialize(Timestamp('1').internal, 0)
now = time()
@ -757,6 +772,8 @@ class TestContainerBroker(unittest.TestCase):
for spidx in range(10):
obj_name = 'object%s' % i
broker.delete_object(obj_name, ages_ago.internal, spidx)
# commit pending file into db
broker._commit_puts()
reclaim_age = now - (2 * 7 * 24 * 60 * 60)
with broker.get() as conn:
self.assertEqual(conn.execute(
@ -843,7 +860,8 @@ class TestContainerBroker(unittest.TestCase):
def test_get_info_is_deleted(self):
ts = make_timestamp_iter()
start = next(ts)
broker = ContainerBroker(':memory:', account='test_account',
broker = ContainerBroker(self.get_db_path(),
account='test_account',
container='test_container')
# create it
broker.initialize(start.internal, POLICIES.default.idx)
@ -892,10 +910,13 @@ class TestContainerBroker(unittest.TestCase):
def test_delete_object(self):
# Test ContainerBroker.delete_object
broker = ContainerBroker(':memory:', account='a', container='c')
broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0)
broker.put_object('o', Timestamp.now().internal, 0, 'text/plain',
'd41d8cd98f00b204e9800998ecf8427e')
# commit pending file into db
broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM object "
@ -905,6 +926,7 @@ class TestContainerBroker(unittest.TestCase):
"WHERE deleted = 1").fetchone()[0], 0)
sleep(.00001)
broker.delete_object('o', Timestamp.now().internal)
broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM object "
@ -915,7 +937,8 @@ class TestContainerBroker(unittest.TestCase):
def test_put_object(self):
# Test ContainerBroker.put_object
broker = ContainerBroker(':memory:', account='a', container='c')
broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0)
# Create initial object
@ -923,6 +946,8 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('"{<object \'&\' name>}"', timestamp, 123,
'application/x-test',
'5af83e3196bf99f440f31f2e1a6c9afe')
# commit pending file into db
broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM object").fetchone()[0],
@ -944,6 +969,7 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('"{<object \'&\' name>}"', timestamp, 123,
'application/x-test',
'5af83e3196bf99f440f31f2e1a6c9afe')
broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM object").fetchone()[0],
@ -967,6 +993,7 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('"{<object \'&\' name>}"', timestamp, 124,
'application/x-test',
'aa0749bacbc79ec65fe206943d8fe449')
broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM object").fetchone()[0],
@ -989,6 +1016,7 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('"{<object \'&\' name>}"', otimestamp, 124,
'application/x-test',
'aa0749bacbc79ec65fe206943d8fe449')
broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM object").fetchone()[0],
@ -1010,6 +1038,7 @@ class TestContainerBroker(unittest.TestCase):
dtimestamp = Timestamp(float(Timestamp(timestamp)) - 1).internal
broker.put_object('"{<object \'&\' name>}"', dtimestamp, 0, '', '',
deleted=1)
broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM object").fetchone()[0],
@ -1032,6 +1061,7 @@ class TestContainerBroker(unittest.TestCase):
timestamp = Timestamp.now().internal
broker.put_object('"{<object \'&\' name>}"', timestamp, 0, '', '',
deleted=1)
broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM object").fetchone()[0],
@ -1047,6 +1077,7 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('"{<object \'&\' name>}"', timestamp, 123,
'application/x-test',
'5af83e3196bf99f440f31f2e1a6c9afe')
broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM object").fetchone()[0],
@ -1095,6 +1126,7 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('"{<object \'&\' name>}"', timestamp, 456,
'application/x-test3',
'6af83e3196bf99f440f31f2e1a6c9afe')
broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM object").fetchone()[0],
@ -1114,7 +1146,8 @@ class TestContainerBroker(unittest.TestCase):
def test_merge_shard_range_single_record(self):
# Test ContainerBroker.merge_shard_range
broker = ContainerBroker(':memory:', account='a', container='c')
broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0)
# Stash these for later
@ -1419,7 +1452,8 @@ class TestContainerBroker(unittest.TestCase):
def test_merge_shard_ranges_deleted(self):
# Test ContainerBroker.merge_shard_ranges sets deleted attribute
broker = ContainerBroker(':memory:', account='a', container='c')
broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0)
# put shard range
broker.merge_shard_ranges(ShardRange('a/o', next(self.ts).internal))
@ -1452,7 +1486,8 @@ class TestContainerBroker(unittest.TestCase):
'storage_policy_index': '2',
'ctype_timestamp': None,
'meta_timestamp': None}
broker = ContainerBroker(':memory:', account='a', container='c')
broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
expect = ('obj', '1234567890.12345', 42, 'text/plain', 'hash_test',
'1', '2', None, None)
@ -1626,7 +1661,8 @@ class TestContainerBroker(unittest.TestCase):
def test_put_object_multiple_encoded_timestamps_using_memory(self):
# Test ContainerBroker.put_object with differing data, content-type
# and metadata timestamps
broker = ContainerBroker(':memory:', account='a', container='c')
broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
self._test_put_object_multiple_encoded_timestamps(broker)
@with_tempdir
@ -1963,7 +1999,8 @@ class TestContainerBroker(unittest.TestCase):
def test_put_object_multiple_explicit_timestamps_using_memory(self):
# Test ContainerBroker.put_object with differing data, content-type
# and metadata timestamps passed as explicit args
broker = ContainerBroker(':memory:', account='a', container='c')
broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
self._test_put_object_multiple_explicit_timestamps(broker)
@with_tempdir
@ -1979,7 +2016,8 @@ class TestContainerBroker(unittest.TestCase):
# Test container listing reports the most recent of data or metadata
# timestamp as last-modified time
ts = make_timestamp_iter()
broker = ContainerBroker(':memory:', account='a', container='c')
broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(next(ts).internal, 0)
# simple 'single' timestamp case
@ -2032,7 +2070,7 @@ class TestContainerBroker(unittest.TestCase):
def test_put_misplaced_object_does_not_effect_container_stats(self):
policy = random.choice(list(POLICIES))
ts = make_timestamp_iter()
broker = ContainerBroker(':memory:',
broker = ContainerBroker(self.get_db_path(),
account='a', container='c')
broker.initialize(next(ts).internal, policy.idx)
# migration tests may not honor policy on initialize
@ -2059,7 +2097,7 @@ class TestContainerBroker(unittest.TestCase):
def test_has_multiple_policies(self):
policy = random.choice(list(POLICIES))
ts = make_timestamp_iter()
broker = ContainerBroker(':memory:',
broker = ContainerBroker(self.get_db_path(),
account='a', container='c')
broker.initialize(next(ts).internal, policy.idx)
# migration tests may not honor policy on initialize
@ -2071,18 +2109,21 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('correct_o', next(ts).internal, 123, 'text/plain',
'5af83e3196bf99f440f31f2e1a6c9afe',
storage_policy_index=policy.idx)
# commit pending file into db
broker._commit_puts()
self.assertFalse(broker.has_multiple_policies())
other_policy = [p for p in POLICIES if p is not policy][0]
broker.put_object('wrong_o', next(ts).internal, 123, 'text/plain',
'5af83e3196bf99f440f31f2e1a6c9afe',
storage_policy_index=other_policy.idx)
broker._commit_puts()
self.assertTrue(broker.has_multiple_policies())
@patch_policies
def test_get_policy_info(self):
policy = random.choice(list(POLICIES))
ts = make_timestamp_iter()
broker = ContainerBroker(':memory:',
broker = ContainerBroker(self.get_db_path(),
account='a', container='c')
broker.initialize(next(ts).internal, policy.idx)
# migration tests may not honor policy on initialize
@ -2099,6 +2140,8 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('correct_o', next(ts).internal, 123, 'text/plain',
'5af83e3196bf99f440f31f2e1a6c9afe',
storage_policy_index=policy.idx)
# commit pending file into db
broker._commit_puts()
policy_stats = broker.get_policy_stats()
expected = {policy.idx: {'bytes_used': 123, 'object_count': 1}}
self.assertEqual(policy_stats, expected)
@ -2109,6 +2152,7 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('wrong_o', next(ts).internal, 123, 'text/plain',
'5af83e3196bf99f440f31f2e1a6c9afe',
storage_policy_index=other_policy.idx)
broker._commit_puts()
policy_stats = broker.get_policy_stats()
expected = {
policy.idx: {'bytes_used': 123, 'object_count': 1},
@ -2119,7 +2163,7 @@ class TestContainerBroker(unittest.TestCase):
@patch_policies
def test_policy_stat_tracking(self):
ts = make_timestamp_iter()
broker = ContainerBroker(':memory:',
broker = ContainerBroker(self.get_db_path(),
account='a', container='c')
# Note: in subclasses of this TestCase that inherit the
# ContainerBrokerMigrationMixin, passing POLICIES.default.idx here has
@ -2151,6 +2195,8 @@ class TestContainerBroker(unittest.TestCase):
# track the size of the latest timestamp put for each object
# in each storage policy
stats[policy_index][name] = size
# commit pending file into db
broker._commit_puts()
policy_stats = broker.get_policy_stats()
if POLICIES.default.idx not in stats:
# unlikely, but check empty default index still in policy stats
@ -2163,7 +2209,7 @@ class TestContainerBroker(unittest.TestCase):
sum(stats[policy_index].values()))
def test_initialize_container_broker_in_default(self):
broker = ContainerBroker(':memory:', account='test1',
broker = ContainerBroker(self.get_db_path(), account='test1',
container='test2')
# initialize with no storage_policy_index argument
@ -2202,7 +2248,7 @@ class TestContainerBroker(unittest.TestCase):
def test_get_info(self):
# Test ContainerBroker.get_info
broker = ContainerBroker(':memory:', account='test1',
broker = ContainerBroker(self.get_db_path(), account='test1',
container='test2')
broker.initialize(Timestamp('1').internal, 0)
@ -2343,7 +2389,7 @@ class TestContainerBroker(unittest.TestCase):
'db_state': 'collapsed'})
def test_set_x_syncs(self):
broker = ContainerBroker(':memory:', account='test1',
broker = ContainerBroker(self.get_db_path(), account='test1',
container='test2')
broker.initialize(Timestamp('1').internal, 0)
@ -2357,7 +2403,7 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual(info['x_container_sync_point2'], 2)
def test_get_report_info(self):
broker = ContainerBroker(':memory:', account='test1',
broker = ContainerBroker(self.get_db_path(), account='test1',
container='test2')
broker.initialize(Timestamp('1').internal, 0)
@ -2529,7 +2575,8 @@ class TestContainerBroker(unittest.TestCase):
self.assertFalse(get_rows(broker))
def test_get_objects(self):
broker = ContainerBroker(':memory:', account='a', container='c')
broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0)
objects_0 = [{'name': 'obj_0_%d' % i,
'created_at': next(self.ts).normal,
@ -2577,7 +2624,8 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual(objects_0 + objects_1, actual)
def test_get_objects_since_row(self):
broker = ContainerBroker(':memory:', account='a', container='c')
broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0)
obj_names = ['obj%03d' % i for i in range(20)]
timestamps = [next(self.ts) for o in obj_names]
@ -2627,7 +2675,8 @@ class TestContainerBroker(unittest.TestCase):
def test_list_objects_iter(self):
# Test ContainerBroker.list_objects_iter
broker = ContainerBroker(':memory:', account='a', container='c')
broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0)
for obj1 in range(4):
for obj2 in range(125):
@ -2777,7 +2826,8 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual([row[0] for row in listing], ['3/0000', '3/0001'])
def test_list_objects_iter_with_reserved_name(self):
broker = ContainerBroker(':memory:', account='a', container='c')
broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(next(self.ts).internal, 0)
broker.put_object(
@ -2936,10 +2986,13 @@ class TestContainerBroker(unittest.TestCase):
}
failures = []
for expected in expectations:
broker = ContainerBroker(':memory:', account='a', container='c')
broker = ContainerBroker(self.get_db_path(),
account='a', container='c')
broker.initialize(next(ts).internal, 0)
for name in expected['objects']:
broker.put_object(name, next(ts).internal, **obj_create_params)
# commit pending file into db
broker._commit_puts()
params = default_listing_params.copy()
params.update(expected['params'])
listing = list(o[0] for o in broker.list_objects_iter(**params))
@ -2954,7 +3007,8 @@ class TestContainerBroker(unittest.TestCase):
def test_list_objects_iter_non_slash(self):
# Test ContainerBroker.list_objects_iter using a
# delimiter that is not a slash
broker = ContainerBroker(':memory:', account='a', container='c')
broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0)
for obj1 in range(4):
for obj2 in range(125):
@ -3071,7 +3125,8 @@ class TestContainerBroker(unittest.TestCase):
def test_list_objects_iter_prefix_delim(self):
# Test ContainerBroker.list_objects_iter
broker = ContainerBroker(':memory:', account='a', container='c')
broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0)
broker.put_object(
@ -3110,7 +3165,8 @@ class TestContainerBroker(unittest.TestCase):
def test_list_objects_iter_order_and_reverse(self):
# Test ContainerBroker.list_objects_iter
broker = ContainerBroker(':memory:', account='a', container='c')
broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0)
broker.put_object(
@ -3152,7 +3208,8 @@ class TestContainerBroker(unittest.TestCase):
def test_double_check_trailing_delimiter(self):
# Test ContainerBroker.list_objects_iter for a
# container that has an odd file with a trailing delimiter
broker = ContainerBroker(':memory:', account='a', container='c')
broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0)
broker.put_object('a', Timestamp.now().internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
@ -3232,7 +3289,8 @@ class TestContainerBroker(unittest.TestCase):
def test_double_check_trailing_delimiter_non_slash(self):
# Test ContainerBroker.list_objects_iter for a
# container that has an odd file with a trailing delimiter
broker = ContainerBroker(':memory:', account='a', container='c')
broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0)
broker.put_object('a', Timestamp.now().internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
@ -3315,7 +3373,8 @@ class TestContainerBroker(unittest.TestCase):
s = s.encode('utf8')
return md5(s, usedforsecurity=False).hexdigest()
broker = ContainerBroker(':memory:', account='a', container='c')
broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0)
broker.put_object('a', Timestamp(1).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
@ -3367,7 +3426,8 @@ class TestContainerBroker(unittest.TestCase):
def test_get_items_since(self):
# test DatabaseBroker.get_items_since
broker = ContainerBroker(':memory:', account='a', container='c')
broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0)
broker.put_object('a', Timestamp(1).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
@ -3380,9 +3440,11 @@ class TestContainerBroker(unittest.TestCase):
def test_sync_merging(self):
# exercise the DatabaseBroker sync functions a bit
broker1 = ContainerBroker(':memory:', account='a', container='c')
broker1 = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker1.initialize(Timestamp('1').internal, 0)
broker2 = ContainerBroker(':memory:', account='a', container='c')
broker2 = ContainerBroker(self.get_db_path(),
account='a', container='c')
broker2.initialize(Timestamp('1').internal, 0)
self.assertEqual(broker2.get_sync('12345'), -1)
broker1.merge_syncs([{'sync_point': 3, 'remote_id': '12345'}])
@ -3390,14 +3452,18 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual(broker2.get_sync('12345'), 3)
def test_merge_items(self):
broker1 = ContainerBroker(':memory:', account='a', container='c')
broker1 = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker1.initialize(Timestamp('1').internal, 0)
broker2 = ContainerBroker(':memory:', account='a', container='c')
broker2 = ContainerBroker(self.get_db_path(),
account='a', container='c')
broker2.initialize(Timestamp('1').internal, 0)
broker1.put_object('a', Timestamp(1).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker1.put_object('b', Timestamp(2).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
# commit pending file into db
broker1._commit_puts()
id = broker1.get_info()['id']
broker2.merge_items(broker1.get_items_since(
broker2.get_sync(id), 1000), id)
@ -3406,6 +3472,7 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual(['a', 'b'], sorted([rec['name'] for rec in items]))
broker1.put_object('c', Timestamp(3).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker1._commit_puts()
broker2.merge_items(broker1.get_items_since(
broker2.get_sync(id), 1000), id)
items = broker2.get_items_since(-1, 1000)
@ -3433,19 +3500,24 @@ class TestContainerBroker(unittest.TestCase):
snowman = u'\N{SNOWMAN}'
if six.PY2:
snowman = snowman.encode('utf-8')
broker1 = ContainerBroker(':memory:', account='a', container='c')
broker1 = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker1.initialize(Timestamp('1').internal, 0)
id = broker1.get_info()['id']
broker2 = ContainerBroker(':memory:', account='a', container='c')
broker2 = ContainerBroker(self.get_db_path(),
account='a', container='c')
broker2.initialize(Timestamp('1').internal, 0)
broker1.put_object(snowman, Timestamp(2).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker1.put_object('b', Timestamp(3).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
# commit pending file into db
broker1._commit_puts()
broker2.merge_items(json.loads(json.dumps(broker1.get_items_since(
broker2.get_sync(id), 1000))), id)
broker1.put_object(snowman, Timestamp(4).internal, 0, 'text/plain',
'd41d8cd98f00b204e9800998ecf8427e')
broker1._commit_puts()
broker2.merge_items(json.loads(json.dumps(broker1.get_items_since(
broker2.get_sync(id), 1000))), id)
items = broker2.get_items_since(-1, 1000)
@ -3459,19 +3531,24 @@ class TestContainerBroker(unittest.TestCase):
def test_merge_items_overwrite(self):
# test DatabaseBroker.merge_items
broker1 = ContainerBroker(':memory:', account='a', container='c')
broker1 = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker1.initialize(Timestamp('1').internal, 0)
id = broker1.get_info()['id']
broker2 = ContainerBroker(':memory:', account='a', container='c')
broker2 = ContainerBroker(self.get_db_path(),
account='a', container='c')
broker2.initialize(Timestamp('1').internal, 0)
broker1.put_object('a', Timestamp(2).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker1.put_object('b', Timestamp(3).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
# commit pending file into db
broker1._commit_puts()
broker2.merge_items(broker1.get_items_since(
broker2.get_sync(id), 1000), id)
broker1.put_object('a', Timestamp(4).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker1._commit_puts()
broker2.merge_items(broker1.get_items_since(
broker2.get_sync(id), 1000), id)
items = broker2.get_items_since(-1, 1000)
@ -3484,19 +3561,24 @@ class TestContainerBroker(unittest.TestCase):
def test_merge_items_post_overwrite_out_of_order(self):
# test DatabaseBroker.merge_items
broker1 = ContainerBroker(':memory:', account='a', container='c')
broker1 = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker1.initialize(Timestamp('1').internal, 0)
id = broker1.get_info()['id']
broker2 = ContainerBroker(':memory:', account='a', container='c')
broker2 = ContainerBroker(self.get_db_path(),
account='a', container='c')
broker2.initialize(Timestamp('1').internal, 0)
broker1.put_object('a', Timestamp(2).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker1.put_object('b', Timestamp(3).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
# commit pending file into db
broker1._commit_puts()
broker2.merge_items(broker1.get_items_since(
broker2.get_sync(id), 1000), id)
broker1.put_object('a', Timestamp(4).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker1._commit_puts()
broker2.merge_items(broker1.get_items_since(
broker2.get_sync(id), 1000), id)
items = broker2.get_items_since(-1, 1000)
@ -3516,6 +3598,7 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual(rec['created_at'], Timestamp(3).internal)
broker1.put_object('b', Timestamp(5).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker1._commit_puts()
broker2.merge_items(broker1.get_items_since(
broker2.get_sync(id), 1000), id)
items = broker2.get_items_since(-1, 1000)
@ -3529,7 +3612,8 @@ class TestContainerBroker(unittest.TestCase):
def test_set_storage_policy_index(self):
ts = make_timestamp_iter()
broker = ContainerBroker(':memory:', account='test_account',
broker = ContainerBroker(self.get_db_path(),
account='test_account',
container='test_container')
timestamp = next(ts)
broker.initialize(timestamp.internal, 0)
@ -3586,7 +3670,8 @@ class TestContainerBroker(unittest.TestCase):
def test_set_storage_policy_index_empty(self):
# Putting an object may trigger migrations, so test with a
# never-had-an-object container to make sure we handle it
broker = ContainerBroker(':memory:', account='test_account',
broker = ContainerBroker(self.get_db_path(),
account='test_account',
container='test_container')
broker.initialize(Timestamp('1').internal, 0)
info = broker.get_info()
@ -3597,7 +3682,8 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual(2, info['storage_policy_index'])
def test_reconciler_sync(self):
broker = ContainerBroker(':memory:', account='test_account',
broker = ContainerBroker(self.get_db_path(),
account='test_account',
container='test_container')
broker.initialize(Timestamp('1').internal, 0)
self.assertEqual(-1, broker.get_reconciler_sync())
@ -5339,6 +5425,7 @@ class TestContainerBroker(unittest.TestCase):
class TestCommonContainerBroker(test_db.TestExampleBroker):
broker_class = ContainerBroker
server_type = 'container'
def setUp(self):
super(TestCommonContainerBroker, self).setUp()
@ -5353,7 +5440,7 @@ class TestCommonContainerBroker(test_db.TestExampleBroker):
storage_policy_index=int(self.policy))
class ContainerBrokerMigrationMixin(object):
class ContainerBrokerMigrationMixin(test_db.TestDbBase):
"""
Mixin for running ContainerBroker against databases created with
older schemas.
@ -5368,6 +5455,7 @@ class ContainerBrokerMigrationMixin(object):
return self.func.__get__(obj, obj_type)
def setUp(self):
super(ContainerBrokerMigrationMixin, self).setUp()
self._imported_create_object_table = \
ContainerBroker.create_object_table
ContainerBroker.create_object_table = \
@ -5408,6 +5496,7 @@ class ContainerBrokerMigrationMixin(object):
self._imported_create_shard_range_table
ContainerBroker.create_policy_stat_table = \
self._imported_create_policy_stat_table
# We need to manually teardown and clean the self.tempdir
def premetadata_create_container_info_table(self, conn, put_timestamp,
@ -5464,7 +5553,8 @@ class TestContainerBrokerBeforeMetadata(ContainerBrokerMigrationMixin,
def setUp(self):
super(TestContainerBrokerBeforeMetadata, self).setUp()
broker = ContainerBroker(':memory:', account='a', container='c')
broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0)
exc = None
with broker.get() as conn:
@ -5476,10 +5566,12 @@ class TestContainerBrokerBeforeMetadata(ContainerBrokerMigrationMixin,
def tearDown(self):
super(TestContainerBrokerBeforeMetadata, self).tearDown()
broker = ContainerBroker(':memory:', account='a', container='c')
broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0)
with broker.get() as conn:
conn.execute('SELECT metadata FROM container_stat')
test_db.TestDbBase.tearDown(self)
def prexsync_create_container_info_table(self, conn, put_timestamp,
@ -5540,7 +5632,8 @@ class TestContainerBrokerBeforeXSync(ContainerBrokerMigrationMixin,
super(TestContainerBrokerBeforeXSync, self).setUp()
ContainerBroker.create_container_info_table = \
prexsync_create_container_info_table
broker = ContainerBroker(':memory:', account='a', container='c')
broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0)
exc = None
with broker.get() as conn:
@ -5553,10 +5646,12 @@ class TestContainerBrokerBeforeXSync(ContainerBrokerMigrationMixin,
def tearDown(self):
super(TestContainerBrokerBeforeXSync, self).tearDown()
broker = ContainerBroker(':memory:', account='a', container='c')
broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0)
with broker.get() as conn:
conn.execute('SELECT x_container_sync_point1 FROM container_stat')
test_db.TestDbBase.tearDown(self)
def prespi_create_object_table(self, conn, *args, **kwargs):
@ -5657,7 +5752,8 @@ class TestContainerBrokerBeforeSPI(ContainerBrokerMigrationMixin,
ContainerBroker.create_container_info_table = \
prespi_create_container_info_table
broker = ContainerBroker(':memory:', account='a', container='c')
broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0)
with self.assertRaises(sqlite3.DatabaseError) as raised, \
broker.get() as conn:
@ -5668,10 +5764,12 @@ class TestContainerBrokerBeforeSPI(ContainerBrokerMigrationMixin,
def tearDown(self):
super(TestContainerBrokerBeforeSPI, self).tearDown()
broker = ContainerBroker(':memory:', account='a', container='c')
broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0)
with broker.get() as conn:
conn.execute('SELECT storage_policy_index FROM container_stat')
test_db.TestDbBase.tearDown(self)
@patch_policies
@with_tempdir
@ -5865,7 +5963,8 @@ class TestContainerBrokerBeforeShardRanges(ContainerBrokerMigrationMixin,
def setUp(self):
super(TestContainerBrokerBeforeShardRanges, self).setUp()
broker = ContainerBroker(':memory:', account='a', container='c')
broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0)
with self.assertRaises(sqlite3.DatabaseError) as raised, \
broker.get() as conn:
@ -5875,11 +5974,13 @@ class TestContainerBrokerBeforeShardRanges(ContainerBrokerMigrationMixin,
def tearDown(self):
super(TestContainerBrokerBeforeShardRanges, self).tearDown()
broker = ContainerBroker(':memory:', account='a', container='c')
broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0)
with broker.get() as conn:
conn.execute('''SELECT *
FROM shard_range''')
test_db.TestDbBase.tearDown(self)
def pre_reported_create_shard_range_table(self, conn):
@ -5933,7 +6034,8 @@ class TestContainerBrokerBeforeShardRangeReportedColumn(
ContainerBroker.create_shard_range_table = \
pre_reported_create_shard_range_table
broker = ContainerBroker(':memory:', account='a', container='c')
broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0)
with self.assertRaises(sqlite3.DatabaseError) as raised, \
broker.get() as conn:
@ -5944,11 +6046,13 @@ class TestContainerBrokerBeforeShardRangeReportedColumn(
def tearDown(self):
super(TestContainerBrokerBeforeShardRangeReportedColumn,
self).tearDown()
broker = ContainerBroker(':memory:', account='a', container='c')
broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0)
with broker.get() as conn:
conn.execute('''SELECT reported
FROM shard_range''')
test_db.TestDbBase.tearDown(self)
@with_tempdir
def test_get_shard_ranges_attempts(self, tempdir):
@ -6161,7 +6265,8 @@ class TestContainerBrokerBeforeShardRangeTombstonesColumn(
ContainerBroker.create_shard_range_table = \
pre_tombstones_create_shard_range_table
broker = ContainerBroker(':memory:', account='a', container='c')
broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0)
with self.assertRaises(sqlite3.DatabaseError) as raised, \
broker.get() as conn:
@ -6172,11 +6277,13 @@ class TestContainerBrokerBeforeShardRangeTombstonesColumn(
def tearDown(self):
super(TestContainerBrokerBeforeShardRangeTombstonesColumn,
self).tearDown()
broker = ContainerBroker(':memory:', account='a', container='c')
broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0)
with broker.get() as conn:
conn.execute('''SELECT tombstones
FROM shard_range''')
test_db.TestDbBase.tearDown(self)
class TestUpdateNewItemFromExisting(unittest.TestCase):