openstack/cinder
Code Issues Proposed changes

db: Vendor 'oslo_db.sqlalchemy.migration'

Browse Source 
This is deprecated and will be removed in a future release of oslo.db.
Even without that stick to prod us, we're going to need to use some of
the sqlalchemy-migrate APIs and it's simpler to talk to this for
everything rather than using oslo.db for some stuff and
sqlalchemy-migrate for the remainder.

Change-Id: I42e670f51591d8fee2cfbd1f1f386f8e04704732
Signed-off-by: Stephen Finucane <stephenfin@redhat.com>
This commit is contained in:
Stephen Finucane 2021-04-20 12:37:26 +01:00
parent e823386cbe
commit 1782729447
5 changed files with 369 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
cinder/cmd/manage.py
View File

@ -63,7 +63,6 @@ import time
from oslo_config import cfg
from oslo_db import exception as db_exc
from oslo_db.sqlalchemy import migration
from oslo_log import log as logging
from oslo_utils import timeutils
import tabulate
@ -204,9 +203,7 @@ class DbCommands(object):
def version(self):
"""Print the current database version."""
print(migration.db_version(db_api.get_engine(),
db_migration.LEGACY_MIGRATIONS_PATH,
db_migration.INIT_VERSION))
print(db_migration.db_version())
@args('age_in_days', type=int,
help='Purge deleted rows older than age in days')

110
cinder/db/migration.py
View File

@ -18,11 +18,16 @@
import os
from migrate import exceptions as migrate_exceptions
from migrate.versioning import api as migrate_api
from migrate.versioning import repository as migrate_repository
from oslo_config import cfg
from oslo_db import exception
from oslo_db import options
from oslo_db.sqlalchemy import migration
import sqlalchemy as sa
from cinder.db.sqlalchemy import api as db_api
from cinder.i18n import _
options.set_defaults(cfg.CONF)
@ -33,13 +38,114 @@ LEGACY_MIGRATIONS_PATH = os.path.join(
)
def _find_migrate_repo(abs_path):
"""Get the project's change script repository
:param abs_path: Absolute path to migrate repository
"""
if not os.path.exists(abs_path):
raise exception.DBMigrationError("Path %s not found" % abs_path)
return migrate_repository.Repository(abs_path)
def _migrate_db_version_control(engine, abs_path, version=None):
"""Mark a database as under this repository's version control.
Once a database is under version control, schema changes should
only be done via change scripts in this repository.
:param engine: SQLAlchemy engine instance for a given database
:param abs_path: Absolute path to migrate repository
:param version: Initial database version
"""
repository = _find_migrate_repo(abs_path)
try:
migrate_api.version_control(engine, repository, version)
except migrate_exceptions.InvalidVersionError as ex:
raise exception.DBMigrationError("Invalid version : %s" % ex)
except migrate_exceptions.DatabaseAlreadyControlledError:
raise exception.DBMigrationError("Database is already controlled.")
return version
def _migrate_db_version(engine, abs_path, init_version):
"""Show the current version of the repository.
:param engine: SQLAlchemy engine instance for a given database
:param abs_path: Absolute path to migrate repository
:param init_version: Initial database version
"""
repository = _find_migrate_repo(abs_path)
try:
return migrate_api.db_version(engine, repository)
except migrate_exceptions.DatabaseNotControlledError:
pass
meta = sa.MetaData()
meta.reflect(bind=engine)
tables = meta.tables
if (
len(tables) == 0 or
'alembic_version' in tables or
'migrate_version' in tables
):
_migrate_db_version_control(engine, abs_path, version=init_version)
return migrate_api.db_version(engine, repository)
msg = _(
"The database is not under version control, but has tables. "
"Please stamp the current version of the schema manually."
)
raise exception.DBMigrationError(msg)
def _migrate_db_sync(engine, abs_path, version=None, init_version=0):
"""Upgrade or downgrade a database.
Function runs the upgrade() or downgrade() functions in change scripts.
:param engine: SQLAlchemy engine instance for a given database
:param abs_path: Absolute path to migrate repository.
:param version: Database will upgrade/downgrade until this version.
If None - database will update to the latest available version.
:param init_version: Initial database version
"""
if version is not None:
try:
version = int(version)
except ValueError:
raise exception.DBMigrationError(_("version should be an integer"))
current_version = _migrate_db_version(engine, abs_path, init_version)
repository = _find_migrate_repo(abs_path)
if version is None or version > current_version:
try:
return migrate_api.upgrade(engine, repository, version)
except Exception as ex:
raise exception.DBMigrationError(ex)
else:
return migrate_api.downgrade(engine, repository, version)
def db_version():
"""Get database version."""
return _migrate_db_version(
db_api.get_engine(),
LEGACY_MIGRATIONS_PATH,
INIT_VERSION)
def db_sync(version=None, engine=None):
"""Migrate the database to `version` or the most recent version."""
if engine is None:
engine = db_api.get_engine()
return migration.db_sync(
return _migrate_db_sync(
engine=engine,
abs_path=LEGACY_MIGRATIONS_PATH,
version=version,

252
cinder/tests/unit/db/test_migration.py Normal file
View File

@ -0,0 +1,252 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import tempfile
from unittest import mock
from migrate import exceptions as migrate_exception
from migrate.versioning import api as migrate_api
from migrate.versioning import repository as migrate_repository
from oslo_db import exception as db_exception
from oslo_db.sqlalchemy import enginefacade
from oslo_db.sqlalchemy import test_fixtures as db_fixtures
from oslotest import base as test_base
import sqlalchemy
from cinder.db import migration
from cinder import utils
class TestMigrationCommon(
db_fixtures.OpportunisticDBTestMixin, test_base.BaseTestCase,
):
def setUp(self):
super().setUp()
self.engine = enginefacade.writer.get_engine()
self.path = tempfile.mkdtemp('test_migration')
self.path1 = tempfile.mkdtemp('test_migration')
self.return_value = '/home/openstack/migrations'
self.return_value1 = '/home/extension/migrations'
self.init_version = 1
self.test_version = 123
self.patcher_repo = mock.patch.object(migrate_repository, 'Repository')
self.repository = self.patcher_repo.start()
self.repository.side_effect = [self.return_value, self.return_value1]
self.mock_api_db = mock.patch.object(migrate_api, 'db_version')
self.mock_api_db_version = self.mock_api_db.start()
self.mock_api_db_version.return_value = self.test_version
def tearDown(self):
os.rmdir(self.path)
self.mock_api_db.stop()
self.patcher_repo.stop()
super().tearDown()
def test_find_migrate_repo_path_not_found(self):
self.assertRaises(
db_exception.DBMigrationError,
migration._find_migrate_repo,
"/foo/bar/",
)
def test_find_migrate_repo_called_once(self):
my_repository = migration._find_migrate_repo(self.path)
self.repository.assert_called_once_with(self.path)
self.assertEqual(self.return_value, my_repository)
def test_find_migrate_repo_called_few_times(self):
repo1 = migration._find_migrate_repo(self.path)
repo2 = migration._find_migrate_repo(self.path1)
self.assertNotEqual(repo1, repo2)
def test_db_version_control(self):
with utils.nested_contexts(
mock.patch.object(migration, '_find_migrate_repo'),
mock.patch.object(migrate_api, 'version_control'),
) as (mock_find_repo, mock_version_control):
mock_find_repo.return_value = self.return_value
version = migration._migrate_db_version_control(
self.engine, self.path, self.test_version)
self.assertEqual(self.test_version, version)
mock_version_control.assert_called_once_with(
self.engine, self.return_value, self.test_version)
@mock.patch.object(migration, '_find_migrate_repo')
@mock.patch.object(migrate_api, 'version_control')
def test_db_version_control_version_less_than_actual_version(
self, mock_version_control, mock_find_repo,
):
mock_find_repo.return_value = self.return_value
mock_version_control.side_effect = \
migrate_exception.DatabaseAlreadyControlledError
self.assertRaises(
db_exception.DBMigrationError,
migration._migrate_db_version_control, self.engine,
self.path, self.test_version - 1)
@mock.patch.object(migration, '_find_migrate_repo')
@mock.patch.object(migrate_api, 'version_control')
def test_db_version_control_version_greater_than_actual_version(
self, mock_version_control, mock_find_repo,
):
mock_find_repo.return_value = self.return_value
mock_version_control.side_effect = \
migrate_exception.InvalidVersionError
self.assertRaises(
db_exception.DBMigrationError,
migration._migrate_db_version_control, self.engine,
self.path, self.test_version + 1)
def test_db_version_return(self):
ret_val = migration._migrate_db_version(
self.engine, self.path, self.init_version)
self.assertEqual(self.test_version, ret_val)
def test_db_version_raise_not_controlled_error_first(self):
with mock.patch.object(
migration, '_migrate_db_version_control',
) as mock_ver:
self.mock_api_db_version.side_effect = [
migrate_exception.DatabaseNotControlledError('oups'),
self.test_version]
ret_val = migration._migrate_db_version(
self.engine, self.path, self.init_version)
self.assertEqual(self.test_version, ret_val)
mock_ver.assert_called_once_with(
self.engine, self.path, version=self.init_version)
def test_db_version_raise_not_controlled_error_tables(self):
with mock.patch.object(sqlalchemy, 'MetaData') as mock_meta:
self.mock_api_db_version.side_effect = \
migrate_exception.DatabaseNotControlledError('oups')
my_meta = mock.MagicMock()
my_meta.tables = {'a': 1, 'b': 2}
mock_meta.return_value = my_meta
self.assertRaises(
db_exception.DBMigrationError, migration._migrate_db_version,
self.engine, self.path, self.init_version)
@mock.patch.object(migrate_api, 'version_control')
def test_db_version_raise_not_controlled_error_no_tables(self, mock_vc):
with mock.patch.object(sqlalchemy, 'MetaData') as mock_meta:
self.mock_api_db_version.side_effect = (
migrate_exception.DatabaseNotControlledError('oups'),
self.init_version)
my_meta = mock.MagicMock()
my_meta.tables = {}
mock_meta.return_value = my_meta
migration._migrate_db_version(
self.engine, self.path, self.init_version)
mock_vc.assert_called_once_with(
self.engine, self.return_value1, self.init_version)
@mock.patch.object(migrate_api, 'version_control')
def test_db_version_raise_not_controlled_alembic_tables(self, mock_vc):
# When there are tables but the alembic control table
# (alembic_version) is present, attempt to version the db.
# This simulates the case where there is are multiple repos (different
# abs_paths) and a different path has been versioned already.
with mock.patch.object(sqlalchemy, 'MetaData') as mock_meta:
self.mock_api_db_version.side_effect = [
migrate_exception.DatabaseNotControlledError('oups'), None]
my_meta = mock.MagicMock()
my_meta.tables = {'alembic_version': 1, 'b': 2}
mock_meta.return_value = my_meta
migration._migrate_db_version(
self.engine, self.path, self.init_version)
mock_vc.assert_called_once_with(
self.engine, self.return_value1, self.init_version)
@mock.patch.object(migrate_api, 'version_control')
def test_db_version_raise_not_controlled_migrate_tables(self, mock_vc):
# When there are tables but the sqlalchemy-migrate control table
# (migrate_version) is present, attempt to version the db.
# This simulates the case where there is are multiple repos (different
# abs_paths) and a different path has been versioned already.
with mock.patch.object(sqlalchemy, 'MetaData') as mock_meta:
self.mock_api_db_version.side_effect = [
migrate_exception.DatabaseNotControlledError('oups'), None]
my_meta = mock.MagicMock()
my_meta.tables = {'migrate_version': 1, 'b': 2}
mock_meta.return_value = my_meta
migration._migrate_db_version(
self.engine, self.path, self.init_version)
mock_vc.assert_called_once_with(
self.engine, self.return_value1, self.init_version)
def test_db_sync_wrong_version(self):
self.assertRaises(
db_exception.DBMigrationError,
migration._migrate_db_sync, self.engine, self.path, 'foo')
@mock.patch.object(migrate_api, 'upgrade')
def test_db_sync_script_not_present(self, upgrade):
# For non existent migration script file sqlalchemy-migrate will raise
# VersionNotFoundError which will be wrapped in DBMigrationError.
upgrade.side_effect = migrate_exception.VersionNotFoundError
self.assertRaises(
db_exception.DBMigrationError,
migration._migrate_db_sync, self.engine, self.path,
self.test_version + 1)
@mock.patch.object(migrate_api, 'upgrade')
def test_db_sync_known_error_raised(self, upgrade):
upgrade.side_effect = migrate_exception.KnownError
self.assertRaises(
db_exception.DBMigrationError,
migration._migrate_db_sync, self.engine, self.path,
self.test_version + 1)
def test_db_sync_upgrade(self):
init_ver = 55
with utils.nested_contexts(
mock.patch.object(migration, '_find_migrate_repo'),
mock.patch.object(migrate_api, 'upgrade')
) as (mock_find_repo, mock_upgrade):
mock_find_repo.return_value = self.return_value
self.mock_api_db_version.return_value = self.test_version - 1
migration._migrate_db_sync(
self.engine, self.path, self.test_version, init_ver)
mock_upgrade.assert_called_once_with(
self.engine, self.return_value, self.test_version)
def test_db_sync_downgrade(self):
with utils.nested_contexts(
mock.patch.object(migration, '_find_migrate_repo'),
mock.patch.object(migrate_api, 'downgrade')
) as (mock_find_repo, mock_downgrade):
mock_find_repo.return_value = self.return_value
self.mock_api_db_version.return_value = self.test_version + 1
migration._migrate_db_sync(
self.engine, self.path, self.test_version)
mock_downgrade.assert_called_once_with(
self.engine, self.return_value, self.test_version)

4
cinder/tests/unit/test_cmd.py
View File

@ -380,7 +380,7 @@ class TestCinderManageCmd(test.TestCase):
self.assertEqual(cinder_manage.OVO_VERSION,
service.object_current_version)
@mock.patch('oslo_db.sqlalchemy.migration.db_version')
@mock.patch('cinder.db.migration.db_version')
def test_db_commands_version(self, db_version):
db_cmds = cinder_manage.DbCommands()
with mock.patch('sys.stdout', new=io.StringIO()):
@ -393,7 +393,7 @@ class TestCinderManageCmd(test.TestCase):
exit = self.assertRaises(SystemExit, db_cmds.sync, version + 1)
self.assertEqual(1, exit.code)
@mock.patch("oslo_db.sqlalchemy.migration.db_sync")
@mock.patch('cinder.db.migration.db_sync')
def test_db_commands_script_not_present(self, db_sync):
db_sync.side_effect = oslo_exception.DBMigrationError(None)
db_cmds = cinder_manage.DbCommands()

6
cinder/utils.py
View File

@ -894,6 +894,12 @@ def create_ordereddict(adict: dict) -> OrderedDict:
key=operator.itemgetter(0)))
@contextlib.contextmanager
def nested_contexts(*contexts):
with contextlib.ExitStack() as stack:
yield [stack.enter_context(c) for c in contexts]
class Semaphore(object):
"""Custom semaphore to workaround eventlet issues with multiprocessing."""
def __init__(self, limit):