diff --git a/bin/manila-manage b/bin/manila-manage index fb62e9554e..295174d119 100755 --- a/bin/manila-manage +++ b/bin/manila-manage @@ -215,11 +215,29 @@ class DbCommands(object): help='Database version') def sync(self, version=None): """Sync the database up to the most recent version.""" - return migration.db_sync(version) + return migration.upgrade(version) def version(self): """Print the current database version.""" - print(migration.db_version()) + print(migration.version()) + + @args('version', nargs='?', default=None, + help='Version to downgrade') + def downgrade(self, version=None): + """Downgrade database to the given version.""" + return migration.downgrade(version) + + @args('--message', help='Revision message') + @args('--authogenerate', help='Autogenerate migration from schema') + def revision(self, message, autogenerate): + """Generate new migration.""" + return migration.revision(message, autogenerate) + + @args('version', nargs='?', default=None, + help='Version to stamp version table with') + def stamp(self, version=None): + """Stamp the revision table with the given version.""" + return migration.stamp(version) class VersionCommands(object): diff --git a/doc/source/man/manila-manage.rst b/doc/source/man/manila-manage.rst index 4e00f7f1cb..3bec1aefcd 100644 --- a/doc/source/man/manila-manage.rst +++ b/doc/source/man/manila-manage.rst @@ -51,6 +51,18 @@ Manila Db Sync the database up to the most recent version. This is the standard way to create the db as well. +``manila-manage db downgrade `` + + Downgrade database to given version. + +``manila-manage db stamp `` + + Stamp database with given revision. + +``manila-manage db revision `` + + Generate new migration. + Manila Logs ~~~~~~~~~~~ diff --git a/manila/db/api.py b/manila/db/api.py index 27fa580323..bcf25b18e1 100644 --- a/manila/db/api.py +++ b/manila/db/api.py @@ -129,34 +129,6 @@ def service_update(context, service_id, values): return IMPL.service_update(context, service_id, values) -################### -def migration_update(context, id, values): - """Update a migration instance.""" - return IMPL.migration_update(context, id, values) - - -def migration_create(context, values): - """Create a migration record.""" - return IMPL.migration_create(context, values) - - -def migration_get(context, migration_id): - """Finds a migration by the id.""" - return IMPL.migration_get(context, migration_id) - - -def migration_get_by_instance_and_status(context, instance_uuid, status): - """Finds a migration by the instance uuid its migrating.""" - return IMPL.migration_get_by_instance_and_status(context, - instance_uuid, - status) - - -def migration_get_all_unconfirmed(context, confirm_window): - """Finds all unconfirmed migrations within the confirmation window.""" - return IMPL.migration_get_all_unconfirmed(context, confirm_window) - - #################### diff --git a/manila/db/migration.py b/manila/db/migration.py index a278d0f219..ef8a773e6f 100644 --- a/manila/db/migration.py +++ b/manila/db/migration.py @@ -18,27 +18,33 @@ """Database setup and migration commands.""" -import os - -from manila.db.sqlalchemy import api as db_api from manila import utils -IMPL = utils.LazyPluggable('db_backend', - sqlalchemy='oslo.db.sqlalchemy.migration') +IMPL = utils.LazyPluggable( + 'db_backend', sqlalchemy='manila.db.migrations.alembic.migration') -INIT_VERSION = 000 -MIGRATE_REPO = os.path.join(os.path.dirname(os.path.abspath(__file__)), - 'sqlalchemy/migrate_repo') +def upgrade(version): + """Upgrade database to 'version' or the most recent version.""" + return IMPL.upgrade(version) -def db_sync(version=None): - """Migrate the database to `version` or the most recent version.""" - return IMPL.db_sync(db_api.get_engine(), MIGRATE_REPO, version=version, - init_version=INIT_VERSION) +def downgrade(version): + """Downgrade database to 'version' or to initial state.""" + return IMPL.downgrade(version) -def db_version(): +def version(): """Display the current database version.""" - return IMPL.db_version(db_api.get_engine(), MIGRATE_REPO, INIT_VERSION) + return IMPL.version() + + +def stamp(version): + """Stamp database with 'version' or the most recent version.""" + return IMPL.stamp(version) + + +def revision(message, autogenerate): + """Generate new migration script.""" + return IMPL.revision(message, autogenerate) diff --git a/manila/db/sqlalchemy/migrate_repo/__init__.py b/manila/db/migrations/__init__.py similarity index 100% rename from manila/db/sqlalchemy/migrate_repo/__init__.py rename to manila/db/migrations/__init__.py diff --git a/manila/db/migrations/alembic.ini b/manila/db/migrations/alembic.ini new file mode 100644 index 0000000000..d39bdf9a98 --- /dev/null +++ b/manila/db/migrations/alembic.ini @@ -0,0 +1,59 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = %(here)s/alembic + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +#sqlalchemy.url = driver://user:pass@localhost/dbname + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/manila/db/sqlalchemy/migrate_repo/versions/__init__.py b/manila/db/migrations/alembic/__init__.py similarity index 100% rename from manila/db/sqlalchemy/migrate_repo/versions/__init__.py rename to manila/db/migrations/alembic/__init__.py diff --git a/manila/db/migrations/alembic/env.py b/manila/db/migrations/alembic/env.py new file mode 100644 index 0000000000..ec819e03c9 --- /dev/null +++ b/manila/db/migrations/alembic/env.py @@ -0,0 +1,41 @@ +# Copyright 2014 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import with_statement + +from alembic import context + +from manila.db.sqlalchemy import api as db_api +from manila.db.sqlalchemy import models as db_models + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + """ + engine = db_api.get_engine() + connection = engine.connect() + target_metadata = db_models.ManilaBase.metadata + context.configure(connection=connection, # pylint: disable=E1101 + target_metadata=target_metadata) + try: + with context.begin_transaction(): # pylint: disable=E1101 + context.run_migrations() # pylint: disable=E1101 + finally: + connection.close() + + +run_migrations_online() diff --git a/manila/db/migrations/alembic/migration.py b/manila/db/migrations/alembic/migration.py new file mode 100644 index 0000000000..a30fdbce53 --- /dev/null +++ b/manila/db/migrations/alembic/migration.py @@ -0,0 +1,83 @@ +# Copyright 2014 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +import alembic +from alembic import config as alembic_config +import alembic.migration as alembic_migration +from oslo.config import cfg + +from manila.db.sqlalchemy import api as db_api + +CONF = cfg.CONF + + +def _alembic_config(): + path = os.path.join(os.path.dirname(__file__), os.pardir, 'alembic.ini') + config = alembic_config.Config(path) + return config + + +def version(): + """Current database version. + + :returns: Database version + :rtype: string + """ + engine = db_api.get_engine() + with engine.connect() as conn: + context = alembic_migration.MigrationContext.configure(conn) + return context.get_current_revision() + + +def upgrade(revision): + """Upgrade database. + + :param version: Desired database version + :type version: string + """ + return alembic.command.upgrade(_alembic_config(), revision or 'head') + + +def downgrade(revision): + """Downgrade database. + + :param version: Desired database version + :type version: string + """ + return alembic.command.downgrade(_alembic_config(), revision or 'base') + + +def stamp(revision): + """Stamp database with provided revision. + Dont run any migrations. + + :param revision: Should match one from repository or head - to stamp + database with most recent revision + :type revision: string + """ + return alembic.command.stamp(_alembic_config(), revision or 'head') + + +def revision(message=None, autogenerate=False): + """Create template for migration. + + :param message: Text that will be used for migration title + :type message: string + :param autogenerate: If True - generates diff based on current database + state + :type autogenerate: bool + """ + return alembic.command.revision(_alembic_config(), message, autogenerate) diff --git a/manila/db/migrations/alembic/script.py.mako b/manila/db/migrations/alembic/script.py.mako new file mode 100644 index 0000000000..95702017ea --- /dev/null +++ b/manila/db/migrations/alembic/script.py.mako @@ -0,0 +1,22 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision} +Create Date: ${create_date} + +""" + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/manila/db/sqlalchemy/migrate_repo/versions/001_manila_init.py b/manila/db/migrations/alembic/versions/162a3e673105_manila_init.py similarity index 88% rename from manila/db/sqlalchemy/migrate_repo/versions/001_manila_init.py rename to manila/db/migrations/alembic/versions/162a3e673105_manila_init.py index cff0adbbf7..6058f20819 100644 --- a/manila/db/sqlalchemy/migrate_repo/versions/001_manila_init.py +++ b/manila/db/migrations/alembic/versions/162a3e673105_manila_init.py @@ -2,52 +2,45 @@ # Copyright 2012 OpenStack LLC. # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. -from oslo.config import cfg +"""manila_init + +Revision ID: 162a3e673105 +Revises: None +Create Date: 2014-07-23 17:51:57.077203 + +""" + +# revision identifiers, used by Alembic. +revision = '162a3e673105' +down_revision = None + +from alembic import op from sqlalchemy import Boolean, Column, DateTime, ForeignKey from sqlalchemy import Integer, MetaData, String, Table, UniqueConstraint - from manila.openstack.common import log as logging -CONF = cfg.CONF LOG = logging.getLogger(__name__) -def upgrade(migrate_engine): +def upgrade(): + migrate_engine = op.get_bind().engine meta = MetaData() meta.bind = migrate_engine - migrations = Table( - 'migrations', meta, - Column('created_at', DateTime), - Column('updated_at', DateTime), - Column('deleted_at', DateTime), - Column('deleted', Integer, default=0), - Column('id', Integer, primary_key=True, nullable=False), - Column('source_compute', String(length=255)), - Column('dest_compute', String(length=255)), - Column('dest_host', String(length=255)), - Column('status', String(length=255)), - Column('instance_uuid', String(length=255)), - Column('old_instance_type_id', Integer), - Column('new_instance_type_id', Integer), - mysql_engine='InnoDB', - mysql_charset='utf8' - ) - services = Table( 'services', meta, Column('created_at', DateTime), @@ -395,7 +388,7 @@ def upgrade(migrate_engine): # create all tables # Take care on create order for those with FK dependencies - tables = [migrations, quotas, services, quota_classes, quota_usages, + tables = [quotas, services, quota_classes, quota_usages, reservations, project_user_quotas, security_services, share_networks, ss_nw_association, share_servers, network_allocations, shares, access_map, @@ -403,19 +396,20 @@ def upgrade(migrate_engine): share_metadata, volume_types, volume_type_extra_specs] for table in tables: - try: - table.create() - except Exception: - LOG.info(repr(table)) - LOG.exception(_('Exception while creating table.')) - raise + if not table.exists(): + try: + table.create() + except Exception: + LOG.info(repr(table)) + LOG.exception(_('Exception while creating table.')) + raise if migrate_engine.name == "mysql": - tables = ["migrate_version", "migrations", "quotas", "services", - "quota_classes", "quota_usages", "reservations", - "project_user_quotas", "share_access_map", "share_snapshots", - "share_metadata", "security_services", "share_networks", - "network_allocations", "shares", "share_servers", + tables = ["quotas", "services", "quota_classes", "quota_usages", + "reservations", "project_user_quotas", "share_access_map", + "share_snapshots", "share_metadata", "security_services", + "share_networks", "network_allocations", "shares", + "share_servers", "share_network_security_service_association", "volume_types", "volume_type_extra_specs", "share_server_backend_details"] @@ -430,6 +424,6 @@ def upgrade(migrate_engine): migrate_engine.execute("ALTER TABLE %s Engine=InnoDB" % table) -def downgrade(migrate_engine): +def downgrade(): raise NotImplementedError('Downgrade from initial Manila install is not' ' supported.') diff --git a/manila/db/sqlalchemy/migrate_repo/README b/manila/db/sqlalchemy/migrate_repo/README deleted file mode 100644 index 6218f8cac4..0000000000 --- a/manila/db/sqlalchemy/migrate_repo/README +++ /dev/null @@ -1,4 +0,0 @@ -This is a database migration repository. - -More information at -http://code.google.com/p/sqlalchemy-migrate/ diff --git a/manila/db/sqlalchemy/migrate_repo/manage.py b/manila/db/sqlalchemy/migrate_repo/manage.py deleted file mode 100644 index 09e340f44f..0000000000 --- a/manila/db/sqlalchemy/migrate_repo/manage.py +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env python -from migrate.versioning.shell import main -if __name__ == '__main__': - main(debug='False', repository='.') diff --git a/manila/db/sqlalchemy/migrate_repo/migrate.cfg b/manila/db/sqlalchemy/migrate_repo/migrate.cfg deleted file mode 100644 index 14b4e4f130..0000000000 --- a/manila/db/sqlalchemy/migrate_repo/migrate.cfg +++ /dev/null @@ -1,20 +0,0 @@ -[db_settings] -# Used to identify which repository this database is versioned under. -# You can use the name of your project. -repository_id=manila - -# The name of the database table used to track the schema version. -# This name shouldn't already be used by your project. -# If this is changed once a database is under version control, you'll need to -# change the table name in each database too. -version_table=migrate_version - -# When committing a change script, Migrate will attempt to generate the -# sql for all supported databases; normally, if one of them fails - probably -# because you don't have that database installed - it is ignored and the -# commit continues, perhaps ending successfully. -# Databases in this list MUST compile successfully during a commit, or the -# entire commit will fail. List the databases your application will actually -# be using to ensure your updates to that database work properly. -# This must be a list; example: ['postgres','sqlite'] -required_dbs=[] diff --git a/manila/db/sqlalchemy/models.py b/manila/db/sqlalchemy/models.py index b03c5c746b..8e778d7f0d 100644 --- a/manila/db/sqlalchemy/models.py +++ b/manila/db/sqlalchemy/models.py @@ -170,24 +170,6 @@ class Reservation(BASE, ManilaBase): # 'QuotaUsage.deleted == 0)') -class Migration(BASE, ManilaBase): - """Represents a running host-to-host migration.""" - __tablename__ = 'migrations' - id = Column(Integer, primary_key=True, nullable=False) - # NOTE(tr3buchet): the ____compute variables are instance['host'] - source_compute = Column(String(255)) - dest_compute = Column(String(255)) - # NOTE(tr3buchet): dest_host, btw, is an ip address - dest_host = Column(String(255)) - old_instance_type_id = Column(Integer()) - new_instance_type_id = Column(Integer()) - instance_uuid = Column(String(255), - ForeignKey('instances.uuid'), - nullable=True) - # TODO(_cerberus_): enum - status = Column(String(255)) - - class Share(BASE, ManilaBase): """Represents an NFS and CIFS shares.""" __tablename__ = 'shares' @@ -385,7 +367,8 @@ class ShareServer(BASE, ManilaBase): nullable=True) host = Column(String(255), nullable=False) status = Column(Enum(constants.STATUS_INACTIVE, constants.STATUS_ACTIVE, - constants.STATUS_ERROR), + constants.STATUS_ERROR, constants.STATUS_DELETING, + constants.STATUS_CREATING), default=constants.STATUS_INACTIVE) network_allocations = relationship( "NetworkAllocation", @@ -446,8 +429,7 @@ def register_models(): connection is lost and needs to be reestablished. """ from sqlalchemy import create_engine - models = (Migration, - Service, + models = (Service, Share, ShareAccessMapping, ShareSnapshot diff --git a/manila/test.py b/manila/test.py index 06fe189139..6470c6bf82 100644 --- a/manila/test.py +++ b/manila/test.py @@ -34,6 +34,7 @@ import testtools from manila.db import migration from manila.db.sqlalchemy import api as db_api +from manila.db.sqlalchemy import models as db_models from manila.openstack.common import log as logging from manila.openstack.common import timeutils from manila import rpc @@ -68,13 +69,12 @@ class Database(fixtures.Fixture): self.engine.dispose() conn = self.engine.connect() if sql_connection == "sqlite://": - if db_migrate.db_version() > db_migrate.INIT_VERSION: - return + self.setup_sqlite(db_migrate) else: testdb = os.path.join(CONF.state_path, sqlite_db) + db_migrate.upgrade('head') if os.path.exists(testdb): return - db_migrate.db_sync() if sql_connection == "sqlite://": conn = self.engine.connect() self._DB = "".join(line for line in conn.connection.iterdump()) @@ -95,6 +95,12 @@ class Database(fixtures.Fixture): os.path.join(CONF.state_path, self.sqlite_db), ) + def setup_sqlite(self, db_migrate): + if db_migrate.version(): + return + db_models.BASE.metadata.create_all(self.engine) + db_migrate.stamp('head') + class StubOutForTesting(object): def __init__(self, parent): diff --git a/manila/tests/db/test_alembic_commands.py b/manila/tests/db/test_alembic_commands.py new file mode 100644 index 0000000000..c2d36b0300 --- /dev/null +++ b/manila/tests/db/test_alembic_commands.py @@ -0,0 +1,75 @@ +# Copyright 2014 Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import alembic +import mock + +from manila.db import migration +from manila import test + + +class AlembicCommandTestCase(test.TestCase): + def setUp(self): + super(AlembicCommandTestCase, self).setUp() + self.config_patcher = mock.patch( + 'manila.db.migrations.alembic.migration._alembic_config') + self.config = self.config_patcher.start() + self.config.return_value = 'fake_config' + self.addCleanup(self.config_patcher.stop) + + @mock.patch('alembic.command.upgrade') + def test_upgrade(self, upgrade): + migration.upgrade('version_1') + upgrade.assert_called_once_with('fake_config', 'version_1') + + @mock.patch('alembic.command.upgrade') + def test_upgrade_none_version(self, upgrade): + migration.upgrade(None) + upgrade.assert_called_once_with('fake_config', 'head') + + @mock.patch('alembic.command.downgrade') + def test_downgrade(self, downgrade): + migration.downgrade('version_1') + downgrade.assert_called_once_with('fake_config', 'version_1') + + @mock.patch('alembic.command.downgrade') + def test_downgrade_none_verison(self, downgrade): + migration.downgrade(None) + downgrade.assert_called_once_with('fake_config', 'base') + + @mock.patch('alembic.command.stamp') + def test_stamp(self, stamp): + migration.stamp('version_1') + stamp.assert_called_once_with('fake_config', 'version_1') + + @mock.patch('alembic.command.stamp') + def test_stamp_none_version(self, stamp): + migration.stamp(None) + stamp.assert_called_once_with('fake_config', 'head') + + @mock.patch('alembic.command.revision') + def test_revision(self, revision): + migration.revision('test_message', 'autogenerate_value') + revision.assert_called_once_with('fake_config', 'test_message', + 'autogenerate_value') + + @mock.patch.object(alembic.migration.MigrationContext, 'configure', + mock.Mock()) + def test_version(self): + context = mock.Mock() + context.get_current_revision = mock.Mock() + alembic.migration.MigrationContext.configure.return_value = context + migration.version() + context.get_current_revision.assert_called_once_with() diff --git a/manila/tests/db/test_share_servers.py b/manila/tests/db/test_share_servers.py index f796644be0..cb959aec02 100644 --- a/manila/tests/db/test_share_servers.py +++ b/manila/tests/db/test_share_servers.py @@ -86,7 +86,7 @@ class ShareServerTableTestCase(test.TestCase): update = { 'share_network_id': 'update_net', 'host': 'update_host', - 'status': 'updated_status' + 'status': 'ACTIVE' } server = self._create_share_server() updated_server = db.share_server_update(self.ctxt, server['id'], diff --git a/manila/tests/network/test_share_network_db.py b/manila/tests/network/test_share_network_db.py index 009e222110..91f03b8691 100644 --- a/manila/tests/network/test_share_network_db.py +++ b/manila/tests/network/test_share_network_db.py @@ -42,6 +42,7 @@ class ShareNetworkDBTest(test.TestCase): 'neutron_net_id': 'fake net id', 'neutron_subnet_id': 'fake subnet id', 'project_id': self.fake_context.project_id, + 'user_id': 'fake_user_id', 'network_type': 'vlan', 'segmentation_id': 1000, 'cidr': '10.0.0.0/24', diff --git a/manila/tests/test_migrations.py b/manila/tests/test_migrations.py index 597a60fcab..346c4f42d0 100644 --- a/manila/tests/test_migrations.py +++ b/manila/tests/test_migrations.py @@ -19,15 +19,16 @@ Tests for database migrations. """ -import os - -from migrate.versioning import api as migration_api -from migrate.versioning import repository +from alembic import script +import mock from oslo.db.sqlalchemy import test_base from oslo.db.sqlalchemy import test_migrations from sqlalchemy.sql import text -import manila.db.sqlalchemy.migrate_repo +from manila.db.migrations.alembic import migration +from manila.openstack.common import log as logging + +LOG = logging.getLogger('manila.tests.test_migrations') class ManilaMigrationsCheckers(test_migrations.WalkVersionsMixin): @@ -38,29 +39,116 @@ class ManilaMigrationsCheckers(test_migrations.WalkVersionsMixin): @property def INIT_VERSION(self): - return 000 + pass @property def REPOSITORY(self): - migrate_file = manila.db.sqlalchemy.migrate_repo.__file__ - return repository.Repository( - os.path.abspath(os.path.dirname(migrate_file))) + pass @property def migration_api(self): - return migration_api + return migration @property def migrate_engine(self): return self.engine + def _walk_versions(self, snake_walk=False, downgrade=True): + # Determine latest version script from the repo, then + # upgrade from 1 through to the latest, with no data + # in the databases. This just checks that the schema itself + # upgrades successfully. + + # Place the database under version control + alembic_cfg = migration._alembic_config() + script_directory = script.ScriptDirectory.from_config(alembic_cfg) + + self.assertIsNone(self.migration_api.version()) + + versions = [ver for ver in script_directory.walk_revisions()] + + LOG.debug('latest version is %s', versions[0].revision) + + prev_version = 'base' + for version in reversed(versions): + self._migrate_up(version.revision, with_data=True) + if snake_walk and prev_version: + downgraded = self._migrate_down(prev_version, with_data=True) + if downgraded: + self._migrate_up(version.revision) + prev_version = version.revision + + prev_version = 'base' + if downgrade: + for version in versions: + self._migrate_down(version.revision) + downgraded = self._migrate_down(prev_version) + if snake_walk and downgraded: + self._migrate_up(version.revision) + self._migrate_down(prev_version) + prev_version = version.revision + + def _migrate_down(self, version, with_data=False): + try: + self.migration_api.downgrade(version) + except NotImplementedError: + # NOTE(sirp): some migrations, namely release-level + # migrations, don't support a downgrade. + return False + + self.assertEqual(version, self.migration_api.version()) + + # NOTE(sirp): `version` is what we're downgrading to (i.e. the 'target' + # version). So if we have any downgrade checks, they need to be run for + # the previous (higher numbered) migration. + if with_data: + post_downgrade = getattr( + self, "_post_downgrade_%s" % (version), None) + if post_downgrade: + post_downgrade(self.engine) + + return True + + def _migrate_up(self, version, with_data=False): + """migrate up to a new version of the db. + + We allow for data insertion and post checks at every + migration version with special _pre_upgrade_### and + _check_### functions in the main test. + """ + # NOTE(sdague): try block is here because it's impossible to debug + # where a failed data migration happens otherwise + try: + if with_data: + data = None + pre_upgrade = getattr( + self, "_pre_upgrade_%s" % version, None) + if pre_upgrade: + data = pre_upgrade(self.engine) + + self.migration_api.upgrade(version) + self.assertEqual(version, self.migration_api.version()) + if with_data: + check = getattr(self, "_check_%s" % version, None) + if check: + check(self.engine, data) + except Exception as e: + LOG.error(_("Failed to migrate to version %(version)s on engine " + "%(engine)s. Exception while running the migration: " + "%(exception)s") % {'version': version, + 'engine': self.engine, + 'exception': e}) + raise + def test_walk_versions(self): """ Walks all version scripts for each tested database, ensuring that there are no errors in the version scripts for each engine """ - self._walk_versions(snake_walk=self.snake_walk, - downgrade=self.downgrade) + with mock.patch('manila.db.sqlalchemy.api.get_engine', + return_value=self.engine): + self._walk_versions(snake_walk=self.snake_walk, + downgrade=self.downgrade) class TestManilaMigrationsMySQL(ManilaMigrationsCheckers, @@ -69,7 +157,9 @@ class TestManilaMigrationsMySQL(ManilaMigrationsCheckers, def test_mysql_innodb(self): """Test that table creation on mysql only builds InnoDB tables.""" - self._walk_versions(snake_walk=False, downgrade=False) + with mock.patch('manila.db.sqlalchemy.api.get_engine', + return_value=self.engine): + self._walk_versions(snake_walk=False, downgrade=False) # sanity check sanity_check = """SELECT count(*) @@ -86,7 +176,7 @@ class TestManilaMigrationsMySQL(ManilaMigrationsCheckers, FROM information_schema.TABLES WHERE table_schema = :database AND engine != 'InnoDB' - AND table_name != 'migrate_version';""" + AND table_name != 'alembic_version';""" count = self.engine.execute( text(noninnodb_query), diff --git a/manila/tests/test_share.py b/manila/tests/test_share.py index f956529d76..c03b574457 100644 --- a/manila/tests/test_share.py +++ b/manila/tests/test_share.py @@ -137,7 +137,7 @@ class ShareTestCase(test.TestCase): return db.share_access_create(context.get_admin_context(), access) @staticmethod - def _create_share_server(state='new', share_network_id=None, host=None): + def _create_share_server(state='ACTIVE', share_network_id=None, host=None): """Create a share server object.""" srv = {} srv['host'] = host @@ -207,7 +207,9 @@ class ShareTestCase(test.TestCase): def test_create_share_from_snapshot_with_server(self): """Test share can be created from snapshot if server exists.""" - server = self._create_share_server(share_network_id='net-id',) + network = self._create_share_network() + server = self._create_share_server(share_network_id=network['id'], + host='fake_host') parent_share = self._create_share(share_network_id='net-id', share_server_id=server['id']) share = self._create_share() @@ -344,7 +346,8 @@ class ShareTestCase(test.TestCase): def fake_setup_server(context, share_network, *args, **kwargs): return self._create_share_server( - share_network_id=share_network['id']) + share_network_id=share_network['id'], + host='fake_host') self.share_manager.driver.create_share = mock.Mock( return_value='fake_location') @@ -423,8 +426,7 @@ class ShareTestCase(test.TestCase): share_net = self._create_share_network() share = self._create_share(share_network_id=share_net['id']) share_srv = self._create_share_server( - share_network_id=share_net['id'], host=self.share_manager.host, - state='ACTIVE') + share_network_id=share_net['id'], host=self.share_manager.host) share_id = share['id'] @@ -513,8 +515,7 @@ class ShareTestCase(test.TestCase): sec_service = self._create_security_service(share_net['id']) share_srv = self._create_share_server( share_network_id=share_net['id'], - host=self.share_manager.host, - state='ACTIVE' + host=self.share_manager.host ) share = self._create_share(share_network_id=share_net['id'], share_server_id=share_srv['id']) @@ -541,8 +542,7 @@ class ShareTestCase(test.TestCase): share_net = self._create_share_network() share_srv = self._create_share_server( share_network_id=share_net['id'], - host=self.share_manager.host, - state='ACTIVE' + host=self.share_manager.host ) share = self._create_share(share_network_id=share_net['id'], share_server_id=share_srv['id']) @@ -560,8 +560,7 @@ class ShareTestCase(test.TestCase): share_net = self._create_share_network() share_srv = self._create_share_server( share_network_id=share_net['id'], - host=self.share_manager.host, - state='ACTIVE' + host=self.share_manager.host ) share = self._create_share(share_network_id=share_net['id'], share_server_id=share_srv['id']) @@ -576,8 +575,7 @@ class ShareTestCase(test.TestCase): share_net = self._create_share_network() share_srv = self._create_share_server( share_network_id=share_net['id'], - host=self.share_manager.host, - state='ACTIVE' + host=self.share_manager.host ) share = self._create_share(share_network_id=share_net['id'], share_server_id=share_srv['id']) diff --git a/requirements.txt b/requirements.txt index 3cfd7d9711..72be9a9c1e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ pbr>=0.6,!=0.7,<1.0 +alembic>=0.6.4 anyjson>=0.3.3 argparse Babel>=1.3 @@ -23,7 +24,6 @@ python-keystoneclient>=0.10.0 Routes>=1.12.3,!=2.0 six>=1.7.0 SQLAlchemy>=0.8.4,<=0.8.99,>=0.9.7,<=0.9.99 -sqlalchemy-migrate>=0.9.1 stevedore>=0.14 python-cinderclient>=1.0.7 python-novaclient>=2.17.0