Use Alembic instead of Sqlalchemy-migrate in Manila

Alembic offers the following functionality:
- Can emit ALTER statements to a database in order to change
  the structure of tables and other constructs
- Provides a system whereby "migration scripts" may be constructed;
  each script indicates a particular series of steps that can "upgrade"
  a target database to a new version, and optionally a series of steps
  that can "downgrade" similarly, doing the same steps in reverse.
- Allows the scripts to execute in some sequential manner.

1. Add Alembic migrations support.
2. Move 001_manila_init.py migration to
manila/db/sqlalchemy/alembic/versions/162a3e673105_manila_init.py.
3. Remove manila/db/sqlalchemy/migrate_repo directory.
4. Fix unit tests.
5. Add ability to runtime updrade/downgrade db.

Implements bp alembic-instead-of-sqlalchemy-migrate

Change-Id: Iadc0d9596e826323ba19bd25be741c401b90b688

bin/manila-manage

@@ -215,11 +215,29 @@ class DbCommands(object):
           help='Database version')
     def sync(self, version=None):
         """Sync the database up to the most recent version."""
-        return migration.db_sync(version)
+        return migration.upgrade(version)
 
     def version(self):
         """Print the current database version."""
-        print(migration.db_version())
+        print(migration.version())
+
+    @args('version', nargs='?', default=None,
+          help='Version to downgrade')
+    def downgrade(self, version=None):
+        """Downgrade database to the given version."""
+        return migration.downgrade(version)
+
+    @args('--message', help='Revision message')
+    @args('--authogenerate', help='Autogenerate migration from schema')
+    def revision(self, message, autogenerate):
+        """Generate new migration."""
+        return migration.revision(message, autogenerate)
+
+    @args('version', nargs='?', default=None,
+          help='Version to stamp version table with')
+    def stamp(self, version=None):
+        """Stamp the revision table with the given version."""
+        return migration.stamp(version)
 
 
 class VersionCommands(object):

doc/source/man/manila-manage.rst

@@ -51,6 +51,18 @@ Manila Db
 
     Sync the database up to the most recent version. This is the standard way to create the db as well.
 
+``manila-manage db downgrade <version>``
+
+    Downgrade database to given version.
+
+``manila-manage db stamp <version>``
+
+    Stamp database with given revision.
+
+``manila-manage db revision <message> <authogenerate>``
+
+    Generate new migration.
+
 Manila Logs
 ~~~~~~~~~~~
 

manila/db/api.py

@@ -129,34 +129,6 @@ def service_update(context, service_id, values):
     return IMPL.service_update(context, service_id, values)
 
 
-###################
-def migration_update(context, id, values):
-    """Update a migration instance."""
-    return IMPL.migration_update(context, id, values)
-
-
-def migration_create(context, values):
-    """Create a migration record."""
-    return IMPL.migration_create(context, values)
-
-
-def migration_get(context, migration_id):
-    """Finds a migration by the id."""
-    return IMPL.migration_get(context, migration_id)
-
-
-def migration_get_by_instance_and_status(context, instance_uuid, status):
-    """Finds a migration by the instance uuid its migrating."""
-    return IMPL.migration_get_by_instance_and_status(context,
-                                                     instance_uuid,
-                                                     status)
-
-
-def migration_get_all_unconfirmed(context, confirm_window):
-    """Finds all unconfirmed migrations within the confirmation window."""
-    return IMPL.migration_get_all_unconfirmed(context, confirm_window)
-
-
 ####################
 
 

manila/db/migration.py

@@ -18,27 +18,33 @@
 
 """Database setup and migration commands."""
 
-import os
-
-from manila.db.sqlalchemy import api as db_api
 from manila import utils
 
 
-IMPL = utils.LazyPluggable('db_backend',
+IMPL = utils.LazyPluggable(
-                           sqlalchemy='oslo.db.sqlalchemy.migration')
+    'db_backend', sqlalchemy='manila.db.migrations.alembic.migration')
 
 
-INIT_VERSION = 000
+def upgrade(version):
-MIGRATE_REPO = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+    """Upgrade database to 'version' or the most recent version."""
-                            'sqlalchemy/migrate_repo')
+    return IMPL.upgrade(version)
 
 
-def db_sync(version=None):
+def downgrade(version):
-    """Migrate the database to `version` or the most recent version."""
+    """Downgrade database to 'version' or to initial state."""
-    return IMPL.db_sync(db_api.get_engine(), MIGRATE_REPO, version=version,
+    return IMPL.downgrade(version)
-                        init_version=INIT_VERSION)
 
 
-def db_version():
+def version():
     """Display the current database version."""
-    return IMPL.db_version(db_api.get_engine(), MIGRATE_REPO, INIT_VERSION)
+    return IMPL.version()
+
+
+def stamp(version):
+    """Stamp database with 'version' or the most recent version."""
+    return IMPL.stamp(version)
+
+
+def revision(message, autogenerate):
+    """Generate new migration script."""
+    return IMPL.revision(message, autogenerate)

manila/db/migrations/alembic.ini

@@ -0,0 +1,59 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = %(here)s/alembic
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# max length of characters to apply to the
+# "slug" field
+#truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+#sqlalchemy.url = driver://user:pass@localhost/dbname
+
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

manila/db/migrations/alembic/env.py

@@ -0,0 +1,41 @@
+# Copyright 2014 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from __future__ import with_statement
+
+from alembic import context
+
+from manila.db.sqlalchemy import api as db_api
+from manila.db.sqlalchemy import models as db_models
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+    """
+    engine = db_api.get_engine()
+    connection = engine.connect()
+    target_metadata = db_models.ManilaBase.metadata
+    context.configure(connection=connection,  # pylint: disable=E1101
+                      target_metadata=target_metadata)
+    try:
+        with context.begin_transaction():  # pylint: disable=E1101
+            context.run_migrations()  # pylint: disable=E1101
+    finally:
+        connection.close()
+
+
+run_migrations_online()

manila/db/migrations/alembic/migration.py

@@ -0,0 +1,83 @@
+# Copyright 2014 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+
+import alembic
+from alembic import config as alembic_config
+import alembic.migration as alembic_migration
+from oslo.config import cfg
+
+from manila.db.sqlalchemy import api as db_api
+
+CONF = cfg.CONF
+
+
+def _alembic_config():
+    path = os.path.join(os.path.dirname(__file__), os.pardir, 'alembic.ini')
+    config = alembic_config.Config(path)
+    return config
+
+
+def version():
+    """Current database version.
+
+    :returns: Database version
+    :rtype: string
+    """
+    engine = db_api.get_engine()
+    with engine.connect() as conn:
+        context = alembic_migration.MigrationContext.configure(conn)
+        return context.get_current_revision()
+
+
+def upgrade(revision):
+    """Upgrade database.
+
+    :param version: Desired database version
+    :type version: string
+    """
+    return alembic.command.upgrade(_alembic_config(), revision or 'head')
+
+
+def downgrade(revision):
+    """Downgrade database.
+
+    :param version: Desired database version
+    :type version: string
+    """
+    return alembic.command.downgrade(_alembic_config(), revision or 'base')
+
+
+def stamp(revision):
+    """Stamp database with provided revision.
+    Dont run any migrations.
+
+    :param revision: Should match one from repository or head - to stamp
+    database with most recent revision
+    :type revision: string
+    """
+    return alembic.command.stamp(_alembic_config(), revision or 'head')
+
+
+def revision(message=None, autogenerate=False):
+    """Create template for migration.
+
+    :param message: Text that will be used for migration title
+    :type message: string
+    :param autogenerate: If True - generates diff based on current database
+    state
+    :type autogenerate: bool
+    """
+    return alembic.command.revision(_alembic_config(), message, autogenerate)

manila/db/migrations/alembic/script.py.mako

@@ -0,0 +1,22 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision}
+Create Date: ${create_date}
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}

manila/db/migrations/alembic/versions/162a3e673105_manila_init.py

@@ -2,52 +2,45 @@
 
 # Copyright 2012 OpenStack LLC.
 #
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
+# not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
+# a copy of the License at
 #
-#         http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
 #
-#    Unless required by applicable law or agreed to in writing, software
+# Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
+# License for the specific language governing permissions and limitations
-#    under the License.
+# under the License.
 
-from oslo.config import cfg
+"""manila_init
+
+Revision ID: 162a3e673105
+Revises: None
+Create Date: 2014-07-23 17:51:57.077203
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '162a3e673105'
+down_revision = None
+
+from alembic import op
 from sqlalchemy import Boolean, Column, DateTime, ForeignKey
 from sqlalchemy import Integer, MetaData, String, Table, UniqueConstraint
 
-
 from manila.openstack.common import log as logging
 
-CONF = cfg.CONF
 
 LOG = logging.getLogger(__name__)
 
 
-def upgrade(migrate_engine):
+def upgrade():
+    migrate_engine = op.get_bind().engine
     meta = MetaData()
     meta.bind = migrate_engine
 
-    migrations = Table(
-        'migrations', meta,
-        Column('created_at', DateTime),
-        Column('updated_at', DateTime),
-        Column('deleted_at', DateTime),
-        Column('deleted', Integer, default=0),
-        Column('id', Integer, primary_key=True, nullable=False),
-        Column('source_compute', String(length=255)),
-        Column('dest_compute', String(length=255)),
-        Column('dest_host', String(length=255)),
-        Column('status', String(length=255)),
-        Column('instance_uuid', String(length=255)),
-        Column('old_instance_type_id', Integer),
-        Column('new_instance_type_id', Integer),
-        mysql_engine='InnoDB',
-        mysql_charset='utf8'
-    )
-
     services = Table(
         'services', meta,
         Column('created_at', DateTime),
@@ -395,7 +388,7 @@ def upgrade(migrate_engine):
 
     # create all tables
     # Take care on create order for those with FK dependencies
-    tables = [migrations, quotas, services, quota_classes, quota_usages,
+    tables = [quotas, services, quota_classes, quota_usages,
               reservations, project_user_quotas, security_services,
               share_networks, ss_nw_association,
               share_servers, network_allocations, shares, access_map,
@@ -403,19 +396,20 @@ def upgrade(migrate_engine):
               share_metadata, volume_types, volume_type_extra_specs]
 
     for table in tables:
-        try:
+        if not table.exists():
-            table.create()
+            try:
-        except Exception:
+                table.create()
-            LOG.info(repr(table))
+            except Exception:
-            LOG.exception(_('Exception while creating table.'))
+                LOG.info(repr(table))
-            raise
+                LOG.exception(_('Exception while creating table.'))
+                raise
 
     if migrate_engine.name == "mysql":
-        tables = ["migrate_version", "migrations", "quotas", "services",
+        tables = ["quotas", "services", "quota_classes", "quota_usages",
-                  "quota_classes", "quota_usages", "reservations",
+                  "reservations", "project_user_quotas", "share_access_map",
-                  "project_user_quotas", "share_access_map", "share_snapshots",
+                  "share_snapshots", "share_metadata", "security_services",
-                  "share_metadata", "security_services", "share_networks",
+                  "share_networks", "network_allocations", "shares",
-                  "network_allocations", "shares", "share_servers",
+                  "share_servers",
                   "share_network_security_service_association", "volume_types",
                   "volume_type_extra_specs", "share_server_backend_details"]
 
@@ -430,6 +424,6 @@ def upgrade(migrate_engine):
         migrate_engine.execute("ALTER TABLE %s Engine=InnoDB" % table)
 
 
-def downgrade(migrate_engine):
+def downgrade():
     raise NotImplementedError('Downgrade from initial Manila install is not'
                               ' supported.')

manila/db/sqlalchemy/migrate_repo/README

@@ -1,4 +0,0 @@
-This is a database migration repository.
-
-More information at
-http://code.google.com/p/sqlalchemy-migrate/

manila/db/sqlalchemy/migrate_repo/manage.py

@@ -1,4 +0,0 @@
-#!/usr/bin/env python
-from migrate.versioning.shell import main
-if __name__ == '__main__':
-    main(debug='False', repository='.')

manila/db/sqlalchemy/migrate_repo/migrate.cfg

@@ -1,20 +0,0 @@
-[db_settings]
-# Used to identify which repository this database is versioned under.
-# You can use the name of your project.
-repository_id=manila
-
-# The name of the database table used to track the schema version.
-# This name shouldn't already be used by your project.
-# If this is changed once a database is under version control, you'll need to 
-# change the table name in each database too. 
-version_table=migrate_version
-
-# When committing a change script, Migrate will attempt to generate the 
-# sql for all supported databases; normally, if one of them fails - probably
-# because you don't have that database installed - it is ignored and the 
-# commit continues, perhaps ending successfully. 
-# Databases in this list MUST compile successfully during a commit, or the 
-# entire commit will fail. List the databases your application will actually 
-# be using to ensure your updates to that database work properly.
-# This must be a list; example: ['postgres','sqlite']
-required_dbs=[]

manila/db/sqlalchemy/models.py

@@ -170,24 +170,6 @@ class Reservation(BASE, ManilaBase):
 #                         'QuotaUsage.deleted == 0)')
 
 
-class Migration(BASE, ManilaBase):
-    """Represents a running host-to-host migration."""
-    __tablename__ = 'migrations'
-    id = Column(Integer, primary_key=True, nullable=False)
-    # NOTE(tr3buchet): the ____compute variables are instance['host']
-    source_compute = Column(String(255))
-    dest_compute = Column(String(255))
-    # NOTE(tr3buchet): dest_host, btw, is an ip address
-    dest_host = Column(String(255))
-    old_instance_type_id = Column(Integer())
-    new_instance_type_id = Column(Integer())
-    instance_uuid = Column(String(255),
-                           ForeignKey('instances.uuid'),
-                           nullable=True)
-    # TODO(_cerberus_): enum
-    status = Column(String(255))
-
-
 class Share(BASE, ManilaBase):
     """Represents an NFS and CIFS shares."""
     __tablename__ = 'shares'
@@ -385,7 +367,8 @@ class ShareServer(BASE, ManilaBase):
                               nullable=True)
     host = Column(String(255), nullable=False)
     status = Column(Enum(constants.STATUS_INACTIVE, constants.STATUS_ACTIVE,
-                         constants.STATUS_ERROR),
+                         constants.STATUS_ERROR, constants.STATUS_DELETING,
+                         constants.STATUS_CREATING),
                     default=constants.STATUS_INACTIVE)
     network_allocations = relationship(
         "NetworkAllocation",
@@ -446,8 +429,7 @@ def register_models():
     connection is lost and needs to be reestablished.
     """
     from sqlalchemy import create_engine
-    models = (Migration,
+    models = (Service,
-              Service,
               Share,
               ShareAccessMapping,
               ShareSnapshot

manila/test.py

@@ -34,6 +34,7 @@ import testtools
 
 from manila.db import migration
 from manila.db.sqlalchemy import api as db_api
+from manila.db.sqlalchemy import models as db_models
 from manila.openstack.common import log as logging
 from manila.openstack.common import timeutils
 from manila import rpc
@@ -68,13 +69,12 @@ class Database(fixtures.Fixture):
         self.engine.dispose()
         conn = self.engine.connect()
         if sql_connection == "sqlite://":
-            if db_migrate.db_version() > db_migrate.INIT_VERSION:
+            self.setup_sqlite(db_migrate)
-                return
         else:
             testdb = os.path.join(CONF.state_path, sqlite_db)
+            db_migrate.upgrade('head')
             if os.path.exists(testdb):
                 return
-        db_migrate.db_sync()
         if sql_connection == "sqlite://":
             conn = self.engine.connect()
             self._DB = "".join(line for line in conn.connection.iterdump())
@@ -95,6 +95,12 @@ class Database(fixtures.Fixture):
                 os.path.join(CONF.state_path, self.sqlite_db),
             )
 
+    def setup_sqlite(self, db_migrate):
+        if db_migrate.version():
+            return
+        db_models.BASE.metadata.create_all(self.engine)
+        db_migrate.stamp('head')
+
 
 class StubOutForTesting(object):
     def __init__(self, parent):

manila/tests/db/test_alembic_commands.py

@@ -0,0 +1,75 @@
+# Copyright 2014 Mirantis Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import alembic
+import mock
+
+from manila.db import migration
+from manila import test
+
+
+class AlembicCommandTestCase(test.TestCase):
+    def setUp(self):
+        super(AlembicCommandTestCase, self).setUp()
+        self.config_patcher = mock.patch(
+            'manila.db.migrations.alembic.migration._alembic_config')
+        self.config = self.config_patcher.start()
+        self.config.return_value = 'fake_config'
+        self.addCleanup(self.config_patcher.stop)
+
+    @mock.patch('alembic.command.upgrade')
+    def test_upgrade(self, upgrade):
+        migration.upgrade('version_1')
+        upgrade.assert_called_once_with('fake_config', 'version_1')
+
+    @mock.patch('alembic.command.upgrade')
+    def test_upgrade_none_version(self, upgrade):
+        migration.upgrade(None)
+        upgrade.assert_called_once_with('fake_config', 'head')
+
+    @mock.patch('alembic.command.downgrade')
+    def test_downgrade(self, downgrade):
+        migration.downgrade('version_1')
+        downgrade.assert_called_once_with('fake_config', 'version_1')
+
+    @mock.patch('alembic.command.downgrade')
+    def test_downgrade_none_verison(self, downgrade):
+        migration.downgrade(None)
+        downgrade.assert_called_once_with('fake_config', 'base')
+
+    @mock.patch('alembic.command.stamp')
+    def test_stamp(self, stamp):
+        migration.stamp('version_1')
+        stamp.assert_called_once_with('fake_config', 'version_1')
+
+    @mock.patch('alembic.command.stamp')
+    def test_stamp_none_version(self, stamp):
+        migration.stamp(None)
+        stamp.assert_called_once_with('fake_config', 'head')
+
+    @mock.patch('alembic.command.revision')
+    def test_revision(self, revision):
+        migration.revision('test_message', 'autogenerate_value')
+        revision.assert_called_once_with('fake_config', 'test_message',
+                                         'autogenerate_value')
+
+    @mock.patch.object(alembic.migration.MigrationContext, 'configure',
+                       mock.Mock())
+    def test_version(self):
+        context = mock.Mock()
+        context.get_current_revision = mock.Mock()
+        alembic.migration.MigrationContext.configure.return_value = context
+        migration.version()
+        context.get_current_revision.assert_called_once_with()

manila/tests/db/test_share_servers.py

@@ -86,7 +86,7 @@ class ShareServerTableTestCase(test.TestCase):
         update = {
             'share_network_id': 'update_net',
             'host': 'update_host',
-            'status': 'updated_status'
+            'status': 'ACTIVE'
         }
         server = self._create_share_server()
         updated_server = db.share_server_update(self.ctxt, server['id'],

manila/tests/network/test_share_network_db.py

@@ -42,6 +42,7 @@ class ShareNetworkDBTest(test.TestCase):
                               'neutron_net_id': 'fake net id',
                               'neutron_subnet_id': 'fake subnet id',
                               'project_id': self.fake_context.project_id,
+                              'user_id': 'fake_user_id',
                               'network_type': 'vlan',
                               'segmentation_id': 1000,
                               'cidr': '10.0.0.0/24',

manila/tests/test_migrations.py

@@ -19,15 +19,16 @@
 Tests for database migrations.
 """
 
-import os
+from alembic import script
-
+import mock
-from migrate.versioning import api as migration_api
-from migrate.versioning import repository
 from oslo.db.sqlalchemy import test_base
 from oslo.db.sqlalchemy import test_migrations
 from sqlalchemy.sql import text
 
-import manila.db.sqlalchemy.migrate_repo
+from manila.db.migrations.alembic import migration
+from manila.openstack.common import log as logging
+
+LOG = logging.getLogger('manila.tests.test_migrations')
 
 
 class ManilaMigrationsCheckers(test_migrations.WalkVersionsMixin):
@@ -38,29 +39,116 @@ class ManilaMigrationsCheckers(test_migrations.WalkVersionsMixin):
 
     @property
     def INIT_VERSION(self):
-        return 000
+        pass
 
     @property
     def REPOSITORY(self):
-        migrate_file = manila.db.sqlalchemy.migrate_repo.__file__
+        pass
-        return repository.Repository(
-            os.path.abspath(os.path.dirname(migrate_file)))
 
     @property
     def migration_api(self):
-        return migration_api
+        return migration
 
     @property
     def migrate_engine(self):
         return self.engine
 
+    def _walk_versions(self, snake_walk=False, downgrade=True):
+        # Determine latest version script from the repo, then
+        # upgrade from 1 through to the latest, with no data
+        # in the databases. This just checks that the schema itself
+        # upgrades successfully.
+
+        # Place the database under version control
+        alembic_cfg = migration._alembic_config()
+        script_directory = script.ScriptDirectory.from_config(alembic_cfg)
+
+        self.assertIsNone(self.migration_api.version())
+
+        versions = [ver for ver in script_directory.walk_revisions()]
+
+        LOG.debug('latest version is %s', versions[0].revision)
+
+        prev_version = 'base'
+        for version in reversed(versions):
+            self._migrate_up(version.revision, with_data=True)
+            if snake_walk and prev_version:
+                downgraded = self._migrate_down(prev_version, with_data=True)
+                if downgraded:
+                    self._migrate_up(version.revision)
+            prev_version = version.revision
+
+        prev_version = 'base'
+        if downgrade:
+            for version in versions:
+                self._migrate_down(version.revision)
+                downgraded = self._migrate_down(prev_version)
+                if snake_walk and downgraded:
+                    self._migrate_up(version.revision)
+                    self._migrate_down(prev_version)
+                prev_version = version.revision
+
+    def _migrate_down(self, version, with_data=False):
+        try:
+            self.migration_api.downgrade(version)
+        except NotImplementedError:
+            # NOTE(sirp): some migrations, namely release-level
+            # migrations, don't support a downgrade.
+            return False
+
+        self.assertEqual(version, self.migration_api.version())
+
+        # NOTE(sirp): `version` is what we're downgrading to (i.e. the 'target'
+        # version). So if we have any downgrade checks, they need to be run for
+        # the previous (higher numbered) migration.
+        if with_data:
+            post_downgrade = getattr(
+                self, "_post_downgrade_%s" % (version), None)
+            if post_downgrade:
+                post_downgrade(self.engine)
+
+        return True
+
+    def _migrate_up(self, version, with_data=False):
+        """migrate up to a new version of the db.
+
+        We allow for data insertion and post checks at every
+        migration version with special _pre_upgrade_### and
+        _check_### functions in the main test.
+        """
+        # NOTE(sdague): try block is here because it's impossible to debug
+        # where a failed data migration happens otherwise
+        try:
+            if with_data:
+                data = None
+                pre_upgrade = getattr(
+                    self, "_pre_upgrade_%s" % version, None)
+                if pre_upgrade:
+                    data = pre_upgrade(self.engine)
+
+            self.migration_api.upgrade(version)
+            self.assertEqual(version, self.migration_api.version())
+            if with_data:
+                check = getattr(self, "_check_%s" % version, None)
+                if check:
+                    check(self.engine, data)
+        except Exception as e:
+            LOG.error(_("Failed to migrate to version %(version)s on engine "
+                        "%(engine)s. Exception while running the migration: "
+                        "%(exception)s") % {'version': version,
+                                            'engine': self.engine,
+                                            'exception': e})
+            raise
+
     def test_walk_versions(self):
         """
         Walks all version scripts for each tested database, ensuring
         that there are no errors in the version scripts for each engine
         """
-        self._walk_versions(snake_walk=self.snake_walk,
+        with mock.patch('manila.db.sqlalchemy.api.get_engine',
-                            downgrade=self.downgrade)
+                        return_value=self.engine):
+            self._walk_versions(snake_walk=self.snake_walk,
+                                downgrade=self.downgrade)
 
 
 class TestManilaMigrationsMySQL(ManilaMigrationsCheckers,
@@ -69,7 +157,9 @@ class TestManilaMigrationsMySQL(ManilaMigrationsCheckers,
 
     def test_mysql_innodb(self):
         """Test that table creation on mysql only builds InnoDB tables."""
-        self._walk_versions(snake_walk=False, downgrade=False)
+        with mock.patch('manila.db.sqlalchemy.api.get_engine',
+                        return_value=self.engine):
+            self._walk_versions(snake_walk=False, downgrade=False)
 
         # sanity check
         sanity_check = """SELECT count(*)
@@ -86,7 +176,7 @@ class TestManilaMigrationsMySQL(ManilaMigrationsCheckers,
             FROM information_schema.TABLES
             WHERE table_schema = :database
                 AND engine != 'InnoDB'
-                AND table_name != 'migrate_version';"""
+                AND table_name != 'alembic_version';"""
 
         count = self.engine.execute(
             text(noninnodb_query),

manila/tests/test_share.py

@@ -137,7 +137,7 @@ class ShareTestCase(test.TestCase):
         return db.share_access_create(context.get_admin_context(), access)
 
     @staticmethod
-    def _create_share_server(state='new', share_network_id=None, host=None):
+    def _create_share_server(state='ACTIVE', share_network_id=None, host=None):
         """Create a share server object."""
         srv = {}
         srv['host'] = host
@@ -207,7 +207,9 @@ class ShareTestCase(test.TestCase):
 
     def test_create_share_from_snapshot_with_server(self):
         """Test share can be created from snapshot if server exists."""
-        server = self._create_share_server(share_network_id='net-id',)
+        network = self._create_share_network()
+        server = self._create_share_server(share_network_id=network['id'],
+                                           host='fake_host')
         parent_share = self._create_share(share_network_id='net-id',
                                           share_server_id=server['id'])
         share = self._create_share()
@@ -344,7 +346,8 @@ class ShareTestCase(test.TestCase):
 
         def fake_setup_server(context, share_network, *args, **kwargs):
             return self._create_share_server(
-                share_network_id=share_network['id'])
+                share_network_id=share_network['id'],
+                host='fake_host')
 
         self.share_manager.driver.create_share = mock.Mock(
             return_value='fake_location')
@@ -423,8 +426,7 @@ class ShareTestCase(test.TestCase):
         share_net = self._create_share_network()
         share = self._create_share(share_network_id=share_net['id'])
         share_srv = self._create_share_server(
-            share_network_id=share_net['id'], host=self.share_manager.host,
+            share_network_id=share_net['id'], host=self.share_manager.host)
-            state='ACTIVE')
 
         share_id = share['id']
 
@@ -513,8 +515,7 @@ class ShareTestCase(test.TestCase):
         sec_service = self._create_security_service(share_net['id'])
         share_srv = self._create_share_server(
             share_network_id=share_net['id'],
-            host=self.share_manager.host,
+            host=self.share_manager.host
-            state='ACTIVE'
         )
         share = self._create_share(share_network_id=share_net['id'],
                                    share_server_id=share_srv['id'])
@@ -541,8 +542,7 @@ class ShareTestCase(test.TestCase):
         share_net = self._create_share_network()
         share_srv = self._create_share_server(
             share_network_id=share_net['id'],
-            host=self.share_manager.host,
+            host=self.share_manager.host
-            state='ACTIVE'
         )
         share = self._create_share(share_network_id=share_net['id'],
                                    share_server_id=share_srv['id'])
@@ -560,8 +560,7 @@ class ShareTestCase(test.TestCase):
         share_net = self._create_share_network()
         share_srv = self._create_share_server(
             share_network_id=share_net['id'],
-            host=self.share_manager.host,
+            host=self.share_manager.host
-            state='ACTIVE'
         )
         share = self._create_share(share_network_id=share_net['id'],
                                    share_server_id=share_srv['id'])
@@ -576,8 +575,7 @@ class ShareTestCase(test.TestCase):
         share_net = self._create_share_network()
         share_srv = self._create_share_server(
             share_network_id=share_net['id'],
-            host=self.share_manager.host,
+            host=self.share_manager.host
-            state='ACTIVE'
         )
         share = self._create_share(share_network_id=share_net['id'],
                                    share_server_id=share_srv['id'])

requirements.txt

@@ -1,4 +1,5 @@
 pbr>=0.6,!=0.7,<1.0
+alembic>=0.6.4
 anyjson>=0.3.3
 argparse
 Babel>=1.3
@@ -23,7 +24,6 @@ python-keystoneclient>=0.10.0
 Routes>=1.12.3,!=2.0
 six>=1.7.0
 SQLAlchemy>=0.8.4,<=0.8.99,>=0.9.7,<=0.9.99
-sqlalchemy-migrate>=0.9.1
 stevedore>=0.14
 python-cinderclient>=1.0.7
 python-novaclient>=2.17.0