Compact DB migrations to Ocata

This compacts all database migrations up to Ocata into one
intial schema to remove the need to apply every database
change along the way.

Change-Id: I53f45472d6c1aeafa35064cc1d80c5376ff32b9b
Signed-off-by: Sean McGinnis <sean.mcginnis@gmail.com>
This commit is contained in:
Sean McGinnis 2019-02-06 16:55:26 -06:00
parent 3449568666
commit 1133619108
No known key found for this signature in database
GPG Key ID: CE7EE4BFAF8D70C8
16 changed files with 68 additions and 466 deletions

View File

@ -26,7 +26,7 @@ from stevedore import driver
from cinder.db.sqlalchemy import api as db_api
INIT_VERSION = 84
INIT_VERSION = 96
_IMPL = None
_LOCK = threading.Lock()

View File

@ -1,68 +0,0 @@
# Copyright (c) 2016 Dell Inc. or its subsidiaries.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import uuid
from oslo_utils import timeutils
import six
from sqlalchemy import MetaData, Table
from cinder.volume import group_types as volume_group_types
def upgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
now = timeutils.utcnow()
group_types = Table('group_types', meta, autoload=True)
group_type_specs = Table('group_type_specs', meta, autoload=True)
# Create a default group_type for migrating cgsnapshots
results = list(group_types.select().where(
group_types.c.name == volume_group_types.DEFAULT_CGSNAPSHOT_TYPE and
group_types.c.deleted is False).
execute())
if not results:
grp_type_id = six.text_type(uuid.uuid4())
group_type_dicts = {
'id': grp_type_id,
'name': volume_group_types.DEFAULT_CGSNAPSHOT_TYPE,
'description': 'Default group type for migrating cgsnapshot',
'created_at': now,
'updated_at': now,
'deleted': False,
'is_public': True,
}
grp_type = group_types.insert()
grp_type.execute(group_type_dicts)
else:
grp_type_id = results[0]['id']
results = list(group_type_specs.select().where(
group_type_specs.c.group_type_id == grp_type_id and
group_type_specs.c.deleted is False).
execute())
if not results:
group_spec_dicts = {
'key': 'consistent_group_snapshot_enabled',
'value': '<is> True',
'group_type_id': grp_type_id,
'created_at': now,
'updated_at': now,
'deleted': False,
}
grp_spec = group_type_specs.insert()
grp_spec.execute(group_spec_dicts)

View File

@ -1,21 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from sqlalchemy import MetaData, Table
def upgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
messages = Table('messages', meta, autoload=True)
messages.c.request_id.alter(nullable=True)

View File

@ -1,36 +0,0 @@
# Copyright (c) 2016 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from sqlalchemy import Boolean, Column, MetaData, String, Table
from sqlalchemy.sql import expression
def upgrade(migrate_engine):
"""Add replication info to clusters table."""
meta = MetaData()
meta.bind = migrate_engine
clusters = Table('clusters', meta, autoload=True)
replication_status = Column('replication_status', String(length=36),
default="not-capable")
active_backend_id = Column('active_backend_id', String(length=255))
frozen = Column('frozen', Boolean, nullable=False, default=False,
server_default=expression.false())
if not hasattr(clusters.c, 'replication_status'):
clusters.create_column(replication_status)
if not hasattr(clusters.c, 'frozen'):
clusters.create_column(frozen)
if not hasattr(clusters.c, 'active_backend_id'):
clusters.create_column(active_backend_id)

View File

@ -1,28 +0,0 @@
# Copyright (c) 2016 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from sqlalchemy import Column
from sqlalchemy import MetaData, String, Table
def upgrade(migrate_engine):
"""Add cluster name to image cache entries."""
meta = MetaData()
meta.bind = migrate_engine
image_cache = Table('image_volume_cache_entries', meta, autoload=True)
cluster_name = Column('cluster_name', String(255), nullable=True)
if not hasattr(image_cache.c, 'cluster_name'):
image_cache.create_column(cluster_name)

View File

@ -1,28 +0,0 @@
# Copyright (c) 2016 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from sqlalchemy import Column, Integer, MetaData, Table, text
def upgrade(migrate_engine):
"""Add race preventer field to workers table."""
meta = MetaData()
meta.bind = migrate_engine
workers = Table('workers', meta, autoload=True)
race_preventer = Column('race_preventer', Integer, nullable=False,
default=0, server_default=text('0'))
if not hasattr(workers.c, 'race_preventer'):
race_preventer.create(workers, populate_default=True)

View File

@ -1,40 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer
from sqlalchemy import MetaData, String, Table
def upgrade(migrate_engine):
"""Add attachment_specs table."""
meta = MetaData()
meta.bind = migrate_engine
Table('volume_attachment', meta, autoload=True)
attachment_specs = Table(
'attachment_specs', meta,
Column('created_at', DateTime(timezone=False)),
Column('updated_at', DateTime(timezone=False)),
Column('deleted_at', DateTime(timezone=False)),
Column('deleted', Boolean(), default=False),
Column('id', Integer, primary_key=True, nullable=False),
Column('attachment_id', String(36),
ForeignKey('volume_attachment.id'),
nullable=False),
Column('key', String(255)),
Column('value', String(255)),
mysql_engine='InnoDB',
mysql_charset='utf8'
)
attachment_specs.create()

View File

@ -1,22 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Mitaka backports.
# Do not use this number for new Newton work. New work starts after
# all the placeholders.
#
# See this for more information:
# http://lists.openstack.org/pipermail/openstack-dev/2013-March/006827.html
def upgrade(migrate_engine):
pass

View File

@ -1,22 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Mitaka backports.
# Do not use this number for new Newton work. New work starts after
# all the placeholders.
#
# See this for more information:
# http://lists.openstack.org/pipermail/openstack-dev/2013-March/006827.html
def upgrade(migrate_engine):
pass

View File

@ -1,22 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Mitaka backports.
# Do not use this number for new Newton work. New work starts after
# all the placeholders.
#
# See this for more information:
# http://lists.openstack.org/pipermail/openstack-dev/2013-March/006827.html
def upgrade(migrate_engine):
pass

View File

@ -1,22 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Mitaka backports.
# Do not use this number for new Newton work. New work starts after
# all the placeholders.
#
# See this for more information:
# http://lists.openstack.org/pipermail/openstack-dev/2013-March/006827.html
def upgrade(migrate_engine):
pass

View File

@ -1,22 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Mitaka backports.
# Do not use this number for new Newton work. New work starts after
# all the placeholders.
#
# See this for more information:
# http://lists.openstack.org/pipermail/openstack-dev/2013-March/006827.html
def upgrade(migrate_engine):
pass

View File

@ -13,12 +13,16 @@
# under the License.
import datetime
import uuid
from oslo_config import cfg
from oslo_utils import timeutils
from sqlalchemy.dialects import mysql
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Index
from sqlalchemy import Integer, MetaData, String, Table, Text, UniqueConstraint
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Index, Integer
from sqlalchemy import MetaData, String, Table, Text, UniqueConstraint, text
from sqlalchemy.sql import expression
from cinder.volume import group_types as volume_group_types
# Get default values via config. The defaults will either
# come from the default values set in the quota option
@ -207,6 +211,22 @@ def define_tables(meta):
mysql_charset='utf8'
)
attachment_specs = Table(
'attachment_specs', meta,
Column('created_at', DateTime(timezone=False)),
Column('updated_at', DateTime(timezone=False)),
Column('deleted_at', DateTime(timezone=False)),
Column('deleted', Boolean(), default=False),
Column('id', Integer, primary_key=True, nullable=False),
Column('attachment_id', String(36),
ForeignKey('volume_attachment.id'),
nullable=False),
Column('key', String(255)),
Column('value', String(255)),
mysql_engine='InnoDB',
mysql_charset='utf8'
)
snapshots = Table(
'snapshots', meta,
Column('created_at', DateTime),
@ -536,6 +556,7 @@ def define_tables(meta):
Column('volume_id', String(36), nullable=False),
Column('size', Integer, nullable=False),
Column('last_used', DateTime, nullable=False),
Column('cluster_name', String(255)),
mysql_engine='InnoDB',
mysql_charset='utf8'
)
@ -544,7 +565,7 @@ def define_tables(meta):
'messages', meta,
Column('id', String(36), primary_key=True, nullable=False),
Column('project_id', String(36), nullable=False),
Column('request_id', String(255), nullable=False),
Column('request_id', String(255)),
Column('resource_type', String(36)),
Column('resource_uuid', String(255), nullable=True),
Column('event_id', String(255), nullable=False),
@ -570,6 +591,10 @@ def define_tables(meta):
Column('disabled', Boolean(), default=False),
Column('disabled_reason', String(255)),
Column('race_preventer', Integer, nullable=False, default=0),
Column('replication_status', String(length=36), default='not-capable'),
Column('active_backend_id', String(length=255)),
Column('frozen', Boolean, nullable=False, default=False,
server_default=expression.false()),
# To remove potential races on creation we have a constraint set on
# name and race_preventer fields, and we set value on creation to 0, so
# 2 clusters with the same name will fail this constraint. On deletion
@ -593,6 +618,8 @@ def define_tables(meta):
Column('status', String(255), nullable=False),
Column('service_id', Integer, ForeignKey('services.id'),
nullable=True),
Column('race_preventer', Integer, nullable=False, default=0,
server_default=text('0')),
UniqueConstraint('resource_type', 'resource_id'),
mysql_engine='InnoDB',
mysql_charset='utf8',
@ -664,6 +691,7 @@ def define_tables(meta):
group_snapshots,
volumes,
volume_attachment,
attachment_specs,
snapshots,
snapshot_metadata,
quality_of_service_specs,
@ -801,3 +829,32 @@ def upgrade(migrate_engine):
'resource_type': 'SENTINEL',
'resource_id': 'SUB-SECOND',
'status': 'OK'})
# Create default group type
group_types = Table('group_types', meta, autoload=True)
group_type_specs = Table('group_type_specs', meta, autoload=True)
now = timeutils.utcnow()
grp_type_id = "%s" % uuid.uuid4()
group_type_dicts = {
'id': grp_type_id,
'name': volume_group_types.DEFAULT_CGSNAPSHOT_TYPE,
'description': 'Default group type for migrating cgsnapshot',
'created_at': now,
'updated_at': now,
'deleted': False,
'is_public': True,
}
grp_type = group_types.insert()
grp_type.execute(group_type_dicts)
group_spec_dicts = {
'key': 'consistent_group_snapshot_enabled',
'value': '<is> True',
'group_type_id': grp_type_id,
'created_at': now,
'updated_at': now,
'deleted': False,
}
grp_spec = group_type_specs.insert()
grp_spec.execute(group_spec_dicts)

View File

@ -1,50 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from sqlalchemy import MetaData, Table, func, select
from cinder import exception
from cinder.i18n import _
WARNING_MSG = _('There are still %(count)i unmigrated records in '
'the %(table)s table. Migration cannot continue '
'until all records have been migrated.')
def upgrade(migrate_engine):
meta = MetaData(migrate_engine)
# CGs to Generic Volume Groups transition
consistencygroups = Table('consistencygroups', meta, autoload=True)
cgsnapshots = Table('cgsnapshots', meta, autoload=True)
for table in (consistencygroups, cgsnapshots):
count = select([func.count()]).select_from(table).where(
table.c.deleted == False).execute().scalar() # NOQA
if count > 0:
msg = WARNING_MSG % {
'count': count,
'table': table.name,
}
raise exception.ValidationError(detail=msg)
# VOLUME_ prefix addition in message IDs
messages = Table('messages', meta, autoload=True)
count = select([func.count()]).select_from(messages).where(
(messages.c.deleted == False) &
(~messages.c.event_id.like('VOLUME_%'))).execute().scalar() # NOQA
if count > 0:
msg = WARNING_MSG % {
'count': count,
'table': 'messages',
}
raise exception.ValidationError(detail=msg)

View File

@ -34,7 +34,6 @@ from sqlalchemy.engine import reflection
from cinder.db import migration
import cinder.db.sqlalchemy.migrate_repo
from cinder.volume import group_types as volume_group_types
class MigrationsMixin(test_migrations.WalkVersionsMixin):
@ -107,13 +106,8 @@ class MigrationsMixin(test_migrations.WalkVersionsMixin):
# manner is provided in Cinder's developer documentation.
# Reviewers: DO NOT ALLOW THINGS TO BE ADDED HERE WITHOUT CARE
exceptions = [
# NOTE(ameade): 87 sets messages.request_id to nullable. This is
# 100% backward compatible and according to MySQL docs such ALTER
# is performed with the same restrictions as column addition, which
# we of course allow.
87,
# NOTE : 104 modifies size of messages.project_id to 255.
# This should be safe for the same reason as migration 87.
# This should be safe according to documentation.
104,
# NOTE(brinzhang): 127 changes size of quota_usage.resource
# to 300. This should be safe for the 'quota_usage' db table,
@ -137,80 +131,6 @@ class MigrationsMixin(test_migrations.WalkVersionsMixin):
self.assertIsInstance(columns.deleted_at.type, self.TIME_TYPE)
self.assertIsInstance(columns.deleted.type, self.BOOL_TYPE)
def _check_086(self, engine, data):
"""Test inserting default cgsnapshot group type."""
self.assertTrue(engine.dialect.has_table(engine.connect(),
"group_types"))
group_types = db_utils.get_table(engine, 'group_types')
t1 = (group_types.select(group_types.c.name ==
volume_group_types.DEFAULT_CGSNAPSHOT_TYPE).
execute().first())
self.assertIsNotNone(t1)
group_specs = db_utils.get_table(engine, 'group_type_specs')
specs = group_specs.select(
group_specs.c.group_type_id == t1.id and
group_specs.c.key == 'consistent_group_snapshot_enabled'
).execute().first()
self.assertIsNotNone(specs)
self.assertEqual('<is> True', specs.value)
def _check_087(self, engine, data):
"""Test request_id column in messages is nullable."""
self.assertTrue(engine.dialect.has_table(engine.connect(),
"messages"))
messages = db_utils.get_table(engine, 'messages')
self.assertIsInstance(messages.c.request_id.type,
self.VARCHAR_TYPE)
self.assertTrue(messages.c.request_id.nullable)
def _check_088(self, engine, data):
"""Test adding replication data to cluster table."""
clusters = db_utils.get_table(engine, 'clusters')
self.assertIsInstance(clusters.c.replication_status.type,
self.VARCHAR_TYPE)
self.assertIsInstance(clusters.c.active_backend_id.type,
self.VARCHAR_TYPE)
self.assertIsInstance(clusters.c.frozen.type,
self.BOOL_TYPE)
def _check_089(self, engine, data):
"""Test adding cluster_name to image volume cache table."""
image_cache = db_utils.get_table(engine, 'image_volume_cache_entries')
self.assertIsInstance(image_cache.c.cluster_name.type,
self.VARCHAR_TYPE)
def _check_090(self, engine, data):
"""Test adding race_preventer to workers table."""
workers = db_utils.get_table(engine, 'workers')
self.assertIsInstance(workers.c.race_preventer.type,
self.INTEGER_TYPE)
def _check_091(self, engine, data):
self.assertTrue(engine.dialect.has_table(engine.connect(),
"attachment_specs"))
attachment = db_utils.get_table(engine, 'attachment_specs')
self.assertIsInstance(attachment.c.created_at.type,
self.TIME_TYPE)
self.assertIsInstance(attachment.c.updated_at.type,
self.TIME_TYPE)
self.assertIsInstance(attachment.c.deleted_at.type,
self.TIME_TYPE)
self.assertIsInstance(attachment.c.deleted.type,
self.BOOL_TYPE)
self.assertIsInstance(attachment.c.id.type,
self.INTEGER_TYPE)
self.assertIsInstance(attachment.c.key.type,
self.VARCHAR_TYPE)
self.assertIsInstance(attachment.c.value.type,
self.VARCHAR_TYPE)
self.assertIsInstance(attachment.c.attachment_id.type,
self.VARCHAR_TYPE)
f_keys = self.get_foreign_key_columns(engine, 'attachment_specs')
self.assertEqual({'attachment_id'}, f_keys)
def _check_098(self, engine, data):
self.assertTrue(engine.dialect.has_table(engine.connect(),
"messages"))

View File

@ -0,0 +1,6 @@
---
upgrade:
- |
The Cinder database can now only be upgraded with changes since the Ocata
release. In order to upgrade from a version prior to that, you must now
upgrade to at least Ocata first.