openstack/oslo.db
Code Issues Proposed changes

Remove sqlalchemy < 1.0.0 compatible layer

Browse Source 
We bump sqlalchemy >=1.0.10, so don't need compatible layer
in master branch now.

Change-Id: Ib3974669893d2f7b059eab38468f127ee2c790af
This commit is contained in:
ChangBo Guo(gcb) 2016-03-14 21:50:22 +08:00
parent 045a6fb9fb
commit 09c531b389
6 changed files with 4 additions and 575 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
oslo_db/sqlalchemy/compat/__init__.py
View File

@ -1,25 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""compatiblity extensions for SQLAlchemy versions.
Elements within this module provide SQLAlchemy features that have been
added at some point but for which oslo.db provides a compatible versions
for previous SQLAlchemy versions.
"""
from oslo_db.sqlalchemy.compat import handle_error as _h_err
# trying to get: "from oslo_db.sqlalchemy import compat; compat.handle_error"
# flake8 won't let me import handle_error directly
handle_error = _h_err.handle_error
__all__ = ['handle_error']

341
oslo_db/sqlalchemy/compat/handle_error.py
View File

@ -1,341 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Provide forwards compatibility for the handle_error event.
See the "handle_error" event at
http://docs.sqlalchemy.org/en/rel_0_9/core/events.html.
"""
import sys
import six
from sqlalchemy.engine import base as engine_base
from sqlalchemy.engine import Engine
from sqlalchemy import event
from sqlalchemy import exc as sqla_exc
from oslo_db.sqlalchemy.compat import utils
def handle_error(engine, listener):
"""Add a handle_error listener for the given :class:`.Engine`.
This listener uses the SQLAlchemy
:meth:`sqlalchemy.event.ConnectionEvents.handle_error`
event.
"""
if utils.sqla_100:
event.listen(engine, "handle_error", listener)
return
assert isinstance(engine, Engine), \
"engine argument must be an Engine instance, not a Connection"
assert utils.sqla_097
_rework_connect_and_revalidate_for_events(engine)
# ctx.engine added per
# https://bitbucket.org/zzzeek/sqlalchemy/issue/3266/
def wrap_listener(ctx):
if isinstance(ctx, engine_base.ExceptionContextImpl):
ctx.engine = ctx.connection.engine
return listener(ctx)
event.listen(engine, "handle_error", wrap_listener)
def _rework_connect_and_revalidate_for_events(engine):
"""Patch the _revalidate_connection() system on Connection.
This applies 1.0's _revalidate_connection() approach into an 0.9
version of SQLAlchemy, and consists of three steps:
1. wrap the pool._creator function, which in 0.9 has a local
call to sqlalchemy.exc.DBAPIError.instance(), so that this exception is
again unwrapped back to the original DBAPI-specific Error, then raise
that. This is essentially the same as if the dbapi.connect() isn't
wrapped in the first place, which is how SQLAlchemy 1.0 now functions.
2. patch the Engine object's raw_connection() method. In SQLAlchemy 1.0,
this is now where the error wrapping occurs when a pool connect attempt
is made. Here, when raw_connection() is called without a hosting
Connection, we send exception raises to
_handle_dbapi_exception_noconnection(), here copied from SQLAlchemy
1.0, which is an alternate version of Connection._handle_dbapi_exception()
tailored for an initial connect failure when there is no
Connection object being dealt with. This allows the error handler
events to be called.
3. patch the Connection class to follow 1.0's behavior for
_revalidate_connection(); here, the call to engine.raw_connection()
will pass the raised error to Connection._handle_dbapi_exception(),
again allowing error handler events to be called.
"""
_orig_connect = engine.pool._creator
def connect():
try:
return _orig_connect()
except sqla_exc.DBAPIError as err:
original_exception = err.orig
raise original_exception
engine.pool._creator = connect
self = engine
def contextual_connect(close_with_result=False, **kwargs):
return self._connection_cls(
self,
self._wrap_pool_connect(self.pool.connect, None),
close_with_result=close_with_result,
**kwargs)
def _wrap_pool_connect(fn, connection):
dialect = self.dialect
try:
return fn()
except dialect.dbapi.Error as e:
if connection is None:
_handle_dbapi_exception_noconnection(
e, dialect, self)
else:
six.reraise(*sys.exc_info())
def raw_connection(_connection=None):
return self._wrap_pool_connect(
self.pool.unique_connection, _connection)
engine.contextual_connect = contextual_connect
engine._wrap_pool_connect = _wrap_pool_connect
engine.raw_connection = raw_connection
class Connection(engine._connection_cls):
@property
def connection(self):
"The underlying DB-API connection managed by this Connection."
try:
return self.__connection
except AttributeError:
try:
return self._revalidate_connection()
except Exception as e:
self._handle_dbapi_exception(e, None, None, None, None)
def _handle_dbapi_exception(self,
e,
statement,
parameters,
cursor,
context):
if self.invalidated:
# 0.9's _handle_dbapi_exception() can't handle
# a Connection that is invalidated already, meaning
# its "__connection" attribute is not set. So if we are
# in that case, call our "no connection" invalidator.
# this is fine as we are only supporting handle_error listeners
# that are applied at the engine level.
_handle_dbapi_exception_noconnection(
e, self.dialect, self.engine)
else:
super(Connection, self)._handle_dbapi_exception(
e, statement, parameters, cursor, context)
def _revalidate_connection(self):
if self._Connection__can_reconnect and self._Connection__invalid:
if self._Connection__transaction is not None:
raise sqla_exc.InvalidRequestError(
"Can't reconnect until invalid "
"transaction is rolled back")
self._Connection__connection = self.engine.raw_connection(
_connection=self)
self._Connection__invalid = False
return self._Connection__connection
raise sqla_exc.ResourceClosedError("This Connection is closed")
engine._connection_cls = Connection
def _handle_dbapi_exception_noconnection(e, dialect, engine):
exc_info = sys.exc_info()
is_disconnect = dialect.is_disconnect(e, None, None)
should_wrap = isinstance(e, dialect.dbapi.Error)
if should_wrap:
sqlalchemy_exception = sqla_exc.DBAPIError.instance(
None,
None,
e,
dialect.dbapi.Error,
connection_invalidated=is_disconnect)
else:
sqlalchemy_exception = None
newraise = None
ctx = ExceptionContextImpl(
e, sqlalchemy_exception, engine, None, None, None,
None, None, is_disconnect)
if hasattr(engine, '_oslo_handle_error_events'):
fns = engine._oslo_handle_error_events
else:
fns = engine.dispatch.handle_error
for fn in fns:
try:
# handler returns an exception;
# call next handler in a chain
per_fn = fn(ctx)
if per_fn is not None:
ctx.chained_exception = newraise = per_fn
except Exception as _raised:
# handler raises an exception - stop processing
newraise = _raised
break
if sqlalchemy_exception and \
is_disconnect != ctx.is_disconnect:
sqlalchemy_exception.connection_invalidated = \
is_disconnect = ctx.is_disconnect
if newraise:
six.reraise(type(newraise), newraise, exc_info[2])
elif should_wrap:
six.reraise(
type(sqlalchemy_exception), sqlalchemy_exception, exc_info[2])
else:
six.reraise(*exc_info)
class ExceptionContextImpl(object):
"""Encapsulate information about an error condition in progress.
This is for forwards compatibility with the
ExceptionContext interface introduced in SQLAlchemy 0.9.7.
It also provides for the "engine" argument added in SQLAlchemy 1.0.0.
"""
def __init__(self, exception, sqlalchemy_exception,
engine, connection, cursor, statement, parameters,
context, is_disconnect):
self.engine = engine
self.connection = connection
self.sqlalchemy_exception = sqlalchemy_exception
self.original_exception = exception
self.execution_context = context
self.statement = statement
self.parameters = parameters
self.is_disconnect = is_disconnect
connection = None
"""The :class:`.Connection` in use during the exception.
This member is present, except in the case of a failure when
first connecting.
"""
engine = None
"""The :class:`.Engine` in use during the exception.
This member should always be present, even in the case of a failure
when first connecting.
"""
cursor = None
"""The DBAPI cursor object.
May be None.
"""
statement = None
"""String SQL statement that was emitted directly to the DBAPI.
May be None.
"""
parameters = None
"""Parameter collection that was emitted directly to the DBAPI.
May be None.
"""
original_exception = None
"""The exception object which was caught.
This member is always present.
"""
sqlalchemy_exception = None
"""The :class:`sqlalchemy.exc.StatementError` which wraps the original,
and will be raised if exception handling is not circumvented by the event.
May be None, as not all exception types are wrapped by SQLAlchemy.
For DBAPI-level exceptions that subclass the dbapi's Error class, this
field will always be present.
"""
chained_exception = None
"""The exception that was returned by the previous handler in the
exception chain, if any.
If present, this exception will be the one ultimately raised by
SQLAlchemy unless a subsequent handler replaces it.
May be None.
"""
execution_context = None
"""The :class:`.ExecutionContext` corresponding to the execution
operation in progress.
This is present for statement execution operations, but not for
operations such as transaction begin/end. It also is not present when
the exception was raised before the :class:`.ExecutionContext`
could be constructed.
Note that the :attr:`.ExceptionContext.statement` and
:attr:`.ExceptionContext.parameters` members may represent a
different value than that of the :class:`.ExecutionContext`,
potentially in the case where a
:meth:`.ConnectionEvents.before_cursor_execute` event or similar
modified the statement/parameters to be sent.
May be None.
"""
is_disconnect = None
"""Represent whether the exception as occurred represents a "disconnect"
condition.
This flag will always be True or False within the scope of the
:meth:`.ConnectionEvents.handle_error` handler.
"""

4
oslo_db/sqlalchemy/exc_filters.py
View File

@ -15,11 +15,11 @@ import collections
import logging import logging
import re import re
from sqlalchemy import event
from sqlalchemy import exc as sqla_exc from sqlalchemy import exc as sqla_exc
from oslo_db._i18n import _LE from oslo_db._i18n import _LE
from oslo_db import exception from oslo_db import exception
from oslo_db.sqlalchemy import compat
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
@ -380,7 +380,7 @@ def handler(context):
def register_engine(engine): def register_engine(engine):
compat.handle_error(engine, handler) event.listen(engine, "handle_error", handler)
def handle_connect_error(engine): def handle_connect_error(engine):

3
oslo_db/sqlalchemy/provision.py
View File

@ -31,7 +31,6 @@ import testresources
from oslo_db._i18n import _LI from oslo_db._i18n import _LI
from oslo_db import exception from oslo_db import exception
from oslo_db.sqlalchemy.compat import utils as compat_utils
from oslo_db.sqlalchemy import session from oslo_db.sqlalchemy import session
from oslo_db.sqlalchemy import utils from oslo_db.sqlalchemy import utils
@ -530,7 +529,7 @@ class PostgresqlBackendImpl(BackendImpl):
conn.execute("DROP DATABASE %s" % ident) conn.execute("DROP DATABASE %s" % ident)
def drop_additional_objects(self, conn): def drop_additional_objects(self, conn):
enums = compat_utils.get_postgresql_enums(conn) enums = [e['name'] for e in sqlalchemy.inspect(conn).get_enums()]
for e in enums: for e in enums:
conn.execute("DROP TYPE %s" % e) conn.execute("DROP TYPE %s" % e)

194
oslo_db/tests/sqlalchemy/test_handle_error.py
View File

@ -1,194 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Test the compatibility layer for the handle_error() event.
This event is added as of SQLAlchemy 0.9.7; oslo_db provides a compatibility
layer for prior SQLAlchemy versions.
"""
import mock
from oslotest import base as test_base
import sqlalchemy as sqla
from sqlalchemy.sql import column
from sqlalchemy.sql import literal
from sqlalchemy.sql import select
from sqlalchemy.types import Integer
from sqlalchemy.types import TypeDecorator
from oslo_db.sqlalchemy.compat import handle_error
from oslo_db.sqlalchemy.compat import utils
from oslo_db.tests import utils as test_utils
class MyException(Exception):
pass
class ExceptionReraiseTest(test_base.BaseTestCase):
def setUp(self):
super(ExceptionReraiseTest, self).setUp()
self.engine = engine = sqla.create_engine("sqlite://")
self.addCleanup(engine.dispose)
def _fixture(self):
engine = self.engine
def err(context):
if "ERROR ONE" in str(context.statement):
raise MyException("my exception")
handle_error(engine, err)
def test_exception_event_altered(self):
self._fixture()
with mock.patch.object(self.engine.dialect.execution_ctx_cls,
"handle_dbapi_exception") as patched:
matchee = self.assertRaises(
MyException,
self.engine.execute, "SELECT 'ERROR ONE' FROM I_DONT_EXIST"
)
self.assertEqual(1, patched.call_count)
self.assertEqual("my exception", matchee.args[0])
def test_exception_event_non_altered(self):
self._fixture()
with mock.patch.object(self.engine.dialect.execution_ctx_cls,
"handle_dbapi_exception") as patched:
self.assertRaises(
sqla.exc.DBAPIError,
self.engine.execute, "SELECT 'ERROR TWO' FROM I_DONT_EXIST"
)
self.assertEqual(1, patched.call_count)
def test_is_disconnect_not_interrupted(self):
self._fixture()
with test_utils.nested(
mock.patch.object(
self.engine.dialect.execution_ctx_cls,
"handle_dbapi_exception"
),
mock.patch.object(
self.engine.dialect, "is_disconnect",
lambda *args: True
)
) as (handle_dbapi_exception, is_disconnect):
with self.engine.connect() as conn:
self.assertRaises(
MyException,
conn.execute, "SELECT 'ERROR ONE' FROM I_DONT_EXIST"
)
self.assertEqual(1, handle_dbapi_exception.call_count)
self.assertTrue(conn.invalidated)
def test_no_is_disconnect_not_invalidated(self):
self._fixture()
with test_utils.nested(
mock.patch.object(
self.engine.dialect.execution_ctx_cls,
"handle_dbapi_exception"
),
mock.patch.object(
self.engine.dialect, "is_disconnect",
lambda *args: False
)
) as (handle_dbapi_exception, is_disconnect):
with self.engine.connect() as conn:
self.assertRaises(
MyException,
conn.execute, "SELECT 'ERROR ONE' FROM I_DONT_EXIST"
)
self.assertEqual(1, handle_dbapi_exception.call_count)
self.assertFalse(conn.invalidated)
def test_exception_event_ad_hoc_context(self):
engine = self.engine
nope = MyException("nope")
class MyType(TypeDecorator):
impl = Integer
def process_bind_param(self, value, dialect):
raise nope
listener = mock.Mock(return_value=None)
handle_error(engine, listener)
self.assertRaises(
sqla.exc.StatementError,
engine.execute,
select([1]).where(column('foo') == literal('bar', MyType))
)
ctx = listener.mock_calls[0][1][0]
self.assertTrue(ctx.statement.startswith("SELECT 1 "))
self.assertIs(ctx.is_disconnect, False)
self.assertIs(ctx.original_exception, nope)
def _test_alter_disconnect(self, orig_error, evt_value):
engine = self.engine
def evt(ctx):
ctx.is_disconnect = evt_value
handle_error(engine, evt)
# if we are under sqla 0.9.7, and we are expecting to take
# an "is disconnect" exception and make it not a disconnect,
# that isn't supported b.c. the wrapped handler has already
# done the invalidation.
expect_failure = not utils.sqla_097 and orig_error and not evt_value
with mock.patch.object(engine.dialect,
"is_disconnect",
mock.Mock(return_value=orig_error)):
with engine.connect() as c:
conn_rec = c.connection._connection_record
try:
c.execute("SELECT x FROM nonexistent")
assert False
except sqla.exc.StatementError as st:
self.assertFalse(expect_failure)
# check the exception's invalidation flag
self.assertEqual(st.connection_invalidated, evt_value)
# check the Connection object's invalidation flag
self.assertEqual(c.invalidated, evt_value)
# this is the ConnectionRecord object; it's invalidated
# when its .connection member is None
self.assertEqual(conn_rec.connection is None, evt_value)
except NotImplementedError as ne:
self.assertTrue(expect_failure)
self.assertEqual(
str(ne),
"Can't reset 'disconnect' status of exception once it "
"is set with this version of SQLAlchemy")
def test_alter_disconnect_to_true(self):
self._test_alter_disconnect(False, True)
self._test_alter_disconnect(True, True)
def test_alter_disconnect_to_false(self):
self._test_alter_disconnect(True, False)
self._test_alter_disconnect(False, False)

12
oslo_db/tests/sqlalchemy/test_utils.py
View File

@ -482,11 +482,7 @@ class TestMigrationUtils(db_test_base.DbTestCase):
foo=fooColumn) foo=fooColumn)
table = utils.get_table(self.engine, table_name) table = utils.get_table(self.engine, table_name)
# NOTE(boris-42): There is no way to check has foo type CustomType.
# but sqlalchemy will set it to NullType. This has
# been fixed upstream in recent SA versions
if SA_VERSION < (0, 9, 0):
self.assertTrue(isinstance(table.c.foo.type, NullType))
self.assertTrue(isinstance(table.c.deleted.type, Integer)) self.assertTrue(isinstance(table.c.deleted.type, Integer))
def test_change_deleted_column_type_to_boolean(self): def test_change_deleted_column_type_to_boolean(self):
@ -537,12 +533,6 @@ class TestMigrationUtils(db_test_base.DbTestCase):
Column('deleted', Integer)) Column('deleted', Integer))
table.create() table.create()
# reflection of custom types has been fixed upstream
if SA_VERSION < (0, 9, 0):
self.assertRaises(exception.ColumnError,
utils.change_deleted_column_type_to_boolean,
self.engine, table_name)
fooColumn = Column('foo', CustomType()) fooColumn = Column('foo', CustomType())
utils.change_deleted_column_type_to_boolean(self.engine, table_name, utils.change_deleted_column_type_to_boolean(self.engine, table_name,
foo=fooColumn) foo=fooColumn)