Add eventlet.tpool.Proxy for DB API calls

Ability to use thread pooling for DB API calls should be returned in
oslo.db as DB API wrapper.

Base on the fact that this wrapper is optional, `eventlet` should
not be added in requirements.

bp add-tpool-proxy-wrapper

Change-Id: I2343556c157e9f0f695e14ca0283914ef23c972c

oslo/db/concurrency.py

@@ -0,0 +1,81 @@
+# Copyright 2014 Mirantis.inc
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import copy
+import logging
+import threading
+
+from oslo.config import cfg
+
+from oslo.db import api
+from oslo.db.openstack.common.gettextutils import _LE
+
+
+LOG = logging.getLogger(__name__)
+
+tpool_opts = [
+    cfg.BoolOpt('use_tpool',
+                default=False,
+                deprecated_name='dbapi_use_tpool',
+                deprecated_group='DEFAULT',
+                help='Enable the experimental use of thread pooling for '
+                     'all DB API calls'),
+]
+
+
+class TpoolDbapiWrapper(object):
+    """DB API wrapper class.
+
+    This wraps the oslo DB API with an option to be able to use eventlet's
+    thread pooling. Since the CONF variable may not be loaded at the time
+    this class is instantiated, we must look at it on the first DB API call.
+    """
+
+    def __init__(self, conf, backend_mapping):
+        self._db_api = None
+        self._backend_mapping = backend_mapping
+        self._conf = conf
+        self._conf.register_opts(tpool_opts, 'database')
+        self._lock = threading.Lock()
+
+    @property
+    def _api(self):
+        if not self._db_api:
+            with self._lock:
+                if not self._db_api:
+                    db_api = api.DBAPI.from_config(
+                        conf=self._conf, backend_mapping=self._backend_mapping)
+                    if self._conf.database.use_tpool:
+                        try:
+                            from eventlet import tpool
+                        except ImportError:
+                            LOG.exception(_LE("'eventlet' is required for "
+                                              "TpoolDbapiWrapper."))
+                            raise
+                        self._db_api = tpool.Proxy(db_api)
+                    else:
+                        self._db_api = db_api
+        return self._db_api
+
+    def __getattr__(self, key):
+        return getattr(self._api, key)
+
+
+def list_opts():
+    """Returns a list of oslo.config options available in this module.
+
+    :returns: a list of (group_name, opts) tuples
+    """
+    return [('database', copy.deepcopy(tpool_opts))]

setup.cfg

@@ -27,6 +27,7 @@ namespace_packages =
 [entry_points]
 oslo.config.opts =
     oslo.db = oslo.db.options:list_opts
+    oslo.db.concurrency = oslo.db.concurrency:list_opts
 
 oslo.db.migration =
     alembic = oslo.db.sqlalchemy.migration_cli.ext_alembic:AlembicExtension

tests/test_concurrency.py

@@ -0,0 +1,108 @@
+# Copyright 2014 Mirantis.inc
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import sys
+
+import mock
+
+from oslo.db import concurrency
+from tests import utils as test_utils
+
+FAKE_BACKEND_MAPPING = {'sqlalchemy': 'fake.db.sqlalchemy.api'}
+
+
+class TpoolDbapiWrapperTestCase(test_utils.BaseTestCase):
+
+    def setUp(self):
+        super(TpoolDbapiWrapperTestCase, self).setUp()
+        self.db_api = concurrency.TpoolDbapiWrapper(
+            conf=self.conf, backend_mapping=FAKE_BACKEND_MAPPING)
+
+        # NOTE(akurilin): We are not  going to add `eventlet` to `oslo.db` in
+        # requirements (`requirements.txt` and `test-requirements.txt`) due to
+        # the following reasons:
+        #  - supporting of eventlet's thread pooling is totally optional;
+        #  - we don't need to test `tpool.Proxy` functionality itself,
+        #    because it's a tool from the third party library;
+        #  - `eventlet` would prevent us from running unit tests on Python 3.x
+        #    versions, because it doesn't support them yet.
+        #
+        # As we don't test `tpool.Proxy`, we can safely mock it in tests.
+
+        self.proxy = mock.MagicMock()
+        self.eventlet = mock.MagicMock()
+        self.eventlet.tpool.Proxy.return_value = self.proxy
+        sys.modules['eventlet'] = self.eventlet
+        self.addCleanup(sys.modules.pop, 'eventlet', None)
+
+    @mock.patch('oslo.db.api.DBAPI')
+    def test_db_api_common(self, mock_db_api):
+        # test context:
+        #     CONF.database.use_tpool == False
+        #     eventlet is installed
+        # expected result:
+        #     TpoolDbapiWrapper should wrap DBAPI
+
+        fake_db_api = mock.MagicMock()
+        mock_db_api.from_config.return_value = fake_db_api
+
+        # get access to some db-api method
+        self.db_api.fake_call_1
+
+        mock_db_api.from_config.assert_called_once_with(
+            conf=self.conf, backend_mapping=FAKE_BACKEND_MAPPING)
+        self.assertEqual(self.db_api._db_api, fake_db_api)
+        self.assertFalse(self.eventlet.tpool.Proxy.called)
+
+        # get access to other db-api method to be sure that api didn't changed
+        self.db_api.fake_call_2
+
+        self.assertEqual(self.db_api._db_api, fake_db_api)
+        self.assertFalse(self.eventlet.tpool.Proxy.called)
+        self.assertEqual(1, mock_db_api.from_config.call_count)
+
+    @mock.patch('oslo.db.api.DBAPI')
+    def test_db_api_config_change(self, mock_db_api):
+        # test context:
+        #     CONF.database.use_tpool == True
+        #     eventlet is installed
+        # expected result:
+        #     TpoolDbapiWrapper should wrap tpool proxy
+
+        fake_db_api = mock.MagicMock()
+        mock_db_api.from_config.return_value = fake_db_api
+        self.conf.set_override('use_tpool', True, group='database')
+
+        # get access to some db-api method
+        self.db_api.fake_call
+
+        # CONF.database.use_tpool is True, so we get tpool proxy in this case
+        mock_db_api.from_config.assert_called_once_with(
+            conf=self.conf, backend_mapping=FAKE_BACKEND_MAPPING)
+        self.eventlet.tpool.Proxy.assert_called_once_with(fake_db_api)
+        self.assertEqual(self.db_api._db_api, self.proxy)
+
+    @mock.patch('oslo.db.api.DBAPI')
+    def test_db_api_without_installed_eventlet(self, mock_db_api):
+        # test context:
+        #     CONF.database.use_tpool == True
+        #     eventlet is not installed
+        # expected result:
+        #     raise ImportError
+
+        self.conf.set_override('use_tpool', True, group='database')
+        del sys.modules['eventlet']
+
+        self.assertRaises(ImportError, getattr, self.db_api, 'fake')