Refactor/prepare db common utils for neutron-lib

Extract all the common utils from common_db_mixin.py in preparation
for moving them to neutron-lib.

This is a preliminary step in preparation for refactoring the
CommonDbMixin class and moving it to neutron-lib also.

Partial Blueprint: neutron-lib

Change-Id: I3cba375a8162cb68e8f988f22f5c8b1ce7915180

doc/source/devref/effective_neutron.rst

@@ -159,7 +159,7 @@ Document common pitfalls as well as good practices done during database developm
   section because internal transaction from ``_do_other_thing_with_created_object``
   has been rolled back. To avoid this nested transactions should be used.
   For such cases help function ``safe_creation`` has been created in
-  ``neutron/db/common_db_mixin.py``.
+  ``neutron/db/_utils.py``.
   So, the example above should be replaced with:
 
   .. code:: python

neutron/db/_utils.py

@@ -0,0 +1,123 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+NOTE: This module shall not be used by external projects. It will be moved
+      to neutron-lib in due course, and then it can be used from there.
+"""
+
+import contextlib
+
+from oslo_log import log as logging
+from oslo_utils import excutils
+import six
+from sqlalchemy.ext import associationproxy
+
+from neutron._i18n import _LE
+from neutron.api.v2 import attributes
+
+LOG = logging.getLogger(__name__)
+
+
+@contextlib.contextmanager
+def _noop_context_manager():
+    yield
+
+
+def safe_creation(context, create_fn, delete_fn, create_bindings,
+                  transaction=True):
+    '''This function wraps logic of object creation in safe atomic way.
+
+    In case of exception, object is deleted.
+
+    More information when this method could be used can be found in
+    developer guide - Effective Neutron: Database interaction section.
+    http://docs.openstack.org/developer/neutron/devref/effective_neutron.html
+
+    :param context: context
+
+    :param create_fn: function without arguments that is called to create
+        object and returns this object.
+
+    :param delete_fn: function that is called to delete an object. It is
+        called with object's id field as an argument.
+
+    :param create_bindings: function that is called to create bindings for
+        an object. It is called with object's id field as an argument.
+
+    :param transaction: if true the whole operation will be wrapped in a
+        transaction. if false, no transaction will be used.
+    '''
+    cm = (context.session.begin(subtransactions=True)
+          if transaction else _noop_context_manager())
+    with cm:
+        obj = create_fn()
+        try:
+            value = create_bindings(obj['id'])
+        except Exception:
+            with excutils.save_and_reraise_exception():
+                try:
+                    delete_fn(obj['id'])
+                except Exception as e:
+                    LOG.error(_LE("Cannot clean up created object %(obj)s. "
+                                  "Exception: %(exc)s"), {'obj': obj['id'],
+                                                          'exc': e})
+        return obj, value
+
+
+def model_query_scope_is_project(context, model):
+    # Unless a context has 'admin' or 'advanced-service' rights the
+    # query will be scoped to a single project_id
+    return ((not context.is_admin and hasattr(model, 'project_id')) and
+            (not context.is_advsvc and hasattr(model, 'project_id')))
+
+
+def model_query(context, model):
+    query = context.session.query(model)
+    # define basic filter condition for model query
+    query_filter = None
+    if model_query_scope_is_project(context, model):
+        query_filter = (model.tenant_id == context.tenant_id)
+
+    if query_filter is not None:
+        query = query.filter(query_filter)
+    return query
+
+
+# NOTE: This used to be CommonDbMixin._fields()
+def resource_fields(resource, fields):
+    """Return only the resource items that are in fields.
+
+    :param resource: a resource dictionary
+    :type resource: dict
+    :param fields: a list of fields to select from the resource
+    :type fields: list
+
+    """
+    if fields:
+        resource = {key: item for key, item in resource.items()
+                    if key in fields}
+    return attributes.populate_project_info(resource)
+
+
+# NOTE: This used to be CommonDbMixin._filter_non_model_columns
+def filter_non_model_columns(data, model):
+    """Return the attributes from data which are model columns.
+
+    Return a new dict with items from data that whose keys are columns in
+    the model or are association proxies of the model.
+    """
+    columns = [c.name for c in model.__table__.columns]
+    return dict((k, v) for (k, v) in
+                six.iteritems(data) if k in columns or
+                isinstance(getattr(model, k, None),
+                           associationproxy.AssociationProxy))

neutron/db/address_scope_db.py

@@ -19,6 +19,7 @@ from sqlalchemy.orm import exc
 from neutron._i18n import _
 from neutron.api.v2 import attributes as attr
 from neutron.common import _deprecate
+from neutron.db import _utils as db_utils
 from neutron.db import db_base_plugin_v2
 from neutron.db.models import address_scope as address_scope_model
 from neutron.extensions import address_scope as ext_address_scope
@@ -33,13 +34,14 @@ class AddressScopeDbMixin(ext_address_scope.AddressScopePluginBase):
 
     __native_bulk_support = True
 
-    def _make_address_scope_dict(self, address_scope, fields=None):
+    @staticmethod
+    def _make_address_scope_dict(address_scope, fields=None):
         res = {'id': address_scope['id'],
                'name': address_scope['name'],
                'tenant_id': address_scope['tenant_id'],
                'shared': address_scope['shared'],
                'ip_version': address_scope['ip_version']}
-        return self._fields(res, fields)
+        return db_utils.resource_fields(res, fields)
 
     def _get_address_scope(self, context, id):
         try:

neutron/db/agents_db.py

@@ -38,6 +38,7 @@ from neutron.callbacks import resources
 from neutron.common import _deprecate
 from neutron.common import constants as n_const
 from neutron import context
+from neutron.db import _utils as db_utils
 from neutron.db import api as db_api
 from neutron.db.models import agent as agent_model
 from neutron.extensions import agent as ext_agent
@@ -226,7 +227,7 @@ class AgentDbMixin(ext_agent.AgentPluginBase, AgentAvailabilityZoneMixin):
         res['resource_versions'] = self._get_dict(agent, 'resource_versions',
                                                   ignore_missing=True)
         res['availability_zone'] = agent['availability_zone']
-        return self._fields(res, fields)
+        return db_utils.resource_fields(res, fields)
 
     @db_api.retry_if_session_inactive()
     def delete_agent(self, context, id):

neutron/db/allowedaddresspairs_db.py

@@ -16,6 +16,7 @@
 from neutron_lib.api import validators
 
 from neutron.api.v2 import attributes as attr
+from neutron.db import _utils as db_utils
 from neutron.db import db_base_plugin_v2
 
 from neutron.common import utils
@@ -86,11 +87,12 @@ class AllowedAddressPairsMixin(object):
             for pair in pairs:
                 pair.delete()
 
-    def _make_allowed_address_pairs_dict(self, allowed_address_pairs,
+    @staticmethod
+    def _make_allowed_address_pairs_dict(allowed_address_pairs,
                                          fields=None):
         res = {'mac_address': allowed_address_pairs['mac_address'],
                'ip_address': allowed_address_pairs['ip_address']}
-        return self._fields(res, fields)
+        return db_utils.resource_fields(res, fields)
 
     def _has_address_pairs(self, port):
         return (validators.is_attr_set(port['port'][addr_pair.ADDRESS_PAIRS])

neutron/db/common_db_mixin.py

@@ -13,89 +13,28 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import contextlib
 import weakref
 
 from neutron_lib.db import utils as db_utils
 from oslo_db.sqlalchemy import utils as sa_utils
 from oslo_log import log as logging
-from oslo_utils import excutils
 import six
 from sqlalchemy import and_
 from sqlalchemy.ext import associationproxy
 from sqlalchemy import or_
 from sqlalchemy import sql
 
-from neutron._i18n import _LE
 from neutron.api.v2 import attributes
-
+from neutron.db import _utils as ndb_utils
 
 LOG = logging.getLogger(__name__)
 
 
-@contextlib.contextmanager
+# TODO(HenryG): Remove these when available in neutron-lib
-def _noop_context_manager():
+safe_creation = ndb_utils.safe_creation
-    yield
+model_query_scope = ndb_utils.model_query_scope_is_project
-
+model_query = ndb_utils.model_query
-
+resource_fields = ndb_utils.resource_fields
-def safe_creation(context, create_fn, delete_fn, create_bindings,
-                  transaction=True):
-    '''This function wraps logic of object creation in safe atomic way.
-
-    In case of exception, object is deleted.
-
-    More information when this method could be used can be found in
-    developer guide - Effective Neutron: Database interaction section.
-    http://docs.openstack.org/developer/neutron/devref/effective_neutron.html
-
-    :param context: context
-
-    :param create_fn: function without arguments that is called to create
-        object and returns this object.
-
-    :param delete_fn: function that is called to delete an object. It is
-        called with object's id field as an argument.
-
-    :param create_bindings: function that is called to create bindings for
-        an object. It is called with object's id field as an argument.
-
-    :param transaction: if true the whole operation will be wrapped in a
-        transaction. if false, no transaction will be used.
-    '''
-    cm = (context.session.begin(subtransactions=True)
-          if transaction else _noop_context_manager())
-    with cm:
-        obj = create_fn()
-        try:
-            value = create_bindings(obj['id'])
-        except Exception:
-            with excutils.save_and_reraise_exception():
-                try:
-                    delete_fn(obj['id'])
-                except Exception as e:
-                    LOG.error(_LE("Cannot clean up created object %(obj)s. "
-                                  "Exception: %(exc)s"), {'obj': obj['id'],
-                                                          'exc': e})
-        return obj, value
-
-
-def model_query_scope(context, model):
-    # Unless a context has 'admin' or 'advanced-service' rights the
-    # query will be scoped to a single tenant_id
-    return ((not context.is_admin and hasattr(model, 'tenant_id')) and
-            (not context.is_advsvc and hasattr(model, 'tenant_id')))
-
-
-def model_query(context, model):
-    query = context.session.query(model)
-    # define basic filter condition for model query
-    query_filter = None
-    if model_query_scope(context, model):
-        query_filter = (model.tenant_id == context.tenant_id)
-
-    if query_filter is not None:
-        query = query.filter(query_filter)
-    return query
 
 
 class CommonDbMixin(object):
@@ -147,14 +86,15 @@ class CommonDbMixin(object):
         """
         return weakref.proxy(self)
 
+    # TODO(HenryG): Remove this when available in neutron-lib
     def model_query_scope(self, context, model):
-        return model_query_scope(context, model)
+        return ndb_utils.model_query_scope_is_project(context, model)
 
     def _model_query(self, context, model):
         query = context.session.query(model)
         # define basic filter condition for model query
         query_filter = None
-        if self.model_query_scope(context, model):
+        if ndb_utils.model_query_scope_is_project(context, model):
             if hasattr(model, 'rbac_entries'):
                 query = query.outerjoin(model.rbac_entries)
                 rbac_model = model.rbac_entries.property.mapper.class_
@@ -189,11 +129,9 @@ class CommonDbMixin(object):
             query = query.filter(query_filter)
         return query
 
+    # TODO(HenryG): Remove this when available in neutron-lib
     def _fields(self, resource, fields):
-        if fields:
+        return ndb_utils.resource_fields(resource, fields)
-            resource = {key: item for key, item in resource.items()
-                        if key in fields}
-        return attributes.populate_project_info(resource)
 
     def _get_by_id(self, context, model, id):
         query = self._model_query(context, model)
@@ -315,12 +253,6 @@ class CommonDbMixin(object):
             return getattr(self, '_get_%s' % resource)(context, marker)
         return None
 
+    # TODO(HenryG): Remove this when available in neutron-lib
     def _filter_non_model_columns(self, data, model):
-        """Remove all the attributes from data which are not columns or
+        return ndb_utils.filter_non_model_columns(data, model)
-        association proxies of the model passed as second parameter
-        """
-        columns = [c.name for c in model.__table__.columns]
-        return dict((k, v) for (k, v) in
-                    six.iteritems(data) if k in columns or
-                    isinstance(getattr(model, k, None),
-                               associationproxy.AssociationProxy))

neutron/db/db_base_plugin_common.py

@@ -26,6 +26,7 @@ from neutron.api.v2 import attributes
 from neutron.common import constants as n_const
 from neutron.common import exceptions
 from neutron.common import utils
+from neutron.db import _utils as db_utils
 from neutron.db import common_db_mixin
 from neutron.db import models_v2
 from neutron.objects import subnet as subnet_obj
@@ -153,7 +154,7 @@ class DbBasePluginCommon(common_db_mixin.CommonDbMixin):
         res['shared'] = self._is_network_shared(context, subnet.networks)
         # Call auxiliary extend functions, if any
         self._apply_dict_extend_functions(attributes.SUBNETS, res, subnet)
-        return self._fields(res, fields)
+        return db_utils.resource_fields(res, fields)
 
     def _make_subnetpool_dict(self, subnetpool, fields=None):
         default_prefixlen = str(subnetpool['default_prefixlen'])
@@ -173,7 +174,7 @@ class DbBasePluginCommon(common_db_mixin.CommonDbMixin):
                'address_scope_id': subnetpool['address_scope_id']}
         self._apply_dict_extend_functions(attributes.SUBNETPOOLS, res,
                                           subnetpool)
-        return self._fields(res, fields)
+        return db_utils.resource_fields(res, fields)
 
     def _make_port_dict(self, port, fields=None,
                         process_extensions=True):
@@ -193,7 +194,7 @@ class DbBasePluginCommon(common_db_mixin.CommonDbMixin):
         if process_extensions:
             self._apply_dict_extend_functions(
                 attributes.PORTS, res, port)
-        return self._fields(res, fields)
+        return db_utils.resource_fields(res, fields)
 
     def _get_network(self, context, id):
         try:
@@ -278,7 +279,7 @@ class DbBasePluginCommon(common_db_mixin.CommonDbMixin):
         if process_extensions:
             self._apply_dict_extend_functions(
                 attributes.NETWORKS, res, network)
-        return self._fields(res, fields)
+        return db_utils.resource_fields(res, fields)
 
     def _is_network_shared(self, context, network):
         # The shared attribute for a network now reflects if the network
@@ -315,20 +316,22 @@ class DbBasePluginCommon(common_db_mixin.CommonDbMixin):
                  'ip_address': ip["ip_address"]}
                 for ip in ips]
 
-    def _port_filter_hook(self, context, original_model, conditions):
+    @staticmethod
+    def _port_filter_hook(context, original_model, conditions):
         # Apply the port filter only in non-admin and non-advsvc context
-        if self.model_query_scope(context, original_model):
+        if db_utils.model_query_scope_is_project(context, original_model):
             conditions |= (
                 (context.tenant_id == models_v2.Network.tenant_id) &
                 (models_v2.Network.id == models_v2.Port.network_id))
         return conditions
 
-    def _port_query_hook(self, context, original_model, query):
+    @staticmethod
+    def _port_query_hook(context, original_model, query):
         # we need to outerjoin to networks if the model query scope
         # is necessary so we can filter based on network id. without
         # this the conditions in the filter hook cause the networks
         # table to be added to the FROM statement so we get lots of
         # duplicated rows that break the COUNT operation
-        if self.model_query_scope(context, original_model):
+        if db_utils.model_query_scope_is_project(context, original_model):
             query = query.outerjoin(models_v2.Network)
         return query

neutron/db/db_base_plugin_v2.py

@@ -42,6 +42,7 @@ from neutron.common import exceptions as n_exc
 from neutron.common import ipv6_utils
 from neutron.common import utils
 from neutron import context as ctx
+from neutron.db import _utils as ndb_utils
 from neutron.db import api as db_api
 from neutron.db import db_base_plugin_common
 from neutron.db import ipam_pluggable_backend
@@ -394,8 +395,8 @@ class NeutronDbPluginV2(db_base_plugin_common.DbBasePluginCommon,
             # The filter call removes attributes from the body received from
             # the API that are logically tied to network resources but are
             # stored in other database tables handled by extensions
-            network.update(self._filter_non_model_columns(n,
+            network.update(
-                                                          models_v2.Network))
+                ndb_utils.filter_non_model_columns(n, models_v2.Network))
         return self._make_network_dict(network, context=context)
 
     @db_api.retry_if_session_inactive()

neutron/db/external_net_db.py

@@ -26,6 +26,7 @@ from neutron.callbacks import exceptions as c_exc
 from neutron.callbacks import registry
 from neutron.callbacks import resources
 from neutron.common import _deprecate
+from neutron.db import _utils as db_utils
 from neutron.db import db_base_plugin_v2
 from neutron.db.models import external_net as ext_net_models
 from neutron.db.models import l3 as l3_models
@@ -51,12 +52,13 @@ class External_net_db_mixin(object):
                                  ext_net_models.ExternalNetwork.network_id))
         return query
 
-    def _network_filter_hook(self, context, original_model, conditions):
+    @staticmethod
+    def _network_filter_hook(context, original_model, conditions):
         if conditions is not None and not hasattr(conditions, '__iter__'):
             conditions = (conditions, )
         # Apply the external network filter only in non-admin and non-advsvc
         # context
-        if self.model_query_scope(context, original_model):
+        if db_utils.model_query_scope_is_project(context, original_model):
             # the table will already be joined to the rbac entries for the
             # shared check so we don't need to worry about ensuring that
             rbac_model = original_model.rbac_entries.property.mapper.class_

neutron/db/flavors_db.py

@@ -18,6 +18,7 @@ from oslo_utils import uuidutils
 from sqlalchemy.orm import exc as sa_exc
 
 from neutron.common import _deprecate
+from neutron.db import _utils as db_utils
 from neutron.db import common_db_mixin
 from neutron.db.models import flavor as flavor_models
 from neutron.db import servicetype_db as sdb
@@ -49,7 +50,8 @@ class FlavorsDbMixin(common_db_mixin.CommonDbMixin):
         except sa_exc.NoResultFound:
             raise ext_flavors.ServiceProfileNotFound(sp_id=sp_id)
 
-    def _make_flavor_dict(self, flavor_db, fields=None):
+    @staticmethod
+    def _make_flavor_dict(flavor_db, fields=None):
         res = {'id': flavor_db['id'],
                'name': flavor_db['name'],
                'description': flavor_db['description'],
@@ -59,9 +61,10 @@ class FlavorsDbMixin(common_db_mixin.CommonDbMixin):
         if flavor_db.service_profiles:
             res['service_profiles'] = [sp['service_profile_id']
                                        for sp in flavor_db.service_profiles]
-        return self._fields(res, fields)
+        return db_utils.resource_fields(res, fields)
 
-    def _make_service_profile_dict(self, sp_db, fields=None):
+    @staticmethod
+    def _make_service_profile_dict(sp_db, fields=None):
         res = {'id': sp_db['id'],
                'description': sp_db['description'],
                'driver': sp_db['driver'],
@@ -70,7 +73,7 @@ class FlavorsDbMixin(common_db_mixin.CommonDbMixin):
         if sp_db.flavors:
             res['flavors'] = [fl['flavor_id']
                               for fl in sp_db.flavors]
-        return self._fields(res, fields)
+        return db_utils.resource_fields(res, fields)
 
     def _ensure_flavor_not_in_use(self, context, flavor_id):
         """Checks that flavor is not associated with service instance."""
@@ -177,7 +180,8 @@ class FlavorsDbMixin(common_db_mixin.CommonDbMixin):
                     sp_id=service_profile_id, fl_id=flavor_id)
             context.session.delete(binding)
 
-    def get_flavor_service_profile(self, context,
+    @staticmethod
+    def get_flavor_service_profile(context,
                                    service_profile_id, flavor_id, fields=None):
         with context.session.begin(subtransactions=True):
             binding = (
@@ -191,7 +195,7 @@ class FlavorsDbMixin(common_db_mixin.CommonDbMixin):
                     sp_id=service_profile_id, fl_id=flavor_id)
         res = {'service_profile_id': service_profile_id,
                'flavor_id': flavor_id}
-        return self._fields(res, fields)
+        return db_utils.resource_fields(res, fields)
 
     def create_service_profile(self, context, service_profile):
         sp = service_profile['service_profile']
@@ -283,7 +287,7 @@ class FlavorsDbMixin(common_db_mixin.CommonDbMixin):
         res = {'driver': sp_db.driver,
                'provider': providers[0].get('name')}
 
-        return [self._fields(res, fields)]
+        return [db_utils.resource_fields(res, fields)]
 
 
 _deprecate._MovedGlobals()

neutron/db/ipam_backend_mixin.py

@@ -32,6 +32,7 @@ from neutron.common import constants
 from neutron.common import exceptions as n_exc
 from neutron.common import ipv6_utils
 from neutron.common import utils as common_utils
+from neutron.db import _utils as db_utils
 from neutron.db import db_base_plugin_common
 from neutron.db.models import segment as segment_model
 from neutron.db.models import subnet_service_type as sst_model
@@ -107,8 +108,8 @@ class IpamBackendMixin(db_base_plugin_common.DbBasePluginCommon):
         if (new_mac and new_mac != db_port.mac_address and
                 self._is_mac_in_use(context, network_id, new_mac)):
             raise exc.MacAddressInUse(net_id=network_id, mac=new_mac)
-        db_port.update(self._filter_non_model_columns(new_port,
+        db_port.update(db_utils.filter_non_model_columns(new_port,
-                                                      models_v2.Port))
+                                                         models_v2.Port))
 
     def _update_subnet_host_routes(self, context, id, s):
 

neutron/db/l3_db.py

@@ -38,6 +38,7 @@ from neutron.common import constants as n_const
 from neutron.common import ipv6_utils
 from neutron.common import rpc as n_rpc
 from neutron.common import utils
+from neutron.db import _utils as db_utils
 from neutron.db import api as db_api
 from neutron.db import common_db_mixin
 from neutron.db.models import l3 as l3_models
@@ -135,7 +136,7 @@ class L3_NAT_dbonly_mixin(l3.RouterPluginBase,
         # plugins.
         if process_extensions:
             self._apply_dict_extend_functions(l3.ROUTERS, res, router)
-        return self._fields(res, fields)
+        return db_utils.resource_fields(res, fields)
 
     def filter_allocating_and_missing_routers(self, context, routers):
         """Filter out routers that shouldn't go to the agent.
@@ -974,7 +975,7 @@ class L3_NAT_dbonly_mixin(l3.RouterPluginBase,
         # plugins.
         if process_extensions:
             self._apply_dict_extend_functions(l3.FLOATINGIPS, res, floatingip)
-        return self._fields(res, fields)
+        return db_utils.resource_fields(res, fields)
 
     def _get_router_for_floatingip(self, context, internal_port,
                                    internal_subnet_id,

neutron/db/l3_hamode_db.py

@@ -34,9 +34,9 @@ from neutron.api.v2 import attributes
 from neutron.common import _deprecate
 from neutron.common import constants as n_const
 from neutron.common import utils as n_utils
+from neutron.db import _utils as db_utils
 from neutron.db import api as db_api
 from neutron.db.availability_zone import router as router_az_db
-from neutron.db import common_db_mixin
 from neutron.db import l3_dvr_db
 from neutron.db.l3_dvr_db import is_distributed_router
 from neutron.db.models import agent as agent_model
@@ -246,7 +246,7 @@ class L3_HA_NAT_db_mixin(l3_dvr_db.L3_NAT_with_dvr_db_mixin,
         deletion = functools.partial(self._core_plugin.delete_network,
                                      admin_ctx)
 
-        network, ha_network = common_db_mixin.safe_creation(
+        network, ha_network = db_utils.safe_creation(
             context, creation, deletion, content, transaction=False)
         try:
             self._create_ha_subnet(admin_ctx, network['id'], tenant_id)
@@ -327,9 +327,9 @@ class L3_HA_NAT_db_mixin(l3_dvr_db.L3_NAT_with_dvr_db_mixin,
                                     router_id)
         deletion = functools.partial(self._core_plugin.delete_port, context,
                                      l3_port_check=False)
-        port, bindings = common_db_mixin.safe_creation(context, creation,
+        port, bindings = db_utils.safe_creation(context, creation,
-                                                       deletion, content,
+                                                deletion, content,
-                                                       transaction=False)
+                                                transaction=False)
         return bindings
 
     def _create_ha_interfaces(self, context, router, ha_network):

neutron/db/metering/metering_db.py

@@ -19,6 +19,7 @@ from sqlalchemy import orm
 from neutron.api.rpc.agentnotifiers import metering_rpc_agent_api
 from neutron.common import _deprecate
 from neutron.common import constants
+from neutron.db import _utils as db_utils
 from neutron.db import common_db_mixin as base_db
 from neutron.db.models import l3 as l3_models
 from neutron.db.models import metering as metering_models
@@ -35,13 +36,14 @@ class MeteringDbMixin(metering.MeteringPluginBase,
     def __init__(self):
         self.meter_rpc = metering_rpc_agent_api.MeteringAgentNotifyAPI()
 
-    def _make_metering_label_dict(self, metering_label, fields=None):
+    @staticmethod
+    def _make_metering_label_dict(metering_label, fields=None):
         res = {'id': metering_label['id'],
                'name': metering_label['name'],
                'description': metering_label['description'],
                'shared': metering_label['shared'],
                'tenant_id': metering_label['tenant_id']}
-        return self._fields(res, fields)
+        return db_utils.resource_fields(res, fields)
 
     def create_metering_label(self, context, metering_label):
         m = metering_label['metering_label']
@@ -91,13 +93,14 @@ class MeteringDbMixin(metering.MeteringPluginBase,
                                     marker_obj=marker_obj,
                                     page_reverse=page_reverse)
 
-    def _make_metering_label_rule_dict(self, metering_label_rule, fields=None):
+    @staticmethod
+    def _make_metering_label_rule_dict(metering_label_rule, fields=None):
         res = {'id': metering_label_rule['id'],
                'metering_label_id': metering_label_rule['metering_label_id'],
                'direction': metering_label_rule['direction'],
                'remote_ip_prefix': metering_label_rule['remote_ip_prefix'],
                'excluded': metering_label_rule['excluded']}
-        return self._fields(res, fields)
+        return db_utils.resource_fields(res, fields)
 
     def get_metering_label_rules(self, context, filters=None, fields=None,
                                  sorts=None, limit=None, marker=None,

neutron/db/portsecurity_db_common.py

@@ -12,6 +12,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from neutron.db import _utils as db_utils
 from neutron.extensions import portsecurity as psec
 from neutron.objects import network
 from neutron.objects.port.extensions import port_security as p_ps
@@ -91,7 +92,8 @@ class PortSecurityDbCommon(object):
             self._process_port_security_create(
                 context, obj_cls, res_name, req, res)
 
-    def _make_port_security_dict(self, res, res_name, fields=None):
+    @staticmethod
+    def _make_port_security_dict(res, res_name, fields=None):
         res_ = {'%s_id' % res_name: res.id,
                 psec.PORTSECURITY: res.port_security_enabled}
-        return self._fields(res_, fields)
+        return db_utils.resource_fields(res_, fields)

neutron/db/qos/api.py

@@ -14,7 +14,7 @@ from oslo_db import exception as oslo_db_exception
 from sqlalchemy.orm import exc as orm_exc
 
 from neutron.common import exceptions as n_exc
-from neutron.db import common_db_mixin as db
+from neutron.db import _utils as db_utils
 from neutron.db.qos import models
 
 
@@ -32,8 +32,8 @@ def create_policy_network_binding(context, policy_id, network_id):
 def delete_policy_network_binding(context, policy_id, network_id):
     try:
         with context.session.begin(subtransactions=True):
-            db_object = (db.model_query(context,
+            db_object = (db_utils.model_query(context,
-                                        models.QosNetworkPolicyBinding)
+                                              models.QosNetworkPolicyBinding)
                          .filter_by(policy_id=policy_id,
                                     network_id=network_id).one())
             context.session.delete(db_object)
@@ -43,7 +43,7 @@ def delete_policy_network_binding(context, policy_id, network_id):
 
 
 def get_network_ids_by_network_policy_binding(context, policy_id):
-    query = (db.model_query(context, models.QosNetworkPolicyBinding)
+    query = (db_utils.model_query(context, models.QosNetworkPolicyBinding)
             .filter_by(policy_id=policy_id).all())
     return [entry.network_id for entry in query]
 
@@ -62,7 +62,8 @@ def create_policy_port_binding(context, policy_id, port_id):
 def delete_policy_port_binding(context, policy_id, port_id):
     try:
         with context.session.begin(subtransactions=True):
-            db_object = (db.model_query(context, models.QosPortPolicyBinding)
+            db_object = (db_utils.model_query(context,
+                                              models.QosPortPolicyBinding)
                          .filter_by(policy_id=policy_id,
                                     port_id=port_id).one())
             context.session.delete(db_object)
@@ -72,6 +73,6 @@ def delete_policy_port_binding(context, policy_id, port_id):
 
 
 def get_port_ids_by_port_policy_binding(context, policy_id):
-    query = (db.model_query(context, models.QosPortPolicyBinding)
+    query = (db_utils.model_query(context, models.QosPortPolicyBinding)
             .filter_by(policy_id=policy_id).all())
     return [entry.port_id for entry in query]

neutron/db/quota/api.py

@@ -19,8 +19,8 @@ import sqlalchemy as sa
 from sqlalchemy.orm import exc as orm_exc
 from sqlalchemy import sql
 
+from neutron.db import _utils as db_utils
 from neutron.db import api as db_api
-from neutron.db import common_db_mixin as common_db_api
 from neutron.db.quota import models as quota_models
 
 
@@ -52,7 +52,7 @@ def get_quota_usage_by_resource_and_tenant(context, resource, tenant_id,
     :returns: a QuotaUsageInfo instance
     """
 
-    query = common_db_api.model_query(context, quota_models.QuotaUsage)
+    query = db_utils.model_query(context, quota_models.QuotaUsage)
     query = query.filter_by(resource=resource, tenant_id=tenant_id)
 
     if lock_for_update:
@@ -69,7 +69,7 @@ def get_quota_usage_by_resource_and_tenant(context, resource, tenant_id,
 
 @db_api.retry_if_session_inactive()
 def get_quota_usage_by_resource(context, resource):
-    query = common_db_api.model_query(context, quota_models.QuotaUsage)
+    query = db_utils.model_query(context, quota_models.QuotaUsage)
     query = query.filter_by(resource=resource)
     return [QuotaUsageInfo(item.resource,
                            item.tenant_id,
@@ -79,7 +79,7 @@ def get_quota_usage_by_resource(context, resource):
 
 @db_api.retry_if_session_inactive()
 def get_quota_usage_by_tenant_id(context, tenant_id):
-    query = common_db_api.model_query(context, quota_models.QuotaUsage)
+    query = db_utils.model_query(context, quota_models.QuotaUsage)
     query = query.filter_by(tenant_id=tenant_id)
     return [QuotaUsageInfo(item.resource,
                            item.tenant_id,
@@ -102,7 +102,7 @@ def set_quota_usage(context, resource, tenant_id,
                   or a delta (default to False)
     """
     with db_api.autonested_transaction(context.session):
-        query = common_db_api.model_query(context, quota_models.QuotaUsage)
+        query = db_utils.model_query(context, quota_models.QuotaUsage)
         query = query.filter_by(resource=resource).filter_by(
             tenant_id=tenant_id)
         usage_data = query.first()
@@ -135,7 +135,7 @@ def set_quota_usage_dirty(context, resource, tenant_id, dirty=True):
     :param dirty: the desired value for the dirty bit (defaults to True)
     :returns: 1 if the quota usage data were updated, 0 otherwise.
     """
-    query = common_db_api.model_query(context, quota_models.QuotaUsage)
+    query = db_utils.model_query(context, quota_models.QuotaUsage)
     query = query.filter_by(resource=resource).filter_by(tenant_id=tenant_id)
     return query.update({'dirty': dirty})
 
@@ -151,7 +151,7 @@ def set_resources_quota_usage_dirty(context, resources, tenant_id, dirty=True):
     :param dirty: the desired value for the dirty bit (defaults to True)
     :returns: the number of records for which the bit was actually set.
     """
-    query = common_db_api.model_query(context, quota_models.QuotaUsage)
+    query = db_utils.model_query(context, quota_models.QuotaUsage)
     query = query.filter_by(tenant_id=tenant_id)
     if resources:
         query = query.filter(quota_models.QuotaUsage.resource.in_(resources))
@@ -168,7 +168,7 @@ def set_all_quota_usage_dirty(context, resource, dirty=True):
     :returns: the number of tenants for which the dirty bit was
               actually updated
     """
-    query = common_db_api.model_query(context, quota_models.QuotaUsage)
+    query = db_utils.model_query(context, quota_models.QuotaUsage)
     query = query.filter_by(resource=resource)
     return query.update({'dirty': dirty})
 

neutron/db/quota/driver.py

@@ -17,8 +17,8 @@ from neutron_lib import exceptions
 from oslo_log import log
 
 from neutron.common import exceptions as n_exc
+from neutron.db import _utils as db_utils
 from neutron.db import api as db_api
-from neutron.db import common_db_mixin as common_db
 from neutron.db.quota import api as quota_api
 from neutron.db.quota import models as quota_models
 
@@ -66,7 +66,7 @@ class DbQuotaDriver(object):
                             for key, resource in resources.items())
 
         # update with tenant specific limits
-        q_qry = common_db.model_query(context, quota_models.Quota).filter_by(
+        q_qry = db_utils.model_query(context, quota_models.Quota).filter_by(
             tenant_id=tenant_id)
         for item in q_qry:
             tenant_quota[item['resource']] = item['limit']

neutron/db/rbac_db_mixin.py

@@ -20,6 +20,7 @@ from sqlalchemy.orm import exc
 from neutron.callbacks import events
 from neutron.callbacks import exceptions as c_exc
 from neutron.callbacks import registry
+from neutron.db import _utils as db_utils
 from neutron.db import api as db_api
 from neutron.db import common_db_mixin
 from neutron.db import rbac_db_models as models
@@ -56,11 +57,12 @@ class RbacPluginMixin(common_db_mixin.CommonDbMixin):
             raise ext_rbac.DuplicateRbacPolicy()
         return self._make_rbac_policy_dict(db_entry)
 
-    def _make_rbac_policy_dict(self, db_entry, fields=None):
+    @staticmethod
+    def _make_rbac_policy_dict(db_entry, fields=None):
         res = {f: db_entry[f] for f in ('id', 'tenant_id', 'target_tenant',
                                         'action', 'object_id')}
         res['object_type'] = db_entry.object_type
-        return self._fields(res, fields)
+        return db_utils.resource_fields(res, fields)
 
     @db_api.retry_if_session_inactive()
     def update_rbac_policy(self, context, id, rbac_policy):

neutron/db/securitygroups_db.py

@@ -29,6 +29,7 @@ from neutron.callbacks import resources
 from neutron.common import _deprecate
 from neutron.common import constants as n_const
 from neutron.common import utils
+from neutron.db import _utils as db_utils
 from neutron.db import api as db_api
 from neutron.db import db_base_plugin_v2
 from neutron.db.models import securitygroup as sg_models
@@ -256,12 +257,13 @@ class SecurityGroupDbMixin(ext_sg.SecurityGroupPluginBase):
                                        for r in security_group.rules]
         self._apply_dict_extend_functions(ext_sg.SECURITYGROUPS, res,
                                           security_group)
-        return self._fields(res, fields)
+        return db_utils.resource_fields(res, fields)
 
-    def _make_security_group_binding_dict(self, security_group, fields=None):
+    @staticmethod
+    def _make_security_group_binding_dict(security_group, fields=None):
         res = {'port_id': security_group['port_id'],
                'security_group_id': security_group['security_group_id']}
-        return self._fields(res, fields)
+        return db_utils.resource_fields(res, fields)
 
     @db_api.retry_if_session_inactive()
     def _create_port_security_group_binding(self, context, port_id,
@@ -479,7 +481,7 @@ class SecurityGroupDbMixin(ext_sg.SecurityGroupPluginBase):
 
         self._apply_dict_extend_functions(ext_sg.SECURITYGROUPRULES, res,
                                           security_group_rule)
-        return self._fields(res, fields)
+        return db_utils.resource_fields(res, fields)
 
     def _make_security_group_rule_filter_dict(self, security_group_rule):
         sgr = security_group_rule['security_group_rule']

neutron/objects/rbac_db.py

@@ -24,8 +24,8 @@ from neutron._i18n import _
 from neutron.callbacks import events
 from neutron.callbacks import registry
 from neutron.common import exceptions as n_exc
+from neutron.db import _utils as db_utils
 from neutron.db import api as db_api
-from neutron.db import common_db_mixin
 from neutron.db import rbac_db_mixin
 from neutron.db import rbac_db_models as models
 from neutron.extensions import rbac as ext_rbac
@@ -68,7 +68,7 @@ class RbacNeutronDbObjectMixin(rbac_db_mixin.RbacPluginMixin,
     def get_shared_with_tenant(context, rbac_db_model, obj_id, tenant_id):
         # NOTE(korzen) This method enables to query within already started
         # session
-        return (common_db_mixin.model_query(context, rbac_db_model).filter(
+        return (db_utils.model_query(context, rbac_db_model).filter(
                 and_(rbac_db_model.object_id == obj_id,
                      rbac_db_model.action == models.ACCESS_SHARED,
                      rbac_db_model.target_tenant.in_(
@@ -92,7 +92,7 @@ class RbacNeutronDbObjectMixin(rbac_db_mixin.RbacPluginMixin,
     @classmethod
     def _get_db_obj_rbac_entries(cls, context, rbac_obj_id, rbac_action):
         rbac_db_model = cls.rbac_db_model
-        return common_db_mixin.model_query(context, rbac_db_model).filter(
+        return db_utils.model_query(context, rbac_db_model).filter(
             and_(rbac_db_model.object_id == rbac_obj_id,
                  rbac_db_model.action == rbac_action))
 

neutron/plugins/ml2/plugin.py

@@ -45,6 +45,7 @@ from neutron.common import ipv6_utils
 from neutron.common import rpc as n_rpc
 from neutron.common import topics
 from neutron.common import utils
+from neutron.db import _utils as db_utils
 from neutron.db import address_scope_db
 from neutron.db import agents_db
 from neutron.db import agentschedulers_db
@@ -844,7 +845,7 @@ class Ml2Plugin(db_base_plugin_v2.NeutronDbPluginV2,
             self.type_manager.extend_network_dict_provider(context, result)
             result[api.MTU] = self._get_network_mtu(result)
 
-        return self._fields(result, fields)
+        return db_utils.resource_fields(result, fields)
 
     @db_api.retry_if_session_inactive()
     def get_networks(self, context, filters=None, fields=None,
@@ -861,7 +862,7 @@ class Ml2Plugin(db_base_plugin_v2.NeutronDbPluginV2,
             for net in nets:
                 net[api.MTU] = self._get_network_mtu(net)
 
-        return [self._fields(net, fields) for net in nets]
+        return [db_utils.resource_fields(net, fields) for net in nets]
 
     def _delete_ports(self, context, port_ids):
         for port_id in port_ids:

neutron/services/auto_allocate/db.py

@@ -25,6 +25,7 @@ from neutron.callbacks import events
 from neutron.callbacks import registry
 from neutron.callbacks import resources
 from neutron.common import exceptions as c_exc
+from neutron.db import _utils as db_utils
 from neutron.db import api as db_api
 from neutron.db import common_db_mixin
 from neutron.db import db_base_plugin_v2
@@ -208,13 +209,14 @@ class AutoAllocatedTopologyMixin(common_db_mixin.CommonDbMixin):
         if network:
             return network['network_id']
 
-    def _response(self, network_id, tenant_id, fields=None):
+    @staticmethod
+    def _response(network_id, tenant_id, fields=None):
         """Build response for auto-allocated network."""
         res = {
             'id': network_id,
             'tenant_id': tenant_id
         }
-        return self._fields(res, fields)
+        return db_utils.resource_fields(res, fields)
 
     def _get_default_external_network(self, context):
         """Get the default external network for the deployment."""

neutron/services/provider_configuration.py

@@ -26,6 +26,7 @@ import stevedore
 
 from neutron._i18n import _, _LW
 from neutron.api.v2 import attributes as attr
+from neutron.db import _utils as db_utils
 
 LOG = logging.getLogger(__name__)
 
@@ -265,17 +266,11 @@ class ProviderConfiguration(object):
                     return False
         return True
 
-    def _fields(self, resource, fields):
-        if fields:
-            return dict(((key, item) for key, item in resource.items()
-                         if key in fields))
-        return resource
-
     def get_service_providers(self, filters=None, fields=None):
-        return [self._fields({'service_type': k[0],
+        return [db_utils.resource_fields({'service_type': k[0],
-                              'name': k[1],
+                                          'name': k[1],
-                              'driver': v['driver'],
+                                          'driver': v['driver'],
-                              'default': v['default']},
+                                          'default': v['default']},
-                             fields)
+                                         fields)
                 for k, v in self.providers.items()
                 if self._check_entry(k, v, filters)]

neutron/services/segments/db.py

@@ -28,6 +28,7 @@ from neutron.callbacks import events
 from neutron.callbacks import registry
 from neutron.callbacks import resources
 from neutron.common import _deprecate
+from neutron.db import _utils as db_utils
 from neutron.db import api as db_api
 from neutron.db import common_db_mixin
 from neutron.db.models import segment as segment_model
@@ -43,7 +44,8 @@ _deprecate._moved_global('SegmentHostMapping', new_module=segment_model)
 class SegmentDbMixin(common_db_mixin.CommonDbMixin):
     """Mixin class to add segment."""
 
-    def _make_segment_dict(self, segment_db, fields=None):
+    @staticmethod
+    def _make_segment_dict(segment_db, fields=None):
         res = {'id': segment_db['id'],
                'network_id': segment_db['network_id'],
                'name': segment_db['name'],
@@ -54,7 +56,7 @@ class SegmentDbMixin(common_db_mixin.CommonDbMixin):
                'hosts': [mapping.host for mapping in
                          segment_db.segment_host_mapping],
                'segment_index': segment_db['segment_index']}
-        return self._fields(res, fields)
+        return db_utils.resource_fields(res, fields)
 
     def _get_segment(self, context, segment_id):
         try:

neutron/tests/unit/db/test_common_db_mixin.py

@@ -16,7 +16,7 @@
 import mock
 
 from neutron import context
-from neutron.db import common_db_mixin
+from neutron.db import _utils as db_utils
 from neutron.tests.unit import testlib_api
 
 
@@ -32,7 +32,7 @@ class TestCommonHelpFunctions(testlib_api.SqlTestCase):
         tx_check = lambda i: setattr(self, '_active',
                                      self.admin_ctx.session.is_active)
         delete_fn = mock.Mock(side_effect=tx_check)
-        self.assertRaises(ValueError, common_db_mixin.safe_creation,
+        self.assertRaises(ValueError, db_utils.safe_creation,
                           self.admin_ctx, create_fn, delete_fn,
                           create_bindings)
         delete_fn.assert_called_once_with(1234)
@@ -42,7 +42,7 @@ class TestCommonHelpFunctions(testlib_api.SqlTestCase):
         create_fn = mock.Mock(return_value={'id': 1234})
         create_bindings = mock.Mock(side_effect=ValueError)
         delete_fn = mock.Mock(side_effect=EnvironmentError)
-        self.assertRaises(ValueError, common_db_mixin.safe_creation,
+        self.assertRaises(ValueError, db_utils.safe_creation,
                           self.admin_ctx, create_fn, delete_fn,
                           create_bindings)
         delete_fn.assert_called_once_with(1234)

neutron/tests/unit/extensions/test_portsecurity.py

@@ -17,6 +17,7 @@ from neutron_lib.api import validators
 from webob import exc
 
 from neutron import context
+from neutron.db import _utils as db_utils
 from neutron.db import db_base_plugin_v2
 from neutron.db import portsecurity_db
 from neutron.db import securitygroups_db
@@ -83,7 +84,7 @@ class PortSecurityTestPlugin(db_base_plugin_v2.NeutronDbPluginV2,
         with context.session.begin(subtransactions=True):
             net = super(PortSecurityTestPlugin, self).get_network(
                 context, id)
-        return self._fields(net, fields)
+        return db_utils.resource_fields(net, fields)
 
     def create_port(self, context, port):
         p = port['port']