Merge "objects: loading synthetic fields from defined ORM relationships."

This commit is contained in:
Jenkins 2016-07-28 14:00:38 +00:00 committed by Gerrit Code Review
commit 8a95d7b0f0
10 changed files with 188 additions and 52 deletions

@ -270,6 +270,11 @@ class NeutronDbObject(NeutronObject):
fields_no_update = []
# dict with name mapping: {'field_name_in_object': 'field_name_in_db'}
# It can be used also as DB relationship mapping to synthetic fields name.
# It is needed to load synthetic fields with one SQL query using side
# loaded entities.
# Examples: {'synthetic_field_name': 'relationship_name_in_model'}
# {'field_name_in_object': 'field_name_in_db'}
fields_need_translation = {}
# obj_extra_fields defines properties that are not part of the model
@ -286,7 +291,8 @@ class NeutronDbObject(NeutronObject):
if field in db_obj and not self.is_synthetic(field):
setattr(self, field, db_obj[field])
break
self.load_synthetic_db_fields()
for obj in objs:
self.load_synthetic_db_fields(obj)
self.obj_reset_changes()
@classmethod
@ -426,7 +432,7 @@ class NeutronDbObject(NeutronObject):
return fields
def load_synthetic_db_fields(self):
def load_synthetic_db_fields(self, db_obj=None):
"""
Load the synthetic fields that are stored in a different table from the
main object.
@ -453,14 +459,29 @@ class NeutronDbObject(NeutronObject):
objclass = objclasses[0]
if len(objclass.foreign_keys.keys()) > 1:
raise NeutronSyntheticFieldMultipleForeignKeys(field=field)
objs = objclass.get_objects(
self.obj_context, **{
k: getattr(
self, v) for k, v in objclass.foreign_keys.items()})
if isinstance(self.fields[field], obj_fields.ObjectField):
setattr(self, field, objs[0] if objs else None)
synthetic_field_db_name = (
self.fields_need_translation.get(field, field))
synth_db_objs = (db_obj.get(synthetic_field_db_name, None)
if db_obj else None)
# synth_db_objs can be list, empty list or None, that is why
# we need 'is not None', because [] is valid case for 'True'
if synth_db_objs is not None:
if not isinstance(synth_db_objs, list):
synth_db_objs = [synth_db_objs]
synth_objs = [objclass._load_object(self.obj_context,
objclass.modify_fields_from_db(obj))
for obj in synth_db_objs]
else:
setattr(self, field, objs)
synth_objs = objclass.get_objects(
self.obj_context, **{
k: getattr(self, v)
for k, v in objclass.foreign_keys.items()})
if isinstance(self.fields[field], obj_fields.ObjectField):
setattr(self, field, synth_objs[0] if synth_objs else None)
else:
setattr(self, field, synth_objs)
self.obj_reset_changes([field])
def create(self):

@ -175,7 +175,8 @@ class Subnet(base.NeutronDbObject):
foreign_keys = {'network_id': 'id'}
fields_need_translation = {
'project_id': 'tenant_id'
'project_id': 'tenant_id',
'host_routes': 'routes'
}
def __init__(self, context=None, **kwargs):

@ -30,8 +30,7 @@ class NetworkPortSecurityDbObjTestCase(obj_test_base.BaseDbObjectTestCase,
def setUp(self):
super(NetworkPortSecurityDbObjTestCase, self).setUp()
self._create_test_network()
for obj in self.db_objs:
obj['network_id'] = self._network['id']
for obj in self.obj_fields:
obj['id'] = self._network['id']
for db_obj, obj_field in zip(self.db_objs, self.obj_fields):
network = self._create_network()
db_obj['network_id'] = network['id']
obj_field['id'] = network['id']

@ -30,8 +30,6 @@ class PortSecurityDbObjTestCase(obj_test_base.BaseDbObjectTestCase,
def setUp(self):
super(PortSecurityDbObjTestCase, self).setUp()
self._create_test_network()
self._create_test_port(self._network)
for obj in self.db_objs:
obj['port_id'] = self._port['id']
for obj in self.obj_fields:
obj['id'] = self._port['id']
self._create_port(id=obj['port_id'],
network_id=self._network['id'])

@ -414,3 +414,9 @@ class QosPolicyDbObjectTestCase(test_base.BaseDbObjectTestCase,
self.context, shared=False)
self.assertEqual(1, len(private_policies))
self.assertEqual('private-policy', private_policies[0].name)
def test_get_objects_queries_constant(self):
# NOTE(korzen) QoSPolicy is using extra queries to reload rules.
# QoSPolicy currently cannot be loaded using constant queries number.
# It can be reworked in follow-up patch.
pass

@ -83,10 +83,11 @@ class QosBandwidthLimitRuleDbObjectTestCase(test_base.BaseDbObjectTestCase,
super(QosBandwidthLimitRuleDbObjectTestCase, self).setUp()
# Prepare policy to be able to insert a rule
generated_qos_policy_id = self.db_obj['qos_policy_id']
policy_obj = policy.QosPolicy(self.context,
id=generated_qos_policy_id)
policy_obj.create()
for obj in self.db_objs:
generated_qos_policy_id = obj['qos_policy_id']
policy_obj = policy.QosPolicy(self.context,
id=generated_qos_policy_id)
policy_obj.create()
class QosDscpMarkingRuleObjectTestCase(test_base.BaseObjectIfaceTestCase):
@ -115,7 +116,8 @@ class QosDscpMarkingRuleDbObjectTestCase(test_base.BaseDbObjectTestCase,
def setUp(self):
super(QosDscpMarkingRuleDbObjectTestCase, self).setUp()
# Prepare policy to be able to insert a rule
generated_qos_policy_id = self.db_obj['qos_policy_id']
policy_obj = policy.QosPolicy(self.context,
id=generated_qos_policy_id)
policy_obj.create()
for obj in self.db_objs:
generated_qos_policy_id = obj['qos_policy_id']
policy_obj = policy.QosPolicy(self.context,
id=generated_qos_policy_id)
policy_obj.create()

@ -39,6 +39,7 @@ from neutron.objects.db import api as obj_db_api
from neutron.objects import subnet
from neutron.tests import base as test_base
from neutron.tests import tools
from neutron.tests.unit.db import test_db_base_plugin_v2
SQLALCHEMY_COMMIT = 'sqlalchemy.engine.Connection._commit_impl'
@ -311,6 +312,11 @@ def remove_timestamps_from_fields(obj_fields):
if field not in TIMESTAMP_FIELDS}
def get_non_synthetic_fields(objclass, obj_fields):
return {field: value for field, value in obj_fields.items()
if not objclass.is_synthetic(field)}
class _BaseObjectTestCase(object):
_test_class = FakeNeutronObject
@ -375,6 +381,20 @@ class _BaseObjectTestCase(object):
def fake_get_objects(self, context, model, **kwargs):
return self.model_map[model]
def _get_object_synthetic_fields(self, objclass):
return [field for field in objclass.synthetic_fields
if objclass.is_object_field(field)]
def _get_ovo_object_class(self, objclass, field):
try:
name = objclass.fields[field].objname
return obj_base.VersionedObjectRegistry.obj_classes().get(name)[0]
except TypeError:
# NOTE(korzen) some synthetic fields are not handled by
# this method, for example the ones that have subclasses, see
# QosRule
return
class BaseObjectIfaceTestCase(_BaseObjectTestCase, test_base.BaseTestCase):
@ -441,9 +461,8 @@ class BaseObjectIfaceTestCase(_BaseObjectTestCase, test_base.BaseTestCase):
for db_obj in db_objs:
for field in self._test_class.synthetic_fields:
if self._test_class.is_object_field(field):
obj_class = obj_base.VersionedObjectRegistry.obj_classes(
).get(self._test_class.fields[
field].objname)[0]
obj_class = self._get_ovo_object_class(self._test_class,
field)
mock_calls.append(
mock.call(
self.context, obj_class.db_model,
@ -707,24 +726,17 @@ class BaseObjectIfaceTestCase(_BaseObjectTestCase, test_base.BaseTestCase):
def test_to_dict_synthetic_fields(self):
cls_ = self._test_class
object_fields = [
field
for field in cls_.synthetic_fields
if cls_.is_object_field(field)
]
object_fields = self._get_object_synthetic_fields(cls_)
if not object_fields:
self.skipTest(
'No object fields found in test class %r' % cls_)
for field in object_fields:
obj = cls_(self.context, **self.obj_fields[0])
objclasses = obj_base.VersionedObjectRegistry.obj_classes(
).get(cls_.fields[field].objname)
if not objclasses:
# NOTE(ihrachys): this test does not handle fields of types
# that are not registered (for example, QosRule)
objclass = self._get_ovo_object_class(cls_, field)
if not objclass:
continue
objclass = objclasses[0]
child = objclass(
self.context, **objclass.modify_fields_from_db(
self.get_random_fields(obj_cls=objclass))
@ -832,10 +844,20 @@ class BaseDbObjectMultipleForeignKeysTestCase(_BaseObjectTestCase,
obj.load_synthetic_db_fields)
class BaseDbObjectTestCase(_BaseObjectTestCase):
class BaseDbObjectTestCase(_BaseObjectTestCase,
test_db_base_plugin_v2.DbOperationBoundMixin):
def setUp(self):
super(BaseDbObjectTestCase, self).setUp()
self.useFixture(tools.CommonDbMixinHooksFixture())
synthetic_fields = self._get_object_synthetic_fields(self._test_class)
for synth_field in synthetic_fields:
objclass = self._get_ovo_object_class(self._test_class,
synth_field)
if not objclass:
continue
for db_obj in self.db_objs:
objclass_fields = self.get_random_fields(objclass)
db_obj[synth_field] = [objclass_fields]
def _create_test_network(self):
# TODO(ihrachys): replace with network.create() once we get an object
@ -844,6 +866,12 @@ class BaseDbObjectTestCase(_BaseObjectTestCase):
models_v2.Network,
{'name': 'test-network1'})
def _create_network(self):
name = "test-network-%s" % tools.get_random_string(4)
return obj_db_api.create_object(self.context,
models_v2.Network,
{'name': name})
def _create_test_subnet(self, network):
test_subnet = {
'project_id': uuidutils.generate_uuid(),
@ -899,6 +927,7 @@ class BaseDbObjectTestCase(_BaseObjectTestCase):
self._port = self._create_port(network_id=network['id'])
def _make_object(self, fields):
fields = get_non_synthetic_fields(self._test_class, fields)
return self._test_class(
self.context, **remove_timestamps_from_fields(fields))
@ -1017,6 +1046,75 @@ class BaseDbObjectTestCase(_BaseObjectTestCase):
new = self._test_class.get_objects(self.context, **filters)
self.assertEqual([obj], new)
def _get_non_synth_fields(self, objclass, db_attrs):
fields = objclass.modify_fields_from_db(db_attrs)
fields = remove_timestamps_from_fields(fields)
fields = get_non_synthetic_fields(objclass, fields)
return fields
def _create_object_with_synthetic_fields(self, db_obj):
cls_ = self._test_class
object_fields = self._get_object_synthetic_fields(cls_)
# create base object
obj = cls_(self.context, **self._get_non_synth_fields(cls_, db_obj))
obj.create()
# create objects that are going to be loaded into the base object
# through synthetic fields
for field in object_fields:
objclass = self._get_ovo_object_class(cls_, field)
if not objclass:
continue
objclass_fields = self._get_non_synth_fields(objclass,
db_obj[field][0])
# make sure children point to the base object
for local_field, foreign_key in objclass.foreign_keys.items():
objclass_fields[local_field] = obj.get(foreign_key)
synth_field_obj = objclass(self.context, **objclass_fields)
synth_field_obj.create()
# populate the base object synthetic fields with created children
if isinstance(cls_.fields[field], obj_fields.ObjectField):
setattr(obj, field, synth_field_obj)
else:
setattr(obj, field, [synth_field_obj])
# reset the object so that we can compare it to other clean objects
obj.obj_reset_changes([field])
return obj
def test_get_object_with_synthetic_fields(self):
object_fields = self._get_object_synthetic_fields(self._test_class)
if not object_fields:
self.skipTest(
'No synthetic object fields found '
'in test class %r' % self._test_class
)
obj = self._create_object_with_synthetic_fields(self.db_objs[0])
listed_obj = self._test_class.get_object(
self.context, **obj._get_composite_keys())
self.assertTrue(listed_obj)
self.assertEqual(obj, listed_obj)
# NOTE(korzen) _list method is used in neutron.tests.db.unit.db.
# test_db_base_plugin_v2.DbOperationBoundMixin in _list_and_count_queries()
# This is used in test_subnet for asserting that number of queries is
# constant. It can be used also for port and network objects when ready.
def _list(self, resource, neutron_context):
cls_ = resource
return cls_.get_objects(neutron_context)
def test_get_objects_queries_constant(self):
iter_db_obj = iter(self.db_objs)
def _create():
self._create_object_with_synthetic_fields(next(iter_db_obj))
self._assert_object_list_queries_constant(_create, self._test_class)
class UniqueObjectBase(test_base.BaseTestCase):
def setUp(self):

@ -16,12 +16,10 @@ from operator import itemgetter
from oslo_utils import uuidutils
from neutron import context
from neutron.db import models_v2
from neutron.db import rbac_db_models
from neutron.objects import base as obj_base
from neutron.objects.db import api as obj_db_api
from neutron.objects import subnet
from neutron.tests import tools
from neutron.tests.unit.objects import test_base as obj_test_base
from neutron.tests.unit import testlib_api
@ -172,12 +170,6 @@ class SubnetDbObjectTestCase(obj_test_base.BaseDbObjectTestCase,
self.assertEqual(2, new.dns_nameservers[1].order)
self.assertEqual(4, new.dns_nameservers[-1].order)
def _create_network(self):
name = "test-network-%s" % tools.get_random_string(4)
return obj_db_api.create_object(self.context,
models_v2.Network,
{'name': name})
def _create_shared_network_rbac_entry(self, network):
attrs = {
'object_id': network['id'],

@ -62,6 +62,14 @@ class SubnetPoolDbObjectTestCase(obj_test_base.BaseDbObjectTestCase,
pool = self._test_class.get_object(self.context, id=self._pool.id)
self.assertItemsEqual(prefixes, pool.prefixes)
def test_get_objects_queries_constant(self):
# TODO(korzen) SubnetPool is using SubnetPoolPrefix object to reload
# prefixes, which costs extra SQL query each time reload_prefixes
# are called in get_object(s). SubnetPool has defined relationship
# for SubnetPoolPrefixes, so it should be possible to reuse side loaded
# values fo this. To be reworked in follow-up patch.
pass
class SubnetPoolPrefixIfaceObjectTestCase(
obj_test_base.BaseObjectIfaceTestCase):

@ -13,8 +13,9 @@
# License for the specific language governing permissions and limitations
# under the License.
import mock
import itertools
import mock
from neutron_lib import exceptions as n_exc
from oslo_db import exception as obj_exc
from oslo_utils import uuidutils
@ -85,8 +86,11 @@ class TrunkDbObjectTestCase(test_base.BaseDbObjectTestCase,
super(TrunkDbObjectTestCase, self).setUp()
self._create_test_network()
sub_ports = []
for obj in self.db_objs:
sub_ports.extend(obj['sub_ports'])
for obj in self.obj_fields:
for obj in itertools.chain(self.obj_fields, sub_ports):
self._create_port(id=obj['port_id'],
network_id=self._network['id'])
@ -121,3 +125,10 @@ class TrunkDbObjectTestCase(test_base.BaseDbObjectTestCase,
self.assertEqual(expected, set(map(_as_tuple, trunk.sub_ports)))
self.assertEqual(expected, set(map(_as_tuple, sub_ports)))
def test_get_objects_queries_constant(self):
# NOTE(korzen) Trunk object has synthetic field 'sub_port' which is
# not defined as model relationship between sub_port and trunk.
# Both sub_port and trunk have relationship to port model, but this
# relationship is not proactively loading the port object.
pass