Merge "Add DictOfMiscValuesField in OVO for dict usage"

neutron/objects/agent.py

@@ -12,7 +12,6 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-from oslo_serialization import jsonutils
 from oslo_versionedobjects import base as obj_base
 from oslo_versionedobjects import fields as obj_fields
 
@@ -41,8 +40,8 @@ class Agent(base.NeutronDbObject):
         'created_at': obj_fields.DateTimeField(tzinfo_aware=False),
         'heartbeat_timestamp': obj_fields.DateTimeField(tzinfo_aware=False),
         'description': obj_fields.StringField(nullable=True),
-        'configurations': obj_fields.DictOfStringsField(),
+        'configurations': common_types.DictOfMiscValuesField(),
-        'resource_versions': obj_fields.DictOfStringsField(nullable=True),
+        'resource_versions': common_types.DictOfMiscValuesField(nullable=True),
         'load': obj_fields.IntegerField(default=0),
     }
 
@@ -50,31 +49,26 @@ class Agent(base.NeutronDbObject):
     def modify_fields_to_db(cls, fields):
         result = super(Agent, cls).modify_fields_to_db(fields)
         if 'configurations' in result:
+            # dump configuration into string, set '' if empty '{}'
             result['configurations'] = (
-                cls.filter_to_json_str(result['configurations']))
+                cls.filter_to_json_str(result['configurations'], default=''))
         if 'resource_versions' in result:
-            if result['resource_versions']:
+            # dump resource version into string, set None if empty '{}' or None
-                result['resource_versions'] = (
+            result['resource_versions'] = (
-                    cls.filter_to_json_str(result['resource_versions']))
+                cls.filter_to_json_str(result['resource_versions']))
-            if not fields['resource_versions']:
-                result['resource_versions'] = None
         return result
 
     @classmethod
     def modify_fields_from_db(cls, db_obj):
         fields = super(Agent, cls).modify_fields_from_db(db_obj)
         if 'configurations' in fields:
-            if fields['configurations']:
+            # load string from DB, set {} if configuration is ''
-                fields['configurations'] = jsonutils.loads(
+            fields['configurations'] = (
-                    fields['configurations'])
+                cls.load_json_from_str(fields['configurations'], default={}))
-            if not fields['configurations']:
-                fields['configurations'] = {}
         if 'resource_versions' in fields:
-            if fields['resource_versions']:
+            # load string from DB, set None if resource_version is None or ''
-                fields['resource_versions'] = jsonutils.loads(
+            fields['resource_versions'] = (
-                    fields['resource_versions'])
+                cls.load_json_from_str(fields['resource_versions']))
-            if not fields['resource_versions']:
-                fields['resource_versions'] = None
         return fields
 
     @property

neutron/objects/base.py

@@ -471,12 +471,14 @@ class NeutronDbObject(NeutronObject):
         return str(value)
 
     @staticmethod
-    def filter_to_json_str(value):
+    def filter_to_json_str(value, default=None):
         def _dict_to_json(v):
-            return jsonutils.dumps(
+            return (
-                collections.OrderedDict(
+                jsonutils.dumps(
-                    sorted(v.items(), key=lambda t: t[0])
+                    collections.OrderedDict(
-                ) if v else {}
+                        sorted(v.items(), key=lambda t: t[0])
+                    )
+                ) if v else default
             )
 
         if isinstance(value, list):
@@ -484,6 +486,13 @@ class NeutronDbObject(NeutronObject):
         v = _dict_to_json(value)
         return v
 
+    @staticmethod
+    def load_json_from_str(field, default=None):
+        value = field or default
+        if value:
+            value = jsonutils.loads(value)
+        return value
+
     def _get_changed_persistent_fields(self):
         fields = self.obj_get_changes()
         for field in self.synthetic_fields:

neutron/objects/common_types.py

@@ -17,6 +17,7 @@ import uuid
 import netaddr
 from neutron_lib import constants as lib_constants
 
+from oslo_serialization import jsonutils
 from oslo_versionedobjects import fields as obj_fields
 import six
 
@@ -200,6 +201,43 @@ class MACAddressField(obj_fields.AutoTypedField):
     AUTO_TYPE = MACAddress()
 
 
+class DictOfMiscValues(obj_fields.FieldType):
+    """DictOfMiscValues custom field
+
+    This custom field is handling dictionary with miscellaneous value types,
+    including integer, float, boolean and list and nested dictionaries.
+    """
+    @staticmethod
+    def coerce(obj, attr, value):
+        if isinstance(value, dict):
+            return value
+        if isinstance(value, six.string_types):
+            try:
+                return jsonutils.loads(value)
+            except Exception:
+                msg = _("Field value %s is not stringified JSON") % value
+                raise ValueError(msg)
+        msg = (_("Field value %s is not type of dict or stringified JSON")
+               % value)
+        raise ValueError(msg)
+
+    @staticmethod
+    def from_primitive(obj, attr, value):
+        return DictOfMiscValues.coerce(obj, attr, value)
+
+    @staticmethod
+    def to_primitive(obj, attr, value):
+        return jsonutils.dumps(value)
+
+    @staticmethod
+    def stringify(value):
+        return jsonutils.dumps(value)
+
+
+class DictOfMiscValuesField(obj_fields.AutoTypedField):
+    AUTO_TYPE = DictOfMiscValues
+
+
 class IPNetwork(obj_fields.FieldType):
     """IPNetwork custom field.
 

neutron/objects/ports.py

@@ -13,7 +13,6 @@
 #    under the License.
 
 import netaddr
-from oslo_serialization import jsonutils
 from oslo_versionedobjects import base as obj_base
 from oslo_versionedobjects import fields as obj_fields
 
@@ -38,19 +37,24 @@ class PortBindingBase(base.NeutronDbObject):
     @classmethod
     def modify_fields_to_db(cls, fields):
         result = super(PortBindingBase, cls).modify_fields_to_db(fields)
-        if 'vif_details' in result:
+        for field in ['profile', 'vif_details']:
-            result['vif_details'] = (
+            if field in result:
-                cls.filter_to_json_str(result['vif_details']))
+                # dump field into string, set '' if empty '{}' or None
+                result[field] = (
+                    cls.filter_to_json_str(result[field], default=''))
         return result
 
     @classmethod
     def modify_fields_from_db(cls, db_obj):
         fields = super(PortBindingBase, cls).modify_fields_from_db(db_obj)
         if 'vif_details' in fields:
-            if fields['vif_details']:
+            # load string from DB into dict, set None if vif_details is ''
-                fields['vif_details'] = jsonutils.loads(fields['vif_details'])
+            fields['vif_details'] = (
-            if not fields['vif_details']:
+                cls.load_json_from_str(fields['vif_details']))
-                fields['vif_details'] = None
+        if 'profile' in fields:
+            # load string from DB into dict, set {} if profile is ''
+            fields['profile'] = (
+                cls.load_json_from_str(fields['profile'], default={}))
         return fields
 
 
@@ -64,9 +68,9 @@ class PortBinding(PortBindingBase):
     fields = {
         'port_id': common_types.UUIDField(),
         'host': obj_fields.StringField(),
-        'profile': obj_fields.StringField(),
+        'profile': common_types.DictOfMiscValuesField(),
         'vif_type': obj_fields.StringField(),
-        'vif_details': obj_fields.DictOfStringsField(nullable=True),
+        'vif_details': common_types.DictOfMiscValuesField(nullable=True),
         'vnic_type': obj_fields.StringField(),
     }
 
@@ -83,9 +87,9 @@ class DistributedPortBinding(PortBindingBase):
     fields = {
         'port_id': common_types.UUIDField(),
         'host': obj_fields.StringField(),
-        'profile': obj_fields.StringField(),
+        'profile': common_types.DictOfMiscValuesField(),
         'vif_type': obj_fields.StringField(),
-        'vif_details': obj_fields.DictOfStringsField(nullable=True),
+        'vif_details': common_types.DictOfMiscValuesField(nullable=True),
         'vnic_type': obj_fields.StringField(),
         # NOTE(ihrachys): Fields below are specific to this type of binding. In
         # the future, we could think of converging different types of bindings

neutron/tests/unit/objects/test_agent.py

@@ -34,10 +34,15 @@ class AgentDbObjectTestCase(obj_test_base.BaseDbObjectTestCase,
         obj.configurations = {}
         obj.update()
 
+        db_fields = obj.modify_fields_to_db(obj)
+        self.assertEqual('', db_fields['configurations'])
+
         obj = agent.Agent.get_object(self.context, id=obj.id)
         self.assertEqual({}, obj.configurations)
 
-        conf = {'key': 'val'}
+        conf = {"tunnel_types": ["vxlan"],
+                "tunneling_ip": "20.0.0.1",
+                "bridge_mappings": {"phys_net1": "br-eth-1"}}
         obj.configurations = conf
         obj.update()
 
@@ -46,7 +51,7 @@ class AgentDbObjectTestCase(obj_test_base.BaseDbObjectTestCase,
 
     def test_resource_versions(self):
         obj = self.objs[0]
-        versions = {'obj1': 'ver1', 'obj2': 'ver2'}
+        versions = {'obj1': 'ver1', 'obj2': 1.1}
         obj.resource_versions = versions
         obj.create()
 
@@ -56,9 +61,15 @@ class AgentDbObjectTestCase(obj_test_base.BaseDbObjectTestCase,
         obj.resource_versions = {}
         obj.update()
 
+        db_fields = obj.modify_fields_to_db(obj)
+        self.assertIsNone(db_fields['resource_versions'])
+
         obj = agent.Agent.get_object(self.context, id=obj.id)
         self.assertIsNone(obj.resource_versions)
 
         obj.resource_versions = None
         obj.update()
         self.assertIsNone(obj.resource_versions)
+
+        db_fields = obj.modify_fields_to_db(obj)
+        self.assertIsNone(db_fields['resource_versions'])

neutron/tests/unit/objects/test_base.py

@@ -360,6 +360,19 @@ class FakeNeutronObject(base.NeutronObject):
         ]
 
 
+@obj_base.VersionedObjectRegistry.register_if(False)
+class FakeNeutronObjectDictOfMiscValues(base.NeutronDbObject):
+    # Version 1.0: Initial version
+    VERSION = '1.0'
+
+    db_model = FakeModel
+
+    fields = {
+        'id': common_types.UUIDField(),
+        'dict_field': common_types.DictOfMiscValuesField(),
+    }
+
+
 def get_random_dscp_mark():
     return random.choice(constants.VALID_DSCP_MARKS)
 
@@ -387,6 +400,22 @@ def get_random_dict_of_strings():
     }
 
 
+def get_random_dict():
+    return {
+        helpers.get_random_string(6): helpers.get_random_string(6),
+        helpers.get_random_string(6): tools.get_random_boolean(),
+        helpers.get_random_string(6): tools.get_random_integer(),
+        helpers.get_random_string(6): [
+            tools.get_random_integer(),
+            helpers.get_random_string(6),
+            tools.get_random_boolean(),
+        ],
+        helpers.get_random_string(6): {
+            helpers.get_random_string(6): helpers.get_random_string(6)
+        }
+    }
+
+
 def get_set_of_random_uuids():
     return {
         uuidutils.generate_uuid()
@@ -396,6 +425,7 @@ def get_set_of_random_uuids():
 
 # NOTE: The keys in this dictionary have alphabetic order.
 FIELD_TYPE_VALUE_GENERATOR_MAP = {
+    common_types.DictOfMiscValuesField: get_random_dict,
     common_types.DomainNameField: get_random_domain_name,
     common_types.DscpMarkField: get_random_dscp_mark,
     common_types.EtherTypeEnumField: tools.get_random_ether_type,
@@ -1189,6 +1219,26 @@ class BaseDbObjectMultipleParentsForForeignKeysTestCase(
         self.assertEqual(fake_children, obj.children)
 
 
+class BaseObjectIfaceDictMiscValuesTestCase(_BaseObjectTestCase,
+                                            test_base.BaseTestCase):
+
+    _test_class = FakeNeutronObjectDictOfMiscValues
+
+    def test_dict_of_misc_values(self):
+        obj_id = uuidutils.generate_uuid()
+        float_value = 1.23
+        misc_list = [True, float_value]
+        obj_dict = {
+            'bool': True,
+            'float': float_value,
+            'misc_list': misc_list
+        }
+        obj = self._test_class(self.context, id=obj_id, dict_field=obj_dict)
+        self.assertTrue(obj.dict_field['bool'])
+        self.assertEqual(float_value, obj.dict_field['float'])
+        self.assertEqual(misc_list, obj.dict_field['misc_list'])
+
+
 class BaseDbObjectTestCase(_BaseObjectTestCase,
                            test_db_base_plugin_v2.DbOperationBoundMixin):
     def setUp(self):
@@ -1692,3 +1742,19 @@ class PagerTestCase(test_base.BaseTestCase):
 
         pager3 = base.Pager()
         self.assertNotEqual(pager, pager3)
+
+
+class OperationOnStringAndJsonTestCase(test_base.BaseTestCase):
+    def test_load_empty_string_to_json(self):
+        for field_val in ['', None]:
+            for default_val in [None, {}]:
+                res = base.NeutronDbObject.load_json_from_str(field_val,
+                                                              default_val)
+                self.assertEqual(res, default_val)
+
+    def test_dump_field_to_string(self):
+        for field_val in [{}, None]:
+            for default_val in ['', None]:
+                res = base.NeutronDbObject.filter_to_json_str(field_val,
+                                                              default_val)
+                self.assertEqual(default_val, res)

neutron/tests/unit/objects/test_common_types.py

@@ -266,3 +266,31 @@ class UUIDFieldTest(test_base.BaseTestCase, TestField):
     def test_stringify(self):
         for in_val, out_val in self.coerce_good_values:
             self.assertEqual('%s' % in_val, self.field.stringify(in_val))
+
+
+class DictOfMiscValuesFieldTest(test_base.BaseTestCase, TestField):
+    def setUp(self):
+        super(DictOfMiscValuesFieldTest, self).setUp()
+        self.field = common_types.DictOfMiscValues
+        test_dict_1 = {'a': True,
+                       'b': 1.23,
+                       'c': ['1', 1.23, True],
+                       'd': {'aa': 'zz'},
+                       'e': '10.0.0.1'}
+        test_dict_str = jsonutils.dumps(test_dict_1)
+        self.coerce_good_values = [
+            (test_dict_1, test_dict_1),
+            (test_dict_str, test_dict_1)
+        ]
+        self.coerce_bad_values = [str(test_dict_1), '{"a":}']
+        self.to_primitive_values = [
+            (test_dict_1, test_dict_str)
+        ]
+        self.from_primitive_values = [
+            (test_dict_str, test_dict_1)
+        ]
+
+    def test_stringify(self):
+        for in_val, out_val in self.coerce_good_values:
+            self.assertEqual(jsonutils.dumps(in_val),
+                             self.field.stringify(in_val))

neutron/tests/unit/objects/test_objects.py

@@ -29,9 +29,9 @@ from neutron.tests import base as test_base
 object_data = {
     '_DefaultSecurityGroup': '1.0-971520cb2e0ec06d747885a0cf78347f',
     'AddressScope': '1.0-25560799db384acfe1549634959a82b4',
-    'Agent': '1.0-7a8de4fedc7d318e7b6883b9747d1adb',
+    'Agent': '1.0-7106cb40117a8d1f042545796ed8787d',
     'AllowedAddressPair': '1.0-9f9186b6f952fbf31d257b0458b852c0',
-    'DistributedPortBinding': '1.0-4df058ae1aeae3ae1c15b8f6a4c692d9',
+    'DistributedPortBinding': '1.0-39c0d17b281991dcb66716fee5a8bef2',
     'DNSNameServer': '1.0-bf87a85327e2d812d1666ede99d9918b',
     'ExtraDhcpOpt': '1.0-632f689cbeb36328995a7aed1d0a78d3',
     'FlatAllocation': '1.0-bf666f24f4642b047eeca62311fbcb41',
@@ -47,7 +47,7 @@ object_data = {
     'NetworkPortSecurity': '1.0-b30802391a87945ee9c07582b4ff95e3',
     'NetworkSegment': '1.0-40707ef6bd9a0bf095038158d995cc7d',
     'Port': '1.0-638f6b09a3809ebd8b2b46293f56871b',
-    'PortBinding': '1.0-f5d3048bec0ac58f08a758427581dff9',
+    'PortBinding': '1.0-189fa2f450a1f24b1423df660eb43d71',
     'PortBindingLevel': '1.0-de66a4c61a083b8f34319fa9dde5b060',
     'PortDNS': '1.0-201cf6d057fde75539c3d1f2bbf05902',
     'PortSecurity': '1.0-b30802391a87945ee9c07582b4ff95e3',

neutron/tests/unit/objects/test_ports.py

@@ -108,9 +108,9 @@ class PortBindingVifDetailsTestCase(testscenarios.WithScenarios,
             self.context, **obj._get_composite_keys())
         self.assertEqual(vif_details, obj.vif_details)
 
-        vif_details['item1'] = 'val2'
+        vif_details['item1'] = 1.23
         del vif_details['item2']
-        vif_details['item3'] = 'val3'
+        vif_details['item3'] = True
 
         obj.vif_details = vif_details
         obj.update()
@@ -121,6 +121,9 @@ class PortBindingVifDetailsTestCase(testscenarios.WithScenarios,
 
         obj.vif_details = None
         obj.update()
+        # here the obj is reloaded from DB,
+        # so we test if vif_details is still none
+        self.assertIsNone(obj.vif_details)
 
         obj = self._test_class.get_object(
             self.context, **obj._get_composite_keys())