Fix serialization of lists in data_models.to_dict

data_models.to_dict didn't convert values inside lists.
When serializing TLSContainer, the intermediates list that contains
bytes types wasn't converted properly.

to_dict now converts the items of a list recursively.

Add a test that checks if TLSContainer.to_dict is serializable.

Story 2009310
Task 43699

Change-Id: I3859c7fcefc89e91fdaecd139143e6a74d8b3c1b

octavia/common/data_models.py

@@ -26,6 +26,55 @@ LOG = logging.getLogger(__name__)
 
 
 class BaseDataModel(object):
+    def _to_dict(self, value, calling_classes=None, recurse=False):
+        calling_classes = calling_classes or []
+        # We need to have json convertible data for storing it in
+        # persistence jobboard backend.
+        if isinstance(value, datetime.datetime):
+            ret = value.isoformat()
+        elif isinstance(value, bytes):
+            ret = value.decode()
+        elif recurse:
+            if isinstance(value, list):
+                ret = []
+                for item in value:
+                    if isinstance(item, BaseDataModel):
+                        if type(self) not in calling_classes:
+                            ret.append(
+                                item.to_dict(calling_classes=(
+                                    calling_classes + [type(self)]),
+                                    recurse=recurse))
+                        else:
+                            # TODO(rm_work): Is the idea that if this list
+                            #  contains ANY BaseDataModel, that all of them
+                            #  are data models, and we may as well quit?
+                            #  Or, were we supposed to append a `None` for
+                            #  each one? I assume the former?
+                            ret = None
+                            break
+                    else:
+                        ret.append(
+                            self._to_dict(item,
+                                          calling_classes=calling_classes,
+                                          recurse=recurse))
+            elif isinstance(value, BaseDataModel):
+                if type(self) not in calling_classes:
+                    ret = value.to_dict(
+                        calling_classes=calling_classes + [type(self)],
+                        recurse=recurse)
+                else:
+                    ret = None
+            else:
+                ret = value
+        else:
+            if isinstance(value, BaseDataModel):
+                ret = None
+            elif isinstance(value, list):
+                ret = []
+            else:
+                ret = value
+        return ret
+
     def to_dict(self, calling_classes=None, recurse=False, **kwargs):
         """Converts a data model to a dictionary."""
         calling_classes = calling_classes or []
@@ -37,50 +86,8 @@ class BaseDataModel(object):
                 # tags is a list, it doesn't need recurse
                 ret[attr] = value
                 continue
-            # We need to have json convertible data for storing it in
+            ret[attr] = self._to_dict(value, calling_classes=calling_classes,
-            # persistence jobboard backend.
+                                      recurse=recurse)
-            if isinstance(value, datetime.datetime):
-                ret[attr] = value.isoformat()
-                continue
-            if isinstance(value, bytes):
-                ret[attr] = value.decode()
-                continue
-            if recurse:
-                if isinstance(getattr(self, attr), list):
-                    ret[attr] = []
-                    for item in value:
-                        if isinstance(item, BaseDataModel):
-                            if type(self) not in calling_classes:
-                                ret[attr].append(
-                                    item.to_dict(calling_classes=(
-                                        calling_classes + [type(self)]),
-                                        recurse=recurse))
-                            else:
-                                # TODO(rm_work): Is the idea that if this list
-                                #  contains ANY BaseDataModel, that all of them
-                                #  are data models, and we may as well quit?
-                                #  Or, were we supposed to append a `None` for
-                                #  each one? I assume the former?
-                                ret[attr] = None
-                                break
-                        else:
-                            ret[attr].append(item)
-                elif isinstance(getattr(self, attr), BaseDataModel):
-                    if type(self) not in calling_classes:
-                        ret[attr] = value.to_dict(
-                            calling_classes=calling_classes + [type(self)],
-                            recurse=recurse)
-                    else:
-                        ret[attr] = None
-                else:
-                    ret[attr] = value
-            else:
-                if isinstance(getattr(self, attr), BaseDataModel):
-                    ret[attr] = None
-                elif isinstance(getattr(self, attr), list):
-                    ret[attr] = []
-                else:
-                    ret[attr] = value
 
         return ret
 

octavia/tests/unit/common/test_data_models.py

@@ -14,6 +14,7 @@
 
 import copy
 import datetime
+import json
 import random
 
 from oslo_utils import uuidutils
@@ -43,6 +44,7 @@ class TestDataModels(base.TestCase):
         self.COMPUTE_ID = uuidutils.generate_uuid()
         self.IMAGE_ID = uuidutils.generate_uuid()
         self.COMPUTE_FLAVOR = uuidutils.generate_uuid()
+        self.TLS_CONTAINER_ID = uuidutils.generate_uuid()
 
         self.LB_obj = data_models.LoadBalancer(
             id=self.LB_ID,
@@ -533,3 +535,21 @@ class TestDataModels(base.TestCase):
 
         # test incrementing an incompatible object
         self.assertRaises(TypeError, stats_1.__iadd__, "boom")
+
+    def test_TLSContainer_serialization(self):
+        tls_container = data_models.TLSContainer(
+            id=self.TLS_CONTAINER_ID,
+            primary_cn='fake_cn',
+            certificate=b'certificate_buffer1',
+            private_key=b'private_key1',
+            passphrase=b'passphrase1',
+            intermediates=[
+                b'intermediate_buffer1',
+                b'intermediate_buffer2',
+            ]
+        )
+        tls_container_dict = tls_container.to_dict(recurse=True)
+        json_buffer = json.dumps(tls_container_dict)
+        json_doc = json.loads(json_buffer)
+
+        self.assertEqual(tls_container_dict, json_doc)

releasenotes/notes/fix-tls-container-serialization-1cb83ad4c9eca3b8.yaml

@@ -0,0 +1,6 @@
+---
+fixes:
+  - |
+    Fix a serialization issue when using TLSContainer with amphorav2 driver
+    with persistence, a list of bytes type in the data model was not correctly
+    converted to serializable data.