Add timestamp changed-since for core resources

This patch add filter timestamp with changed-since and Uts with it.

Change-Id: I9cbdd1a31d843fbfa6c6ffd7eeaf803bea683e93
Partially-Implements: blueprint add-port-timestamp

neutron/services/timestamp/timestamp_db.py

@@ -12,6 +12,9 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import datetime
+import time
+
 from oslo_log import log
 from oslo_utils import timeutils
 from sqlalchemy import event
@@ -19,6 +22,7 @@ from sqlalchemy import exc as sql_exc
 from sqlalchemy.orm import session as se
 
 from neutron._i18n import _LW
+from neutron.common import exceptions as n_exc
 from neutron.db import model_base
 
 LOG = log.getLogger(__name__)
@@ -29,6 +33,36 @@ class TimeStamp_db_mixin(object):
 
     ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S'
 
+    def _change_since_result_filter_hook(self, query, filters):
+        # this block is for change_since query
+        # we get the changed_since string from filters.
+        # And translate it from string to datetime type.
+        # Then compare with the timestamp in db which has
+        # datetime type.
+        values = filters and filters.get('changed_since', [])
+        if not values:
+            return query
+        data = filters['changed_since'][0]
+        try:
+            # this block checks queried timestamp format.
+            datetime.datetime.fromtimestamp(time.mktime(
+                time.strptime(data,
+                              self.ISO8601_TIME_FORMAT)))
+        except Exception:
+            msg = _LW("The input changed_since must be in the "
+                      "following format: YYYY-MM-DDTHH:MM:SS")
+            raise n_exc.InvalidInput(error_message=msg)
+        changed_since_string = timeutils.parse_isotime(data)
+        changed_since = (timeutils.
+                         normalize_time(changed_since_string))
+        target_model_class = list(query._mapper_adapter_map.keys())[0]
+        query = query.join(model_base.StandardAttribute,
+                           target_model_class.standard_attr_id ==
+                           model_base.StandardAttribute.id).filter(
+                           model_base.StandardAttribute.updated_at
+                           >= changed_since)
+        return query
+
     def update_timestamp(self, session, context, instances):
         objs_list = session.new.union(session.dirty)
 

neutron/services/timestamp/timestamp_plugin.py

@@ -14,6 +14,7 @@
 
 from neutron.api.v2 import attributes
 from neutron.db import db_base_plugin_v2
+from neutron.db import models_v2
 from neutron.services import service_base
 from neutron.services.timestamp import timestamp_db as ts_db
 
@@ -32,6 +33,15 @@ class TimeStampPlugin(service_base.ServicePluginBase,
             db_base_plugin_v2.NeutronDbPluginV2.register_dict_extend_funcs(
                 resources, [self.extend_resource_dict_timestamp])
 
+        for model in [models_v2.Network, models_v2.Port, models_v2.Subnet,
+                      models_v2.SubnetPool]:
+            db_base_plugin_v2.NeutronDbPluginV2.register_model_query_hook(
+                model,
+                "change_since_query",
+                None,
+                None,
+                self._change_since_result_filter_hook)
+
     def get_plugin_type(self):
         return 'timestamp_core'
 

neutron/tests/unit/extensions/test_timestamp_core.py

@@ -0,0 +1,225 @@
+# Copyright 2015 HuaWei Technologies.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import datetime
+import six
+
+from oslo_config import cfg
+
+from neutron.db import db_base_plugin_v2
+from neutron.extensions import timestamp_core as timestamp
+from neutron import manager
+from neutron.tests.unit.db import test_db_base_plugin_v2
+
+
+class TimeStampExtensionManager(object):
+
+    def get_resources(self):
+        return []
+
+    def get_actions(self):
+        return []
+
+    def get_request_extensions(self):
+        return []
+
+    def get_extended_resources(self, version):
+        return timestamp.Timestamp_core().get_extended_resources(version)
+
+
+class TimeStampTestPlugin(db_base_plugin_v2.NeutronDbPluginV2):
+    """Just for test with TimeStampPlugin"""
+
+
+class TimeStampChangedsinceTestCase(test_db_base_plugin_v2.
+                                    NeutronDbPluginV2TestCase):
+    plugin = ('neutron.tests.unit.extensions.test_timestamp_core.' +
+              'TimeStampTestPlugin')
+
+    def setUp(self):
+        ext_mgr = TimeStampExtensionManager()
+        super(TimeStampChangedsinceTestCase, self).setUp(plugin=self.plugin,
+                                                         ext_mgr=ext_mgr)
+        self.addCleanup(manager.NeutronManager.
+                        get_service_plugins()['timestamp_core'].
+                        unregister_db_events)
+        self.addCleanup(manager.NeutronManager.clear_instance)
+
+    def setup_coreplugin(self, core_plugin=None):
+        cfg.CONF.set_override('core_plugin', self.plugin)
+
+    def _get_resp_with_changed_since(self, resource_type, changed_since):
+        query_params = 'changed_since=%s' % changed_since
+        req = self.new_list_request('%ss' % resource_type, self.fmt,
+                                    query_params)
+        resources = self.deserialize(self.fmt, req.get_response(self.api))
+        return resources
+
+    def _return_by_timedelay(self, resource, timedelay):
+        resource_type = six.next(six.iterkeys(resource))
+        try:
+            time_create = datetime.datetime.strptime(
+                resource[resource_type]['updated_at'],
+                '%Y-%m-%dT%H:%M:%S')
+        except Exception:
+            time_create = datetime.datetime.strptime(
+                resource[resource_type]['updated_at'],
+                '%Y-%m-%d %H:%M:%S.%f')
+        time_before = datetime.timedelta(seconds=timedelay)
+        addedtime_string = (datetime.datetime.
+                            strftime(time_create + time_before,
+                                     '%Y-%m-%dT%H:%M:%S'))
+        return self._get_resp_with_changed_since(resource_type,
+                                                 addedtime_string)
+
+    def _update_test_resource_by_name(self, resource):
+        resource_type = six.next(six.iterkeys(resource))
+        name = resource[resource_type]['name']
+        data = {resource_type: {'name': '%s_new' % name}}
+        req = self.new_update_request('%ss' % resource_type,
+                                      data,
+                                      resource[resource_type]['id'])
+        res = self.deserialize(self.fmt, req.get_response(self.api))
+        return res
+
+    def _set_timestamp_by_show(self, resource, type):
+        req = self.new_show_request('%ss' % type,
+                                    resource[type]['id'])
+        res = self.deserialize(self.fmt, req.get_response(self.api))
+        resource[type]['created_at'] = res[type]['created_at']
+        resource[type]['updated_at'] = res[type]['updated_at']
+
+    def _list_resources_with_changed_since(self, resource):
+        # assert list results contain the net info when
+        # changed_since equal with the net updated time.
+        resource_type = six.next(six.iterkeys(resource))
+        if resource_type in ['network', 'port']:
+            self._set_timestamp_by_show(resource, resource_type)
+        resources = self._get_resp_with_changed_since(resource_type,
+                                                      resource[resource_type][
+                                                          'updated_at'])
+        self.assertEqual(resource[resource_type]['id'],
+                         resources[resource_type + 's'][0]['id'])
+
+        # assert list results contain the net info when changed_since
+        # is earlier than the net updated time.
+        resources = self._return_by_timedelay(resource, -1)
+        self.assertEqual(resource[resource_type]['id'],
+                         resources[resource_type + 's'][0]['id'])
+
+        # assert list results is Null when changed_since
+        # is later with the net updated time.
+        resources = self._return_by_timedelay(resource, 1)
+        self.assertEqual([], resources[resource_type + 's'])
+
+    def _test_list_mutiple_resources_with_changed_since(self, first, second):
+        resource_type = six.next(six.iterkeys(first))
+        if resource_type in ['network', 'port']:
+            self._set_timestamp_by_show(first, resource_type)
+            self._set_timestamp_by_show(second, resource_type)
+        # update names of second
+        new_second = self._update_test_resource_by_name(second)
+        # now the queue of order by
+        # updated_at is first < new_second
+
+        # test changed_since < first's updated_at
+        resources = self._return_by_timedelay(first, -1)
+        for resource in [first[resource_type]['id'],
+                         new_second[resource_type]['id']]:
+            self.assertIn(resource,
+                          [n['id'] for n in resources[resource_type + 's']])
+
+        # test changed_since = first's updated_at
+        resources = self._return_by_timedelay(first, 0)
+        for resource in [first[resource_type]['id'],
+                         new_second[resource_type]['id']]:
+            self.assertIn(resource,
+                          [n['id'] for n in resources[resource_type + 's']])
+
+        # test first < changed_since < second
+        resources = self._return_by_timedelay(new_second, -1)
+        self.assertIn(new_second[resource_type]['id'],
+                      [n['id'] for n in resources[resource_type + 's']])
+
+        # test first < changed_since = second
+        resources = self._return_by_timedelay(new_second, 0)
+        self.assertIn(new_second[resource_type]['id'],
+                      [n['id'] for n in resources[resource_type + 's']])
+
+        #test first < second < changed_since
+        resources = self._return_by_timedelay(new_second, 3)
+        self.assertEqual({resource_type + 's': []}, resources)
+
+    def test_list_networks_with_changed_since(self):
+        with self.network('net1') as net:
+            self._list_resources_with_changed_since(net)
+
+    def test_list_subnets_with_changed_since(self):
+        with self.network('net2') as net:
+            with self.subnet(network=net) as subnet:
+                self._list_resources_with_changed_since(subnet)
+
+    def test_list_ports_with_changed_since(self):
+        with self.network('net3') as net:
+            with self.subnet(network=net) as subnet:
+                with self.port(subnet=subnet) as port:
+                    self._list_resources_with_changed_since(port)
+
+    def test_list_subnetpools_with_changed_since(self):
+        prefixs = ['3.3.3.3/24', '4.4.4.4/24']
+        with self.subnetpool(prefixs, tenant_id='tenant_one',
+                             name='sp_test02') as subnetpool:
+            self._list_resources_with_changed_since(subnetpool)
+
+    def test_list_mutiple_networks_with_changed_since(self):
+        with self.network('net1') as net1, self.network('net2') as net2:
+            self._test_list_mutiple_resources_with_changed_since(net1, net2)
+
+    def test_list_mutiple_subnets_with_changed_since(self):
+        with self.network('net1') as net1, self.network('net2') as net2:
+            with self.subnet(network=net1) as subnet1, self.subnet(
+                    network=net2) as subnet2:
+                self._test_list_mutiple_resources_with_changed_since(subnet1,
+                                                                     subnet2)
+
+    def test_list_mutiple_subnetpools_with_changed_since(self):
+        prefixs1 = ['3.3.3.3/24', '4.4.4.4/24']
+        prefixs2 = ['5.5.5.5/24', '6.6.6.6/24']
+        with self.subnetpool(prefixs1,
+                             tenant_id='tenant_one',
+                             name='sp01') as sp1:
+            with self.subnetpool(prefixs2,
+                                 tenant_id='tenant_one',
+                                 name='sp02') as sp2:
+                self._test_list_mutiple_resources_with_changed_since(sp1, sp2)
+
+    def test_list_mutiple_ports_with_changed_since(self):
+        with self.network('net') as net:
+            with self.subnet(network=net) as subnet:
+                with self.port(subnet=subnet) as p1, self.port(
+                        subnet=subnet) as p2:
+                    self._test_list_mutiple_resources_with_changed_since(p1,
+                                                                         p2)
+
+    def test_list_resources_with_invalid_changed_since(self):
+        # check when input --changed-since with no arg, then filters
+        # stored as 'True'. And also check other invalid inputs
+        changed_sinces = ['123', 'True', 'AAAA-BB-CCTDD-EE-FFZ',
+                          '9a9b-11-00T99-1a-r3Z', '0000-00-00T00-00-00Z']
+        for resource in ['network', 'subnet', 'port', 'subnetpool']:
+            for changed_since in changed_sinces:
+                req = self.new_list_request('%ss' % resource, self.fmt,
+                                            'changed_since=%s' % changed_since)
+                res = self.deserialize(self.fmt, req.get_response(self.api))
+                self.assertEqual(list(res.values())[0]['type'], 'InvalidInput')

releasenotes/notes/add-timestamp-fields-f9ab949fc88f05f6.yaml

@@ -4,3 +4,6 @@ prelude: >
 features:
   - Add timestamp fields 'created_at', 'updated_at' into neutron core
     resources like network, subnet, port and subnetpool.
+  - And support for querying these resources by changed-since, it will
+    return the resources changed after the specfic time string like
+    YYYY-MM-DDTHH:MM:SS