Support to query volume filter by updated_at/created_at

Support users can query resources by specifying the time
that resources are created at or updated at with time
comparison operators: "gt/gte/eq/neq/lt/lte",
and cinder will return all which match the condition.

The date and time stamp format is ISO 8601: CCYY-MM-DDThh:mm:ss±hh:mm.
The ±hh:mm value, if included, returns the time zone as an offset from UTC.

Add two filters updated_at and created_at in querying API.
"volumes/detail?created_at=gt:2016-01-01T01:00:00,lt:2016-12-31T01:00:00"

Change-Id: I1f43c37c2266e43146637beadc027ccf6dec017e
Partial-Implements: blueprint support-to-query-cinder-resources-filter-by-time-comparison-operators
Co-Authored-By: wangxiyuan <wangxiyuan@huawei.com>

api-ref/source/v3/parameters.yaml

@@ -208,6 +208,26 @@ detail:
   in: query
   required: false
   type: boolean
+filter_created_at:
+  description: |
+    Filters reuslts by a time that resources are created at with time
+    comparison operators: gt/gte/eq/neq/lt/lte.
+    The date and time stamp format is ISO 8601: CCYY-MM-DDThh:mm:ss±hh:mm.
+    The ±hh:mm value, if included, returns the time zone as an offset from UTC.
+  in: query
+  required: false
+  type: string
+  min_version: 3.60
+filter_updated_at:
+  description: |
+    Filters reuslts by a time that resources are updated at with time
+    comaprison operators: gt/gte/eq/neq/lt/lte.
+    The date and time stamp format is ISO 8601: CCYY-MM-DDThh:mm:ss±hh:mm.
+    The ±hh:mm value, if included, returns the time zone as an offset from UTC.
+  in: query
+  required: false
+  type: string
+  min_version: 3.60
 fix_allocated_quotas:
   description: |
     Whether to fix all the non-leaf projects' ``allocation``

api-ref/source/v3/samples/versions/version-show-response.json

@@ -22,7 +22,7 @@
             "min_version": "3.0",
             "status": "CURRENT",
             "updated": "2018-07-17T00:00:00Z",
-            "version": "3.59"
+            "version": "3.60"
         }
     ]
 }

api-ref/source/v3/samples/versions/versions-response.json

@@ -46,7 +46,7 @@
             "min_version": "3.0",
             "status": "CURRENT",
             "updated": "2018-07-17T00:00:00Z",
-            "version": "3.59"
+            "version": "3.60"
         }
     ]
 }

api-ref/source/v3/volumes-v3-volumes.inc

@@ -94,6 +94,8 @@ Request
    - offset: offset
    - marker: marker
    - with_count: with_count
+   - created_at: filter_created_at
+   - updated_at: filter_updated_at
 
 
 Response Parameters
@@ -286,6 +288,8 @@ Request
    - offset: offset
    - marker: marker
    - with_count: with_count
+   - created_at: filter_created_at
+   - updated_at: filter_updated_at
 
 
 Response Parameters

cinder/api/common.py

@@ -385,6 +385,14 @@ def get_enabled_resource_filters(resource=None):
         return {}
 
 
+def get_time_comparsion_operators():
+    """Get list of time comparsion operators.
+
+    This method returns list which contains the allowed comparsion operators.
+    """
+    return ["gt", "gte", "eq", "neq", "lt", "lte"]
+
+
 def convert_filter_attributes(filters, resource):
     for key in filters.copy().keys():
         if resource in ['volume', 'backup',

cinder/api/microversions.py

@@ -159,6 +159,8 @@ GROUP_GROUPSNAPSHOT_PROJECT_ID = '3.58'
 
 SUPPORT_TRANSFER_PAGINATION = '3.59'
 
+VOLUME_TIME_COMPARISON_FILTER = '3.60'
+
 
 def get_mv_header(version):
     """Gets a formatted HTTP microversion header.

cinder/api/openstack/api_version_request.py

@@ -135,6 +135,10 @@ REST_API_VERSION_HISTORY = """
              detail, list group snapshots with detail, show group detail and
              show group snapshot detail APIs.
     * 3.59 - Support volume transfer pagination.
+    * 3.60 - Support filtering on the "updated_at" and "created_at" fields with
+             time comparison operators for the volume summary list
+             ("GET /v3/{project_id}/volumes") and volume detail list
+             ("GET /v3/{project_id}/volumes/detail") requests.
 """
 
 # The minimum and maximum versions of the API supported
@@ -142,7 +146,7 @@ REST_API_VERSION_HISTORY = """
 # minimum version of the API supported.
 # Explicitly using /v2 endpoints will still work
 _MIN_API_VERSION = "3.0"
-_MAX_API_VERSION = "3.59"
+_MAX_API_VERSION = "3.60"
 _LEGACY_API_VERSION2 = "2.0"
 UPDATED = "2018-07-17T00:00:00Z"
 

cinder/api/openstack/rest_api_version_history.rst

@@ -463,3 +463,7 @@ detail APIs.
 ---------------------------------
 Support volume transfer pagination.
 
+3.60
+----
+Add 'created_at' and 'updated_at' to support users can list/detail volumes by
+specifying the time comparison operators along with created_at or updated_at.

cinder/api/v3/volumes.py

@@ -15,6 +15,7 @@
 
 from oslo_log import log as logging
 from oslo_log import versionutils
+from oslo_utils import timeutils
 import six
 from six.moves import http_client
 import webob
@@ -90,10 +91,44 @@ class VolumeController(volumes_v2.VolumeController):
         if req_version.matches(None, mv.BACKUP_UPDATE):
             filters.pop('group_id', None)
 
+        if req_version.matches(None, mv.SUPPORT_TRANSFER_PAGINATION):
+            filters.pop('created_at', None)
+            filters.pop('updated_at', None)
+
         api_utils.remove_invalid_filter_options(
             context, filters,
             self._get_volume_filter_options())
 
+    def _handle_time_comparison_filters(self, filters):
+        for time_comparison_filter in ['created_at', 'updated_at']:
+            if time_comparison_filter in filters:
+                time_filter_dict = {}
+                comparison_units = filters[time_comparison_filter].split(',')
+                operators = common.get_time_comparsion_operators()
+                for comparison_unit in comparison_units:
+                    try:
+                        operator_and_time = comparison_unit.split(":")
+                        comparison_operator = operator_and_time[0]
+                        time = ''
+                        for time_str in operator_and_time[1:-1]:
+                            time += time_str + ":"
+                        time += operator_and_time[-1]
+                        if comparison_operator not in operators:
+                            msg = _(
+                                'Invalid %s operator') % comparison_operator
+                            raise exc.HTTPBadRequest(explanation=msg)
+                    except IndexError:
+                        msg = _('Invalid %s value') % time_comparison_filter
+                        raise exc.HTTPBadRequest(explanation=msg)
+                    try:
+                        parsed_time = timeutils.parse_isotime(time)
+                    except ValueError:
+                        msg = _('Invalid %s value') % time
+                        raise exc.HTTPBadRequest(explanation=msg)
+                    time_filter_dict[comparison_operator] = parsed_time
+
+                filters[time_comparison_filter] = time_filter_dict
+
     def _get_volumes(self, req, is_detail):
         """Returns a list of volumes, transformed through view builder."""
 
@@ -121,6 +156,8 @@ class VolumeController(volumes_v2.VolumeController):
         if 'name' in filters:
             filters['display_name'] = filters.pop('name')
 
+        self._handle_time_comparison_filters(filters)
+
         strict = req.api_version_request.matches(
             mv.VOLUME_LIST_BOOTABLE, None)
         self.volume_api.check_volume_filters(filters, strict)

cinder/db/sqlalchemy/api.py

@@ -413,6 +413,27 @@ def _filter_host(field, value, match_level=None):
     return or_(*conditions)
 
 
+def _filter_time_comparison(field, time_filter_dict):
+    """Generate a filter condition for time comparison operators"""
+
+    conditions = []
+    for operator in time_filter_dict:
+        filter_time = timeutils.normalize_time(time_filter_dict[operator])
+        if operator == 'gt':
+            conditions.append(field.op('>')(filter_time))
+        elif operator == 'gte':
+            conditions.append(field.op('>=')(filter_time))
+        if operator == 'eq':
+            conditions.append(field.op('=')(filter_time))
+        elif operator == 'neq':
+            conditions.append(field.op('!=')(filter_time))
+        if operator == 'lt':
+            conditions.append(field.op('<')(filter_time))
+        elif operator == 'lte':
+            conditions.append(field.op('<=')(filter_time))
+    return or_(*conditions)
+
+
 def _clustered_bool_field_filter(query, field_name, filter_value):
     # Now that we have clusters, a service is disabled/frozen if the service
     # doesn't belong to a cluster or if it belongs to a cluster and the cluster
@@ -2433,6 +2454,19 @@ def _process_volume_filters(query, filters):
         query = query.filter(_filter_host(models.Volume.cluster_name,
                                           cluster_name))
 
+    for time_comparison_filter in ['created_at', 'updated_at']:
+        if filters.get(time_comparison_filter, None):
+            time_filter_dict = filters.pop(time_comparison_filter)
+            try:
+                time_filter_attr = getattr(models.Volume,
+                                           time_comparison_filter)
+                query = query.filter(_filter_time_comparison(time_filter_attr,
+                                                             time_filter_dict))
+            except AttributeError:
+                LOG.debug("%s column could not be found.",
+                          time_comparison_filter)
+                return None
+
     # Apply exact match filters for everything else, ensure that the
     # filter value exists on the model
     for key in filters.keys():

cinder/tests/unit/api/test_common.py

@@ -477,7 +477,7 @@ class GeneralFiltersTest(test.TestCase):
                'expected': ["name", "status", "metadata",
                             "bootable", "migration_status",
                             "availability_zone", "group_id",
-                            "size"]},
+                            "size", "created_at", "updated_at"]},
               {'resource': 'backup',
                'expected': ["name", "status", "volume_id"]},
               {'resource': 'snapshot',

cinder/tests/unit/api/v3/test_volumes.py

@@ -20,6 +20,7 @@ import fixtures
 import iso8601
 from oslo_serialization import jsonutils
 from oslo_utils import strutils
+from oslo_utils import timeutils
 from six.moves import http_client
 import webob
 
@@ -120,18 +121,27 @@ class VolumeApiTest(test.TestCase):
         self._reset_filter_file()
 
     def _create_volume_with_glance_metadata(self):
+        basetime = timeutils.utcnow()
+        td = datetime.timedelta(minutes=1)
+
         vol1 = db.volume_create(self.ctxt, {'display_name': 'test1',
+                                            'created_at': basetime - 3 * td,
+                                            'updated_at': basetime - 2 * td,
                                             'project_id':
                                             self.ctxt.project_id,
                                             'volume_type_id':
-                                                fake.VOLUME_TYPE_ID})
+                                                fake.VOLUME_TYPE_ID,
+                                            'id': fake.VOLUME_ID})
         db.volume_glance_metadata_create(self.ctxt, vol1.id, 'image_name',
                                          'imageTestOne')
         vol2 = db.volume_create(self.ctxt, {'display_name': 'test2',
+                                            'created_at': basetime - td,
+                                            'updated_at': basetime,
                                             'project_id':
                                             self.ctxt.project_id,
                                             'volume_type_id':
-                                                fake.VOLUME_TYPE_ID})
+                                                fake.VOLUME_TYPE_ID,
+                                            'id': fake.VOLUME2_ID})
         db.volume_glance_metadata_create(self.ctxt, vol2.id, 'image_name',
                                          'imageTestTwo')
         db.volume_glance_metadata_create(self.ctxt, vol2.id, 'disk_format',
@@ -984,3 +994,80 @@ class VolumeApiTest(test.TestCase):
         self.assertEqual('host1', attachments[0]['host_name'])
         self.assertEqual('na', attachments[0]['device'])
         self.assertEqual(att_time, attachments[0]['attached_at'])
+
+    @ddt.data(('created_at=gt:', 0), ('created_at=lt:', 2))
+    @ddt.unpack
+    def test_volume_index_filter_by_created_at_with_gt_and_lt(self, change,
+                                                              expect_result):
+        self._create_volume_with_glance_metadata()
+        change_time = timeutils.utcnow() + datetime.timedelta(minutes=1)
+        req = fakes.HTTPRequest.blank(("/v3/volumes?%s%s") %
+                                      (change, change_time))
+        req.environ['cinder.context'] = self.ctxt
+        req.headers = mv.get_mv_header(mv.VOLUME_TIME_COMPARISON_FILTER)
+        req.api_version_request = mv.get_api_version(
+            mv.VOLUME_TIME_COMPARISON_FILTER)
+        res_dict = self.controller.index(req)
+        volumes = res_dict['volumes']
+        self.assertEqual(expect_result, len(volumes))
+
+    @ddt.data(('updated_at=gt:', 0), ('updated_at=lt:', 1))
+    @ddt.unpack
+    def test_vol_filter_by_updated_at_with_gt_and_lt(self, change, result):
+        vols = self._create_volume_with_glance_metadata()
+        change_time = vols[1].updated_at
+        req = fakes.HTTPRequest.blank(("/v3/volumes?%s%s") %
+                                      (change, change_time))
+        req.environ['cinder.context'] = self.ctxt
+        req.headers = mv.get_mv_header(mv.VOLUME_TIME_COMPARISON_FILTER)
+        req.api_version_request = mv.get_api_version(
+            mv.VOLUME_TIME_COMPARISON_FILTER)
+        res_dict = self.controller.index(req)
+        volumes = res_dict['volumes']
+        self.assertEqual(result, len(volumes))
+
+    @ddt.data(('updated_at=eq:', 1, fake.VOLUME2_ID),
+              ('updated_at=neq:', 1, fake.VOLUME_ID))
+    @ddt.unpack
+    def test_vol_filter_by_updated_at_with_eq_and_neq(self, change, result,
+                                                      expected_volume_id):
+        vols = self._create_volume_with_glance_metadata()
+        change_time = vols[1].updated_at
+        req = fakes.HTTPRequest.blank(("/v3/volumes?%s%s") %
+                                      (change, change_time))
+        req.environ['cinder.context'] = self.ctxt
+        req.headers = mv.get_mv_header(mv.VOLUME_TIME_COMPARISON_FILTER)
+        req.api_version_request = mv.get_api_version(
+            mv.VOLUME_TIME_COMPARISON_FILTER)
+        res_dict = self.controller.index(req)
+        volumes = res_dict['volumes']
+        self.assertEqual(result, len(volumes))
+        self.assertEqual(expected_volume_id, volumes[0]['id'])
+
+    @ddt.data('created_at', 'updated_at')
+    def test_volume_filter_by_time_with_invaild_time(self, change):
+        self._create_volume_with_glance_metadata()
+        change_time = '123'
+        req = fakes.HTTPRequest.blank(("/v3/volumes?%s=%s") %
+                                      (change, change_time))
+        req.environ['cinder.context'] = self.ctxt
+        req.headers = mv.get_mv_header(mv.VOLUME_TIME_COMPARISON_FILTER)
+        req.api_version_request = mv.get_api_version(
+            mv.VOLUME_TIME_COMPARISON_FILTER)
+        self.assertRaises(webob.exc.HTTPBadRequest, self.controller.index, req)
+
+    def test_volume_index_filter_by_time_with_lte_and_gte(self):
+        vols = self._create_volume_with_glance_metadata()
+        change_since = vols[1].updated_at
+        change_before = timeutils.utcnow() + datetime.timedelta(minutes=1)
+        req = fakes.HTTPRequest.blank(("/v3/volumes?updated_at=lte:%s&"
+                                       "updated_at=gte:%s") %
+                                      (change_before, change_since))
+        req.environ['cinder.context'] = self.ctxt
+        req.headers = mv.get_mv_header(mv.VOLUME_TIME_COMPARISON_FILTER)
+        req.api_version_request = mv.get_api_version(
+            mv.VOLUME_TIME_COMPARISON_FILTER)
+        res_dict = self.controller.index(req)
+        volumes = res_dict['volumes']
+        self.assertEqual(1, len(volumes))
+        self.assertEqual(vols[1].id, volumes[0]['id'])

etc/cinder/resource_filters.json

@@ -1,7 +1,7 @@
 {
     "volume": ["name", "status", "metadata",
                "bootable", "migration_status", "availability_zone",
-               "group_id", "size"],
+               "group_id", "size", "created_at", "updated_at"],
     "backup": ["name", "status", "volume_id"],
     "snapshot": ["name", "status", "volume_id", "metadata",
                  "availability_zone"],

releasenotes/notes/support-to-query-cinder-resources-filter-by-update-at-and-created-at-32ae9aaea131d598.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - Beginning with microversion 3.60, users may apply time comparison filters
+    to the volume summary list and volume detail list requests by using the
+    "created_at" or "updated_at" fields. Time must be expressed in ISO
+    8601 format. See the 'Block Storage API v3 Reference
+    <https://docs.openstack.org/api-ref/block-storage/v3/index.html>'_ for
+    details.