From 7e98d14a5724efaa8b02d8dc1c5d28cde7ce0ea6 Mon Sep 17 00:00:00 2001 From: wanghao Date: Tue, 9 Aug 2016 11:16:27 +0800 Subject: [PATCH] Support to query volume filter by updated_at/created_at MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Support users can query resources by specifying the time that resources are created at or updated at with time comparison operators: "gt/gte/eq/neq/lt/lte", and cinder will return all which match the condition. The date and time stamp format is ISO 8601: CCYY-MM-DDThh:mm:ss±hh:mm. The ±hh:mm value, if included, returns the time zone as an offset from UTC. Add two filters updated_at and created_at in querying API. "volumes/detail?created_at=gt:2016-01-01T01:00:00,lt:2016-12-31T01:00:00" Change-Id: I1f43c37c2266e43146637beadc027ccf6dec017e Partial-Implements: blueprint support-to-query-cinder-resources-filter-by-time-comparison-operators Co-Authored-By: wangxiyuan --- api-ref/source/v3/parameters.yaml | 20 ++++ .../versions/version-show-response.json | 2 +- .../samples/versions/versions-response.json | 2 +- api-ref/source/v3/volumes-v3-volumes.inc | 4 + cinder/api/common.py | 8 ++ cinder/api/microversions.py | 2 + cinder/api/openstack/api_version_request.py | 6 +- .../openstack/rest_api_version_history.rst | 4 + cinder/api/v3/volumes.py | 37 ++++++++ cinder/db/sqlalchemy/api.py | 34 +++++++ cinder/tests/unit/api/test_common.py | 2 +- cinder/tests/unit/api/v3/test_volumes.py | 91 ++++++++++++++++++- etc/cinder/resource_filters.json | 2 +- ...te-at-and-created-at-32ae9aaea131d598.yaml | 8 ++ 14 files changed, 215 insertions(+), 7 deletions(-) create mode 100644 releasenotes/notes/support-to-query-cinder-resources-filter-by-update-at-and-created-at-32ae9aaea131d598.yaml diff --git a/api-ref/source/v3/parameters.yaml b/api-ref/source/v3/parameters.yaml index 35d3306e90d..52a8182c03c 100644 --- a/api-ref/source/v3/parameters.yaml +++ b/api-ref/source/v3/parameters.yaml @@ -208,6 +208,26 @@ detail: in: query required: false type: boolean +filter_created_at: + description: | + Filters reuslts by a time that resources are created at with time + comparison operators: gt/gte/eq/neq/lt/lte. + The date and time stamp format is ISO 8601: CCYY-MM-DDThh:mm:ss±hh:mm. + The ±hh:mm value, if included, returns the time zone as an offset from UTC. + in: query + required: false + type: string + min_version: 3.60 +filter_updated_at: + description: | + Filters reuslts by a time that resources are updated at with time + comaprison operators: gt/gte/eq/neq/lt/lte. + The date and time stamp format is ISO 8601: CCYY-MM-DDThh:mm:ss±hh:mm. + The ±hh:mm value, if included, returns the time zone as an offset from UTC. + in: query + required: false + type: string + min_version: 3.60 fix_allocated_quotas: description: | Whether to fix all the non-leaf projects' ``allocation`` diff --git a/api-ref/source/v3/samples/versions/version-show-response.json b/api-ref/source/v3/samples/versions/version-show-response.json index dcb0b3c753c..4d5fbb1f49a 100644 --- a/api-ref/source/v3/samples/versions/version-show-response.json +++ b/api-ref/source/v3/samples/versions/version-show-response.json @@ -22,7 +22,7 @@ "min_version": "3.0", "status": "CURRENT", "updated": "2018-07-17T00:00:00Z", - "version": "3.59" + "version": "3.60" } ] } diff --git a/api-ref/source/v3/samples/versions/versions-response.json b/api-ref/source/v3/samples/versions/versions-response.json index 1d5f1c35756..8145f5e58b3 100644 --- a/api-ref/source/v3/samples/versions/versions-response.json +++ b/api-ref/source/v3/samples/versions/versions-response.json @@ -46,7 +46,7 @@ "min_version": "3.0", "status": "CURRENT", "updated": "2018-07-17T00:00:00Z", - "version": "3.59" + "version": "3.60" } ] } \ No newline at end of file diff --git a/api-ref/source/v3/volumes-v3-volumes.inc b/api-ref/source/v3/volumes-v3-volumes.inc index f46176ed4d9..77c20129696 100644 --- a/api-ref/source/v3/volumes-v3-volumes.inc +++ b/api-ref/source/v3/volumes-v3-volumes.inc @@ -94,6 +94,8 @@ Request - offset: offset - marker: marker - with_count: with_count + - created_at: filter_created_at + - updated_at: filter_updated_at Response Parameters @@ -286,6 +288,8 @@ Request - offset: offset - marker: marker - with_count: with_count + - created_at: filter_created_at + - updated_at: filter_updated_at Response Parameters diff --git a/cinder/api/common.py b/cinder/api/common.py index 12642944fd9..23479ca5957 100644 --- a/cinder/api/common.py +++ b/cinder/api/common.py @@ -385,6 +385,14 @@ def get_enabled_resource_filters(resource=None): return {} +def get_time_comparsion_operators(): + """Get list of time comparsion operators. + + This method returns list which contains the allowed comparsion operators. + """ + return ["gt", "gte", "eq", "neq", "lt", "lte"] + + def convert_filter_attributes(filters, resource): for key in filters.copy().keys(): if resource in ['volume', 'backup', diff --git a/cinder/api/microversions.py b/cinder/api/microversions.py index 638eb27fd33..76936bd2aea 100644 --- a/cinder/api/microversions.py +++ b/cinder/api/microversions.py @@ -159,6 +159,8 @@ GROUP_GROUPSNAPSHOT_PROJECT_ID = '3.58' SUPPORT_TRANSFER_PAGINATION = '3.59' +VOLUME_TIME_COMPARISON_FILTER = '3.60' + def get_mv_header(version): """Gets a formatted HTTP microversion header. diff --git a/cinder/api/openstack/api_version_request.py b/cinder/api/openstack/api_version_request.py index 79bb249d523..3e42e4151c5 100644 --- a/cinder/api/openstack/api_version_request.py +++ b/cinder/api/openstack/api_version_request.py @@ -135,6 +135,10 @@ REST_API_VERSION_HISTORY = """ detail, list group snapshots with detail, show group detail and show group snapshot detail APIs. * 3.59 - Support volume transfer pagination. + * 3.60 - Support filtering on the "updated_at" and "created_at" fields with + time comparison operators for the volume summary list + ("GET /v3/{project_id}/volumes") and volume detail list + ("GET /v3/{project_id}/volumes/detail") requests. """ # The minimum and maximum versions of the API supported @@ -142,7 +146,7 @@ REST_API_VERSION_HISTORY = """ # minimum version of the API supported. # Explicitly using /v2 endpoints will still work _MIN_API_VERSION = "3.0" -_MAX_API_VERSION = "3.59" +_MAX_API_VERSION = "3.60" _LEGACY_API_VERSION2 = "2.0" UPDATED = "2018-07-17T00:00:00Z" diff --git a/cinder/api/openstack/rest_api_version_history.rst b/cinder/api/openstack/rest_api_version_history.rst index 404b8496c04..026431c74a1 100644 --- a/cinder/api/openstack/rest_api_version_history.rst +++ b/cinder/api/openstack/rest_api_version_history.rst @@ -463,3 +463,7 @@ detail APIs. --------------------------------- Support volume transfer pagination. +3.60 +---- +Add 'created_at' and 'updated_at' to support users can list/detail volumes by +specifying the time comparison operators along with created_at or updated_at. diff --git a/cinder/api/v3/volumes.py b/cinder/api/v3/volumes.py index d611be1e87f..27d046ca2a6 100644 --- a/cinder/api/v3/volumes.py +++ b/cinder/api/v3/volumes.py @@ -15,6 +15,7 @@ from oslo_log import log as logging from oslo_log import versionutils +from oslo_utils import timeutils import six from six.moves import http_client import webob @@ -90,10 +91,44 @@ class VolumeController(volumes_v2.VolumeController): if req_version.matches(None, mv.BACKUP_UPDATE): filters.pop('group_id', None) + if req_version.matches(None, mv.SUPPORT_TRANSFER_PAGINATION): + filters.pop('created_at', None) + filters.pop('updated_at', None) + api_utils.remove_invalid_filter_options( context, filters, self._get_volume_filter_options()) + def _handle_time_comparison_filters(self, filters): + for time_comparison_filter in ['created_at', 'updated_at']: + if time_comparison_filter in filters: + time_filter_dict = {} + comparison_units = filters[time_comparison_filter].split(',') + operators = common.get_time_comparsion_operators() + for comparison_unit in comparison_units: + try: + operator_and_time = comparison_unit.split(":") + comparison_operator = operator_and_time[0] + time = '' + for time_str in operator_and_time[1:-1]: + time += time_str + ":" + time += operator_and_time[-1] + if comparison_operator not in operators: + msg = _( + 'Invalid %s operator') % comparison_operator + raise exc.HTTPBadRequest(explanation=msg) + except IndexError: + msg = _('Invalid %s value') % time_comparison_filter + raise exc.HTTPBadRequest(explanation=msg) + try: + parsed_time = timeutils.parse_isotime(time) + except ValueError: + msg = _('Invalid %s value') % time + raise exc.HTTPBadRequest(explanation=msg) + time_filter_dict[comparison_operator] = parsed_time + + filters[time_comparison_filter] = time_filter_dict + def _get_volumes(self, req, is_detail): """Returns a list of volumes, transformed through view builder.""" @@ -121,6 +156,8 @@ class VolumeController(volumes_v2.VolumeController): if 'name' in filters: filters['display_name'] = filters.pop('name') + self._handle_time_comparison_filters(filters) + strict = req.api_version_request.matches( mv.VOLUME_LIST_BOOTABLE, None) self.volume_api.check_volume_filters(filters, strict) diff --git a/cinder/db/sqlalchemy/api.py b/cinder/db/sqlalchemy/api.py index 327307f5971..1c355f62868 100644 --- a/cinder/db/sqlalchemy/api.py +++ b/cinder/db/sqlalchemy/api.py @@ -413,6 +413,27 @@ def _filter_host(field, value, match_level=None): return or_(*conditions) +def _filter_time_comparison(field, time_filter_dict): + """Generate a filter condition for time comparison operators""" + + conditions = [] + for operator in time_filter_dict: + filter_time = timeutils.normalize_time(time_filter_dict[operator]) + if operator == 'gt': + conditions.append(field.op('>')(filter_time)) + elif operator == 'gte': + conditions.append(field.op('>=')(filter_time)) + if operator == 'eq': + conditions.append(field.op('=')(filter_time)) + elif operator == 'neq': + conditions.append(field.op('!=')(filter_time)) + if operator == 'lt': + conditions.append(field.op('<')(filter_time)) + elif operator == 'lte': + conditions.append(field.op('<=')(filter_time)) + return or_(*conditions) + + def _clustered_bool_field_filter(query, field_name, filter_value): # Now that we have clusters, a service is disabled/frozen if the service # doesn't belong to a cluster or if it belongs to a cluster and the cluster @@ -2433,6 +2454,19 @@ def _process_volume_filters(query, filters): query = query.filter(_filter_host(models.Volume.cluster_name, cluster_name)) + for time_comparison_filter in ['created_at', 'updated_at']: + if filters.get(time_comparison_filter, None): + time_filter_dict = filters.pop(time_comparison_filter) + try: + time_filter_attr = getattr(models.Volume, + time_comparison_filter) + query = query.filter(_filter_time_comparison(time_filter_attr, + time_filter_dict)) + except AttributeError: + LOG.debug("%s column could not be found.", + time_comparison_filter) + return None + # Apply exact match filters for everything else, ensure that the # filter value exists on the model for key in filters.keys(): diff --git a/cinder/tests/unit/api/test_common.py b/cinder/tests/unit/api/test_common.py index e563df89180..242ec2de812 100644 --- a/cinder/tests/unit/api/test_common.py +++ b/cinder/tests/unit/api/test_common.py @@ -477,7 +477,7 @@ class GeneralFiltersTest(test.TestCase): 'expected': ["name", "status", "metadata", "bootable", "migration_status", "availability_zone", "group_id", - "size"]}, + "size", "created_at", "updated_at"]}, {'resource': 'backup', 'expected': ["name", "status", "volume_id"]}, {'resource': 'snapshot', diff --git a/cinder/tests/unit/api/v3/test_volumes.py b/cinder/tests/unit/api/v3/test_volumes.py index 2197503aa46..5ffb54d1db6 100644 --- a/cinder/tests/unit/api/v3/test_volumes.py +++ b/cinder/tests/unit/api/v3/test_volumes.py @@ -20,6 +20,7 @@ import fixtures import iso8601 from oslo_serialization import jsonutils from oslo_utils import strutils +from oslo_utils import timeutils from six.moves import http_client import webob @@ -120,18 +121,27 @@ class VolumeApiTest(test.TestCase): self._reset_filter_file() def _create_volume_with_glance_metadata(self): + basetime = timeutils.utcnow() + td = datetime.timedelta(minutes=1) + vol1 = db.volume_create(self.ctxt, {'display_name': 'test1', + 'created_at': basetime - 3 * td, + 'updated_at': basetime - 2 * td, 'project_id': self.ctxt.project_id, 'volume_type_id': - fake.VOLUME_TYPE_ID}) + fake.VOLUME_TYPE_ID, + 'id': fake.VOLUME_ID}) db.volume_glance_metadata_create(self.ctxt, vol1.id, 'image_name', 'imageTestOne') vol2 = db.volume_create(self.ctxt, {'display_name': 'test2', + 'created_at': basetime - td, + 'updated_at': basetime, 'project_id': self.ctxt.project_id, 'volume_type_id': - fake.VOLUME_TYPE_ID}) + fake.VOLUME_TYPE_ID, + 'id': fake.VOLUME2_ID}) db.volume_glance_metadata_create(self.ctxt, vol2.id, 'image_name', 'imageTestTwo') db.volume_glance_metadata_create(self.ctxt, vol2.id, 'disk_format', @@ -984,3 +994,80 @@ class VolumeApiTest(test.TestCase): self.assertEqual('host1', attachments[0]['host_name']) self.assertEqual('na', attachments[0]['device']) self.assertEqual(att_time, attachments[0]['attached_at']) + + @ddt.data(('created_at=gt:', 0), ('created_at=lt:', 2)) + @ddt.unpack + def test_volume_index_filter_by_created_at_with_gt_and_lt(self, change, + expect_result): + self._create_volume_with_glance_metadata() + change_time = timeutils.utcnow() + datetime.timedelta(minutes=1) + req = fakes.HTTPRequest.blank(("/v3/volumes?%s%s") % + (change, change_time)) + req.environ['cinder.context'] = self.ctxt + req.headers = mv.get_mv_header(mv.VOLUME_TIME_COMPARISON_FILTER) + req.api_version_request = mv.get_api_version( + mv.VOLUME_TIME_COMPARISON_FILTER) + res_dict = self.controller.index(req) + volumes = res_dict['volumes'] + self.assertEqual(expect_result, len(volumes)) + + @ddt.data(('updated_at=gt:', 0), ('updated_at=lt:', 1)) + @ddt.unpack + def test_vol_filter_by_updated_at_with_gt_and_lt(self, change, result): + vols = self._create_volume_with_glance_metadata() + change_time = vols[1].updated_at + req = fakes.HTTPRequest.blank(("/v3/volumes?%s%s") % + (change, change_time)) + req.environ['cinder.context'] = self.ctxt + req.headers = mv.get_mv_header(mv.VOLUME_TIME_COMPARISON_FILTER) + req.api_version_request = mv.get_api_version( + mv.VOLUME_TIME_COMPARISON_FILTER) + res_dict = self.controller.index(req) + volumes = res_dict['volumes'] + self.assertEqual(result, len(volumes)) + + @ddt.data(('updated_at=eq:', 1, fake.VOLUME2_ID), + ('updated_at=neq:', 1, fake.VOLUME_ID)) + @ddt.unpack + def test_vol_filter_by_updated_at_with_eq_and_neq(self, change, result, + expected_volume_id): + vols = self._create_volume_with_glance_metadata() + change_time = vols[1].updated_at + req = fakes.HTTPRequest.blank(("/v3/volumes?%s%s") % + (change, change_time)) + req.environ['cinder.context'] = self.ctxt + req.headers = mv.get_mv_header(mv.VOLUME_TIME_COMPARISON_FILTER) + req.api_version_request = mv.get_api_version( + mv.VOLUME_TIME_COMPARISON_FILTER) + res_dict = self.controller.index(req) + volumes = res_dict['volumes'] + self.assertEqual(result, len(volumes)) + self.assertEqual(expected_volume_id, volumes[0]['id']) + + @ddt.data('created_at', 'updated_at') + def test_volume_filter_by_time_with_invaild_time(self, change): + self._create_volume_with_glance_metadata() + change_time = '123' + req = fakes.HTTPRequest.blank(("/v3/volumes?%s=%s") % + (change, change_time)) + req.environ['cinder.context'] = self.ctxt + req.headers = mv.get_mv_header(mv.VOLUME_TIME_COMPARISON_FILTER) + req.api_version_request = mv.get_api_version( + mv.VOLUME_TIME_COMPARISON_FILTER) + self.assertRaises(webob.exc.HTTPBadRequest, self.controller.index, req) + + def test_volume_index_filter_by_time_with_lte_and_gte(self): + vols = self._create_volume_with_glance_metadata() + change_since = vols[1].updated_at + change_before = timeutils.utcnow() + datetime.timedelta(minutes=1) + req = fakes.HTTPRequest.blank(("/v3/volumes?updated_at=lte:%s&" + "updated_at=gte:%s") % + (change_before, change_since)) + req.environ['cinder.context'] = self.ctxt + req.headers = mv.get_mv_header(mv.VOLUME_TIME_COMPARISON_FILTER) + req.api_version_request = mv.get_api_version( + mv.VOLUME_TIME_COMPARISON_FILTER) + res_dict = self.controller.index(req) + volumes = res_dict['volumes'] + self.assertEqual(1, len(volumes)) + self.assertEqual(vols[1].id, volumes[0]['id']) diff --git a/etc/cinder/resource_filters.json b/etc/cinder/resource_filters.json index 4b50f878413..967361e692d 100644 --- a/etc/cinder/resource_filters.json +++ b/etc/cinder/resource_filters.json @@ -1,7 +1,7 @@ { "volume": ["name", "status", "metadata", "bootable", "migration_status", "availability_zone", - "group_id", "size"], + "group_id", "size", "created_at", "updated_at"], "backup": ["name", "status", "volume_id"], "snapshot": ["name", "status", "volume_id", "metadata", "availability_zone"], diff --git a/releasenotes/notes/support-to-query-cinder-resources-filter-by-update-at-and-created-at-32ae9aaea131d598.yaml b/releasenotes/notes/support-to-query-cinder-resources-filter-by-update-at-and-created-at-32ae9aaea131d598.yaml new file mode 100644 index 00000000000..a007f2a4a88 --- /dev/null +++ b/releasenotes/notes/support-to-query-cinder-resources-filter-by-update-at-and-created-at-32ae9aaea131d598.yaml @@ -0,0 +1,8 @@ +--- +features: + - Beginning with microversion 3.60, users may apply time comparison filters + to the volume summary list and volume detail list requests by using the + "created_at" or "updated_at" fields. Time must be expressed in ISO + 8601 format. See the 'Block Storage API v3 Reference + '_ for + details.