diff --git a/openstack/dns/v2/_base.py b/openstack/dns/v2/_base.py index 6b2d8804a..16aa81079 100644 --- a/openstack/dns/v2/_base.py +++ b/openstack/dns/v2/_base.py @@ -123,4 +123,5 @@ class Resource(resource.Resource): next_link = uri params['marker'] = marker params['limit'] = limit + return next_link, params diff --git a/openstack/resource.py b/openstack/resource.py index 91186d805..0196e6197 100644 --- a/openstack/resource.py +++ b/openstack/resource.py @@ -1956,6 +1956,7 @@ class Resource(dict): *, microversion: str | None = None, headers: dict[str, str] | None = None, + max_items: int | None = None, **params: ty.Any, ) -> ty.Generator[ty_ext.Self, None, None]: """This method is a generator which yields resource objects. @@ -1978,6 +1979,8 @@ class Resource(dict): :param str microversion: API version to override the negotiated one. :param dict headers: Additional headers to inject into the HTTP request. + :param int max_items: The maximum number of items to return. Typically + this must be used with ``paginated=True``. :param dict params: These keyword arguments are passed through the :meth:`~openstack.resource.QueryParamter._transpose` method to find if any of them match expected query parameters to be sent @@ -2028,6 +2031,17 @@ class Resource(dict): uri = base_path % params uri_params = {} + if max_items and not query_params.get('limit'): + # If a user requested max_items but not a limit, set limit to + # max_items on the assumption that if (a) the value is smaller than + # the maximum server allowed value for limit then we'll be able to + # do a single call to get everything, while (b) if the value is + # larger then the server will ignore the value (or rather use its + # own hardcoded limit) making this is a no-op. + # If a user requested both max_items and limit then we assume they + # know what they're doing. + query_params['limit'] = max_items + limit = query_params.get('limit') for k, v in params.items(): @@ -2079,6 +2093,11 @@ class Resource(dict): marker = None for raw_resource in resources: + # We return as soon as we hit our limit, even if we have items + # remaining + if max_items and total_yielded >= max_items: + return + # Do not allow keys called "self" through. Glance chose # to name a key "self", so we need to pop it out because # we can't send it through cls.existing and into the @@ -2136,7 +2155,7 @@ class Resource(dict): @classmethod def _get_next_link(cls, uri, response, data, marker, limit, total_yielded): next_link = None - params = {} + params: dict[str, str | list[str] | int] = {} if isinstance(data, dict): pagination_key = cls.pagination_key diff --git a/openstack/tests/unit/test_resource.py b/openstack/tests/unit/test_resource.py index bd3b36ae7..0ca99ae25 100644 --- a/openstack/tests/unit/test_resource.py +++ b/openstack/tests/unit/test_resource.py @@ -2313,6 +2313,85 @@ class TestResourceActions(base.TestCase): self.assertEqual(2, len(self.session.get.call_args_list)) self.assertIsInstance(results[0], Test) + def test_list_response_paginated_with_max_items(self): + """Test pagination with a 'max_items' in the response. + + The limit variable is used in two meanings. + To make it clear, we add the max_items parameter and + use this value to determine the number of resources to be returned. + """ + ids = [1, 2, 3, 4] + + def make_mock_response(): + resp = mock.Mock() + resp.status_code = 200 + resp.links = {} + resp.json.return_value = { + "resources": [ + {"id": 1}, + {"id": 2}, + {"id": 3}, + {"id": 4}, + ], + } + return resp + + self.session.get.side_effect = [ + make_mock_response(), + make_mock_response(), + make_mock_response(), + ] + + # Since the limit value is 3 but the max_items value is 2, two resources are returned. + results = self.sot.list( + self.session, limit=3, paginated=True, max_items=2 + ) + + result0 = next(results) + self.assertEqual(result0.id, ids[0]) + result1 = next(results) + self.assertEqual(result1.id, ids[1]) + self.session.get.assert_called_with( + self.base_path, + headers={"Accept": "application/json"}, + params={"limit": 3}, + microversion=None, + ) + self.assertRaises(StopIteration, next, results) + + # max_items is set and limit in unset (so limit defaults to max_items) + results = self.sot.list(self.session, paginated=True, max_items=2) + result0 = next(results) + self.assertEqual(result0.id, ids[0]) + result1 = next(results) + self.assertEqual(result1.id, ids[1]) + self.session.get.assert_called_with( + self.base_path, + headers={"Accept": "application/json"}, + params={"limit": 2}, + microversion=None, + ) + self.assertRaises(StopIteration, next, results) + + # both max_items and limit are set, and max_items is greater than limit + # (the opposite of this test: we should see multiple requests for limit resources each time) + results = self.sot.list( + self.session, limit=1, paginated=True, max_items=3 + ) + result0 = next(results) + self.assertEqual(result0.id, ids[0]) + result1 = next(results) + self.assertEqual(result1.id, ids[1]) + result2 = next(results) + self.assertEqual(result2.id, ids[2]) + self.session.get.assert_called_with( + self.base_path, + headers={"Accept": "application/json"}, + params={"limit": 1}, + microversion=None, + ) + self.assertRaises(StopIteration, next, results) + def test_list_response_paginated_with_microversions(self): class Test(resource.Resource): service = self.service_name @@ -2812,7 +2891,6 @@ class TestResourceActions(base.TestCase): self.session.get.side_effect = [resp1, resp2] results = self.sot.list(self.session, limit=2, paginated=True) - # Get the first page's two items result0 = next(results) self.assertEqual(result0.id, ids[0]) diff --git a/releasenotes/notes/add_max_item_parameter-3ab3c2e1cd2312c5.yaml b/releasenotes/notes/add_max_item_parameter-3ab3c2e1cd2312c5.yaml new file mode 100644 index 000000000..5eb60a84f --- /dev/null +++ b/releasenotes/notes/add_max_item_parameter-3ab3c2e1cd2312c5.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + A new parameter, ``max_items``, is added to the ``Resource.list`` + method. This allows users to specify the maximum number of resources + that should be returned to the user, as opposed to the maximum number + of items that should be requested from the server in a single request. + The latter is already handled by the ``limit`` parameter