resource: add max_items parameter to resource

Currently the limit option is used ambiguously,
the first is the total amount of resources returned,
and the second is the size of the page.

To separate this, we add a parameter called max_items,
which is the total amount of resources returned.

Change-Id: I56fdb5ef480da6bca82462ce3ca4e0cbcb2830de
Signed-off-by: djp <dimsss0607@gmail.com>

openstack/dns/v2/_base.py

@@ -123,4 +123,5 @@ class Resource(resource.Resource):
             next_link = uri
             params['marker'] = marker
             params['limit'] = limit
+
         return next_link, params

openstack/resource.py

@@ -1956,6 +1956,7 @@ class Resource(dict):
         *,
         microversion: str | None = None,
         headers: dict[str, str] | None = None,
+        max_items: int | None = None,
         **params: ty.Any,
     ) -> ty.Generator[ty_ext.Self, None, None]:
         """This method is a generator which yields resource objects.
@@ -1978,6 +1979,8 @@ class Resource(dict):
         :param str microversion: API version to override the negotiated one.
         :param dict headers: Additional headers to inject into the HTTP
             request.
+        :param int max_items: The maximum number of items to return. Typically
+            this must be used with ``paginated=True``.
         :param dict params: These keyword arguments are passed through the
             :meth:`~openstack.resource.QueryParamter._transpose` method
             to find if any of them match expected query parameters to be sent
@@ -2028,6 +2031,17 @@ class Resource(dict):
         uri = base_path % params
         uri_params = {}
 
+        if max_items and not query_params.get('limit'):
+            # If a user requested max_items but not a limit, set limit to
+            # max_items on the assumption that if (a) the value is smaller than
+            # the maximum server allowed value for limit then we'll be able to
+            # do a single call to get everything, while (b) if the value is
+            # larger then the server will ignore the value (or rather use its
+            # own hardcoded limit) making this is a no-op.
+            # If a user requested both max_items and limit then we assume they
+            # know what they're doing.
+            query_params['limit'] = max_items
+
         limit = query_params.get('limit')
 
         for k, v in params.items():
@@ -2079,6 +2093,11 @@ class Resource(dict):
 
             marker = None
             for raw_resource in resources:
+                # We return as soon as we hit our limit, even if we have items
+                # remaining
+                if max_items and total_yielded >= max_items:
+                    return
+
                 # Do not allow keys called "self" through. Glance chose
                 # to name a key "self", so we need to pop it out because
                 # we can't send it through cls.existing and into the
@@ -2136,7 +2155,7 @@ class Resource(dict):
     @classmethod
     def _get_next_link(cls, uri, response, data, marker, limit, total_yielded):
         next_link = None
-        params = {}
+        params: dict[str, str | list[str] | int] = {}
 
         if isinstance(data, dict):
             pagination_key = cls.pagination_key

openstack/tests/unit/test_resource.py

@@ -2313,6 +2313,85 @@ class TestResourceActions(base.TestCase):
         self.assertEqual(2, len(self.session.get.call_args_list))
         self.assertIsInstance(results[0], Test)
 
+    def test_list_response_paginated_with_max_items(self):
+        """Test pagination with a 'max_items' in the response.
+
+        The limit variable is used in two meanings.
+        To make it clear, we add the max_items parameter and
+        use this value to determine the number of resources to be returned.
+        """
+        ids = [1, 2, 3, 4]
+
+        def make_mock_response():
+            resp = mock.Mock()
+            resp.status_code = 200
+            resp.links = {}
+            resp.json.return_value = {
+                "resources": [
+                    {"id": 1},
+                    {"id": 2},
+                    {"id": 3},
+                    {"id": 4},
+                ],
+            }
+            return resp
+
+        self.session.get.side_effect = [
+            make_mock_response(),
+            make_mock_response(),
+            make_mock_response(),
+        ]
+
+        # Since the limit value is 3 but the max_items value is 2, two resources are returned.
+        results = self.sot.list(
+            self.session, limit=3, paginated=True, max_items=2
+        )
+
+        result0 = next(results)
+        self.assertEqual(result0.id, ids[0])
+        result1 = next(results)
+        self.assertEqual(result1.id, ids[1])
+        self.session.get.assert_called_with(
+            self.base_path,
+            headers={"Accept": "application/json"},
+            params={"limit": 3},
+            microversion=None,
+        )
+        self.assertRaises(StopIteration, next, results)
+
+        # max_items is set and limit in unset (so limit defaults to max_items)
+        results = self.sot.list(self.session, paginated=True, max_items=2)
+        result0 = next(results)
+        self.assertEqual(result0.id, ids[0])
+        result1 = next(results)
+        self.assertEqual(result1.id, ids[1])
+        self.session.get.assert_called_with(
+            self.base_path,
+            headers={"Accept": "application/json"},
+            params={"limit": 2},
+            microversion=None,
+        )
+        self.assertRaises(StopIteration, next, results)
+
+        # both max_items and limit are set, and max_items is greater than limit
+        # (the opposite of this test: we should see multiple requests for limit resources each time)
+        results = self.sot.list(
+            self.session, limit=1, paginated=True, max_items=3
+        )
+        result0 = next(results)
+        self.assertEqual(result0.id, ids[0])
+        result1 = next(results)
+        self.assertEqual(result1.id, ids[1])
+        result2 = next(results)
+        self.assertEqual(result2.id, ids[2])
+        self.session.get.assert_called_with(
+            self.base_path,
+            headers={"Accept": "application/json"},
+            params={"limit": 1},
+            microversion=None,
+        )
+        self.assertRaises(StopIteration, next, results)
+
     def test_list_response_paginated_with_microversions(self):
         class Test(resource.Resource):
             service = self.service_name
@@ -2812,7 +2891,6 @@ class TestResourceActions(base.TestCase):
         self.session.get.side_effect = [resp1, resp2]
 
         results = self.sot.list(self.session, limit=2, paginated=True)
-
         # Get the first page's two items
         result0 = next(results)
         self.assertEqual(result0.id, ids[0])

releasenotes/notes/add_max_item_parameter-3ab3c2e1cd2312c5.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    A new parameter, ``max_items``, is added to the ``Resource.list``
+    method. This allows users to specify the maximum number of resources
+    that should be returned to the user, as opposed to the maximum number
+    of items that should be requested from the server in a single request.
+    The latter is already handled by the ``limit`` parameter