Paginate over usage list to return all usages

since nova api 2.40 the os-simple-tenant-usage API supports pagination
and will by default return a number of entities configured internally in
Nova.
This means that when there are many enough projects,
the single call to usage.list() will not return usages for all projects.

This patch effectively copy-pastes the logic to paginate over usage list
results from novaclient/v2/shell.py code.

Change-Id: I1b639fe386b7b7db3223f6965495094b9d51533a
Story: #2005099
Task: #29713

openstackclient/compute/v2/usage.py

@@ -15,8 +15,10 @@
 
 """Usage action implementations"""
 
+import collections
 import datetime
 
+from novaclient import api_versions
 from osc_lib.command import command
 from osc_lib import utils
 import six
@@ -24,6 +26,36 @@ import six
 from openstackclient.i18n import _
 
 
+def _get_usage_marker(usage):
+    marker = None
+    if hasattr(usage, 'server_usages') and usage.server_usages:
+        marker = usage.server_usages[-1]['instance_id']
+    return marker
+
+
+def _get_usage_list_marker(usage_list):
+    marker = None
+    if usage_list:
+        marker = _get_usage_marker(usage_list[-1])
+    return marker
+
+
+def _merge_usage(usage, next_usage):
+    usage.server_usages.extend(next_usage.server_usages)
+    usage.total_hours += next_usage.total_hours
+    usage.total_memory_mb_usage += next_usage.total_memory_mb_usage
+    usage.total_vcpus_usage += next_usage.total_vcpus_usage
+    usage.total_local_gb_usage += next_usage.total_local_gb_usage
+
+
+def _merge_usage_list(usages, next_usage_list):
+    for next_usage in next_usage_list:
+        if next_usage.tenant_id in usages:
+            _merge_usage(usages[next_usage.tenant_id], next_usage)
+        else:
+            usages[next_usage.tenant_id] = next_usage
+
+
 class ListUsage(command.Lister):
     _description = _("List resource usage per project")
 
@@ -83,7 +115,23 @@ class ListUsage(command.Lister):
         else:
             end = now + datetime.timedelta(days=1)
 
-        usage_list = compute_client.usage.list(start, end, detailed=True)
+        if compute_client.api_version < api_versions.APIVersion("2.40"):
+            usage_list = compute_client.usage.list(start, end, detailed=True)
+        else:
+            # If the number of instances used to calculate the usage is greater
+            # than CONF.api.max_limit, the usage will be split across multiple
+            # requests and the responses will need to be merged back together.
+            usages = collections.OrderedDict()
+            usage_list = compute_client.usage.list(start, end, detailed=True)
+            _merge_usage_list(usages, usage_list)
+            marker = _get_usage_list_marker(usage_list)
+            while marker:
+                next_usage_list = compute_client.usage.list(
+                    start, end, detailed=True, marker=marker)
+                marker = _get_usage_list_marker(next_usage_list)
+                if marker:
+                    _merge_usage_list(usages, next_usage_list)
+            usage_list = list(usages.values())
 
         # Cache the project list
         project_cache = {}

openstackclient/tests/unit/compute/v2/fakes.py

@@ -1304,6 +1304,7 @@ class FakeUsage(object):
                     'local_gb': 1,
                     'memory_mb': 512,
                     'name': 'usage-name-' + uuid.uuid4().hex,
+                    'instance_id': uuid.uuid4().hex,
                     'state': 'active',
                     'uptime': 3600,
                     'vcpus': 1

openstackclient/tests/unit/compute/v2/test_usage.py

@@ -14,6 +14,7 @@
 import datetime
 
 import mock
+from novaclient import api_versions
 
 from openstackclient.compute.v2 import usage
 from openstackclient.tests.unit.compute.v2 import fakes as compute_fakes
@@ -104,6 +105,31 @@ class TestUsageList(TestUsage):
         self.assertEqual(self.columns, columns)
         self.assertEqual(tuple(self.data), tuple(data))
 
+    def test_usage_list_with_pagination(self):
+        arglist = []
+        verifylist = [
+            ('start', None),
+            ('end', None),
+        ]
+
+        self.app.client_manager.compute.api_version = api_versions.APIVersion(
+            '2.40')
+        self.usage_mock.list.reset_mock()
+        self.usage_mock.list.side_effect = [self.usages, []]
+
+        parsed_args = self.check_parser(self.cmd, arglist, verifylist)
+
+        columns, data = self.cmd.take_action(parsed_args)
+
+        self.projects_mock.list.assert_called_with()
+        self.usage_mock.list.assert_has_calls([
+            mock.call(mock.ANY, mock.ANY, detailed=True),
+            mock.call(mock.ANY, mock.ANY, detailed=True,
+                      marker=self.usages[0]['server_usages'][0]['instance_id'])
+        ])
+        self.assertEqual(self.columns, columns)
+        self.assertEqual(tuple(self.data), tuple(data))
+
 
 class TestUsageShow(TestUsage):