Reduce page_size if too large response size for search api

For search api, if response size is too large, exception with
error_code 60576 is returned. Catch this kind of exception
and retry with smaller page_size.

Change-Id: If4340b7688420aabc673635f600c1e4b33aa4de3

vmware_nsxlib/tests/unit/v3/test_resources.py

@@ -1919,6 +1919,25 @@ class TestNsxSearch(nsxlib_testcase.NsxClientTestCase):
                           self.nsxlib._build_query,
                           tags=user_tags)
 
+    def test_nsx_search_all_search_pipeline_error(self):
+        """Test search all base method."""
+        mock_search = mock.Mock()
+        mock_search.side_effect = [exceptions.NsxSearchPipelineError,
+                                   exceptions.NsxSearchPipelineError,
+                                   {"cursor": "0",
+                                    "result_count": 2,
+                                    "results": [{"id": "s1"}, {"id": "s2"}]},
+                                   {"results": []}]
+        args = "foo"
+        kwargs = {"a1": "v1", "a2": "v2"}
+        results = self.nsxlib._search_all(mock_search, *args, **kwargs)
+        mock_search.assert_has_calls([
+            mock.call(*args, cursor=0, page_size=None, **kwargs),
+            mock.call(*args, cursor=0, page_size=900, **kwargs),
+            mock.call(*args, cursor=0, page_size=800, **kwargs)])
+        self.assertEqual(2, len(results))
+        self.assertEqual([{"id": "s1"}, {"id": "s2"}], results)
+
     def test_nsx_search_all(self):
         """Test search all base method."""
         mock_search = mock.Mock()
@@ -1933,8 +1952,8 @@ class TestNsxSearch(nsxlib_testcase.NsxClientTestCase):
         kwargs = {"a1": "v1", "a2": "v2"}
         results = self.nsxlib._search_all(mock_search, *args, **kwargs)
         mock_search.assert_has_calls([
-            mock.call(*args, cursor=0, **kwargs),
-            mock.call(*args, cursor=2, **kwargs)])
+            mock.call(*args, cursor=0, page_size=None, **kwargs),
+            mock.call(*args, cursor=2, page_size=None, **kwargs)])
         self.assertEqual(3, len(results))
         self.assertEqual([{"id": "s1"}, {"id": "s2"}, {"id": "s3"}], results)
 

vmware_nsxlib/v3/client.py

@@ -85,6 +85,7 @@ def http_error_to_exception(status_code, error_code, related_error_codes=None):
             {'60508': exceptions.NsxIndexingInProgress,
              '60514': exceptions.NsxSearchTimeout,
              '60515': exceptions.NsxSearchOutOfSync,
+             '60576': exceptions.NsxSearchPipelineError,
              '8327': exceptions.NsxOverlapVlan,
              '500045': exceptions.NsxPendingDelete,
              '500030': exceptions.ResourceInUse,

vmware_nsxlib/v3/exceptions.py

@@ -207,6 +207,10 @@ class NsxSearchOutOfSync(NsxSearchError):
     message = _("Index is currently out of sync")
 
 
+class NsxSearchPipelineError(NsxLibException):
+    message = _("Exception while searching during pipeline execution")
+
+
 class NsxPendingDelete(NsxLibException):
     message = _("An object with the same name is marked for deletion. Either "
                 "use another path or wait for the purge cycle to permanently "

vmware_nsxlib/v3/lib.py

@@ -269,8 +269,19 @@ class NsxLibBase(object, metaclass=abc.ABCMeta):
     def _search_all(self, search_func, *args, **kwargs):
         results = []
         cursor = 0
+        page_size = None
         while True:
-            response = search_func(*args, cursor=cursor, **kwargs)
+            try:
+                response = search_func(*args, cursor=cursor,
+                                       page_size=page_size, **kwargs)
+            except exceptions.NsxSearchPipelineError:
+                if page_size is None:
+                    page_size = 1000
+                page_size -= 100
+                if page_size < 0:
+                    # Set page_size to the minimum value 1
+                    page_size = 1
+                continue
             if not response['results']:
                 return results
             results.extend(response['results'])