openstack/shade
Code Issues Proposed changes

Retry to fetch paginated volumes if we get 404 for next link

Browse Source 
When we get a volume list, it's possible for a volume to disappear
causing the pagination to bork. When that happens we retry to
get the list from scratch for 5 times. If all the attempts fail
we just return what we found.

Change-Id: Ia88d1d8a6b3558f7d5d364a9c78bbf834836d3f7
Signed-off-by: Rosario Di Somma <rosario.disomma@dreamhost.com>
This commit is contained in:
Rosario Di Somma 2017-06-16 18:13:15 +00:00
parent 77b2a441cf
commit 6b325282d3
2 changed files with 130 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
shade/openstackcloud.py
View File

@ -1700,21 +1700,53 @@ class OpenStackCloud(
"""
def _list(data):
volumes.extend(meta.get_and_munchify('volumes', data))
volumes.extend(data.get('volumes', []))
endpoint = None
for l in data.get('volumes_links', []):
if 'rel' in l and 'next' == l['rel']:
endpoint = l['href']
break
if endpoint:
_list(self._volume_client.get(endpoint))
try:
_list(self._volume_client.get(endpoint))
except OpenStackCloudURINotFound:
# Catch and re-raise here because we are making recursive
# calls and we just have context for the log here
self.log.debug(
"While listing volumes, could not find next link"
" {link}.".format(link=data))
raise
if not cache:
warnings.warn('cache argument to list_volumes is deprecated. Use '
'invalidate instead.')
volumes = []
_list(self._volume_client.get('/volumes/detail'))
return self._normalize_volumes(volumes)
# Fetching paginated volumes can fails for several reasons, if
# something goes wrong we'll have to start fetching volumes from
# scratch
attempts = 5
for _ in range(attempts):
volumes = []
data = self._volume_client.get('/volumes/detail')
if 'volumes_links' not in data:
# no pagination needed
volumes.extend(data.get('volumes', []))
break
try:
_list(data)
break
except OpenStackCloudURINotFound:
pass
else:
self.log.debug(
"List volumes failed to retrieve all volumes after"
" {attempts} attempts. Returning what we found.".format(
attempts=attempts))
# list volumes didn't complete succesfully so just return what
# we found
return self._normalize_volumes(
meta.get_and_munchify(key=None, data=volumes))
@_utils.cache_on_arguments()
def list_volume_types(self, get_extra=True):

93
shade/tests/unit/test_volume.py
View File

@ -319,3 +319,96 @@ class TestVolume(base.RequestsMockTestCase):
self.cloud._normalize_volume(vol2)],
self.cloud.list_volumes())
self.assert_calls()
def test_list_volumes_with_pagination_next_link_fails_once(self):
vol1 = meta.obj_to_munch(fakes.FakeVolume('01', 'available', 'vol1'))
vol2 = meta.obj_to_munch(fakes.FakeVolume('02', 'available', 'vol2'))
self.register_uris([
dict(method='GET',
uri=self.get_mock_url(
'volumev2', 'public',
append=['volumes', 'detail']),
json={
'volumes': [vol1],
'volumes_links': [
{'href': self.get_mock_url(
'volumev2', 'public',
append=['volumes', 'detail'],
qs_elements=['marker=01']),
'rel': 'next'}]}),
dict(method='GET',
uri=self.get_mock_url(
'volumev2', 'public',
append=['volumes', 'detail'],
qs_elements=['marker=01']),
status_code=404),
dict(method='GET',
uri=self.get_mock_url(
'volumev2', 'public',
append=['volumes', 'detail']),
json={
'volumes': [vol1],
'volumes_links': [
{'href': self.get_mock_url(
'volumev2', 'public',
append=['volumes', 'detail'],
qs_elements=['marker=01']),
'rel': 'next'}]}),
dict(method='GET',
uri=self.get_mock_url(
'volumev2', 'public',
append=['volumes', 'detail'],
qs_elements=['marker=01']),
json={
'volumes': [vol2],
'volumes_links': [
{'href': self.get_mock_url(
'volumev2', 'public',
append=['volumes', 'detail'],
qs_elements=['marker=02']),
'rel': 'next'}]}),
dict(method='GET',
uri=self.get_mock_url(
'volumev2', 'public',
append=['volumes', 'detail'],
qs_elements=['marker=02']),
json={'volumes': []})])
self.assertEqual(
[self.cloud._normalize_volume(vol1),
self.cloud._normalize_volume(vol2)],
self.cloud.list_volumes())
self.assert_calls()
def test_list_volumes_with_pagination_next_link_fails_all_attempts(self):
vol1 = meta.obj_to_munch(fakes.FakeVolume('01', 'available', 'vol1'))
uris = []
attempts = 5
for i in range(attempts):
uris.extend([
dict(method='GET',
uri=self.get_mock_url(
'volumev2', 'public',
append=['volumes', 'detail']),
json={
'volumes': [vol1],
'volumes_links': [
{'href': self.get_mock_url(
'volumev2', 'public',
append=['volumes', 'detail'],
qs_elements=['marker=01']),
'rel': 'next'}]}),
dict(method='GET',
uri=self.get_mock_url(
'volumev2', 'public',
append=['volumes', 'detail'],
qs_elements=['marker=01']),
status_code=404)])
self.register_uris(uris)
# Check that found volumes are returned even if pagination didn't
# complete because call to get next link 404'ed for all the allowed
# attempts
self.assertEqual(
[self.cloud._normalize_volume(vol1)],
self.cloud.list_volumes())
self.assert_calls()