Merge "PowerMax Driver - U4P failover lock not released on exception"

cinder/tests/unit/volume/drivers/dell_emc/powermax/test_powermax_rest.py

@@ -86,6 +86,19 @@ class PowerMaxRestTest(test.TestCase):
                 self.rest.request('/fake_uri', 'GET')
                 mock_fail.assert_called_once()
 
+    @mock.patch.object(time, 'sleep')
+    def test_rest_request_failover_escape(self, mck_sleep):
+        self.rest.u4p_failover_lock = True
+        response = tpfo.FakeResponse(200, 'Success')
+        with mock.patch.object(self.rest, '_handle_u4p_failover')as mock_fail:
+            with mock.patch.object(self.rest.session, 'request',
+                                   side_effect=[requests.ConnectionError,
+                                                response]):
+                self.rest.u4p_failover_enabled = True
+                self.rest.request('/fake_uri', 'GET')
+                mock_fail.assert_called_once()
+        self.rest.u4p_failover_lock = False
+
     def test_wait_for_job_complete(self):
         rc, job, status, task = self.rest.wait_for_job_complete(
             {'status': 'created', 'jobId': '12345'}, self.data.extra_specs)

cinder/volume/drivers/dell_emc/powermax/rest.py

@@ -39,6 +39,7 @@ U4V_VERSION = '91'
 MIN_U4P_VERSION = '9.1.0.14'
 UCODE_5978 = '5978'
 retry_exc_tuple = (exception.VolumeBackendAPIException,)
+u4p_failover_max_wait = 120
 # HTTP constants
 GET = 'GET'
 POST = 'POST'
@@ -199,11 +200,17 @@ class PowerMaxRest(object):
         :raises: VolumeBackendAPIException, Timeout, ConnectionError,
                  HTTPError, SSLError
         """
-        while self.u4p_failover_lock and not retry:
+        waiting_time = 0
+        while self.u4p_failover_lock and not retry and (
+                waiting_time < u4p_failover_max_wait):
             LOG.warning("Unisphere failover lock in process, holding request "
                         "until lock is released when Unisphere connection "
                         "re-established.")
-            time.sleep(10)
+            sleeptime = 10
+            time.sleep(sleeptime)
+            waiting_time += sleeptime
+            if waiting_time >= u4p_failover_max_wait:
+                self.u4p_failover_lock = False
 
         url, message, status_code, response = None, None, None, None
         if not self.session:
@@ -240,6 +247,8 @@ class PowerMaxRest(object):
                           "received is: %(status_code)s", {
                               'status_code': status_code})
                 message = None
+                if retry:
+                    self.u4p_failover_lock = False
 
             LOG.debug("%(method)s request to %(url)s has returned with "
                       "a status code of: %(status_code)s.", {
@@ -247,6 +256,8 @@ class PowerMaxRest(object):
                           'status_code': status_code})
 
         except r_exc.SSLError as e:
+            if retry:
+                self.u4p_failover_lock = False
             msg = _("The connection to %(base_uri)s has encountered an "
                     "SSL error. Please check your SSL config or supplied "
                     "SSL cert in Cinder configuration. SSL Exception "
@@ -279,6 +290,8 @@ class PowerMaxRest(object):
                                       'exc_msg': e})
 
         except Exception as e:
+            if retry:
+                self.u4p_failover_lock = False
             msg = _("The %(method)s request to URL %(url)s failed with "
                     "exception %(e)s")
             LOG.error(msg, {'method': method, 'url': url,