PowerMax Driver - Failover abilities legacy improvements

Change failover commands to run against the remote array
instead of the primary array. Add in addtitional validation
to prevent failback attempts if the primary array is no
longer available on the Unisphere instance. Part one of
the failover abilities feature updates which is split
into legacy changes and new functionality.

Change-Id: I755acd0c56158557f53b23f8ba7361c621172428
Partially-implements: blueprint powermax-failover-abilities

cinder/tests/unit/volume/drivers/dell_emc/powermax/powermax_data.py

@@ -230,6 +230,13 @@ class PowerMaxData(object):
         volume_type=test_volume_type, host=fake_host,
         replication_driver_data=six.text_type(provider_location3))
 
+    test_rep_volume = fake_volume.fake_volume_obj(
+        context=ctx, name='vol1', size=2, provider_auth=None,
+        provider_location=six.text_type(provider_location),
+        volume_type=test_volume_type, host=fake_host,
+        replication_driver_data=six.text_type(provider_location3),
+        replication_status=fields.ReplicationStatus.ENABLED)
+
     test_attached_volume = fake_volume.fake_volume_obj(
         id='4732de9b-98a4-4b6d-ae4b-3cafb3d34220', context=ctx, name='vol1',
         size=0, provider_auth=None, attach_status='attached',

cinder/tests/unit/volume/drivers/dell_emc/powermax/test_powermax_common.py

@@ -2293,6 +2293,21 @@ class PowerMaxCommonTest(test.TestCase):
         device_ids = self.common._get_volume_device_ids(volumes, array)
         self.assertEqual(ref_device_ids, device_ids)
 
+    @mock.patch.object(common.PowerMaxCommon, '_find_device_on_array',
+                       return_value=tpd.PowerMaxData.device_id)
+    def test_get_volume_device_ids_remote_volumes(self, mck_find):
+        array = self.data.array
+        volumes = [self.data.test_rep_volume]
+        ref_device_ids = [self.data.device_id]
+        replication_details = ast.literal_eval(
+            self.data.test_rep_volume.replication_driver_data)
+        remote_array = replication_details.get(utils.ARRAY)
+        specs = {utils.ARRAY: remote_array}
+        device_ids = self.common._get_volume_device_ids(volumes, array, True)
+        self.assertEqual(ref_device_ids, device_ids)
+        mck_find.assert_called_once_with(
+            self.data.test_rep_volume, specs, True)
+
     def test_get_members_of_volume_group(self):
         array = self.data.array
         group_name = self.data.storagegroup_name_source
@@ -2542,6 +2557,32 @@ class PowerMaxCommonTest(test.TestCase):
             self.common._delete_group(group, volumes)
             mock_clone_chk.assert_called_once()
 
+    @mock.patch.object(rest.PowerMaxRest, 'delete_storage_group')
+    @mock.patch.object(common.PowerMaxCommon, '_failover_replication',
+                       return_value=(True, None))
+    @mock.patch.object(masking.PowerMaxMasking, 'add_volumes_to_storage_group')
+    @mock.patch.object(common.PowerMaxCommon, '_get_volume_device_ids',
+                       return_value=[tpd.PowerMaxData.device_id])
+    @mock.patch.object(provision.PowerMaxProvision, 'create_volume_group')
+    @mock.patch.object(common.PowerMaxCommon, '_initial_setup',
+                       return_value=tpd.PowerMaxData.ex_specs_rep_config_sync)
+    def test_update_volume_list_from_sync_vol_list(
+            self, mck_setup, mck_grp, mck_ids, mck_add, mck_fover, mck_del):
+        vol_list = [self.data.test_rep_volume]
+        vol_ids = [self.data.device_id]
+        remote_array = self.data.remote_array
+        temp_group = 'OS-23_24_007-temp-rdf-sg'
+        extra_specs = self.data.ex_specs_rep_config_sync
+        self.common._update_volume_list_from_sync_vol_list(vol_list, None)
+        mck_grp.assert_called_once_with(remote_array, temp_group, extra_specs)
+        mck_ids.assert_called_once_with(
+            vol_list, remote_array, remote_volumes=True)
+        mck_add.assert_called_once_with(
+            remote_array, vol_ids, temp_group, extra_specs)
+        mck_fover.assert_called_once_with(
+            vol_list, None, temp_group, secondary_backend_id=None, host=True)
+        mck_del.assert_called_once_with(remote_array, temp_group)
+
     @mock.patch.object(
         common.PowerMaxCommon, '_remove_vol_and_cleanup_replication')
     @mock.patch.object(

cinder/tests/unit/volume/drivers/dell_emc/powermax/test_powermax_replication.py

@@ -254,20 +254,134 @@ class PowerMaxReplicationTest(test.TestCase):
                               self.common.get_rdf_details, self.data.array,
                               self.data.rep_config_sync)
 
-    @mock.patch.object(common.PowerMaxCommon, '_sync_check')
+    @mock.patch.object(
-    def test_failover_host(self, mck_sync):
+        common.PowerMaxCommon, '_populate_volume_and_group_update_lists',
+        return_value=('vol_list', 'group_list'))
+    @mock.patch.object(utils.PowerMaxUtils, 'validate_failover_request',
+                       return_value=(True, 'val'))
+    @mock.patch.object(rest.PowerMaxRest, 'get_arrays_list',
+                       return_value=['123'])
+    def test_failover_host(self, mck_arrays, mck_validate, mck_populate):
         volumes = [self.data.test_volume, self.data.test_clone_volume]
-        with mock.patch.object(self.common, '_failover_replication',
+        groups = [self.data.test_group]
-                               return_value=(None, {})) as mock_fo:
+        backend_id = self.data.rep_backend_id_sync
-            self.common.failover_host(volumes)
+        rep_configs = self.common.rep_configs
-            mock_fo.assert_called_once()
+        secondary_id, volume_update_list, group_update_list = (
+            self.common.failover_host(volumes, backend_id, groups))
+        mck_validate.assert_called_once_with(
+            False, backend_id, rep_configs, self.data.array, ['123'])
+        mck_populate.assert_called_once_with(volumes, groups, None)
+        self.assertEqual(backend_id, secondary_id)
+        self.assertEqual('vol_list', volume_update_list)
+        self.assertEqual('group_list', group_update_list)
 
+    @mock.patch.object(utils.PowerMaxUtils, 'validate_failover_request',
+                       return_value=(False, 'val'))
+    @mock.patch.object(rest.PowerMaxRest, 'get_arrays_list',
+                       return_value=['123'])
+    def test_failover_host_invalid(self, mck_arrays, mck_validate):
+        volumes = [self.data.test_volume, self.data.test_clone_volume]
+        backend_id = self.data.rep_backend_id_sync
+        rep_configs = self.common.rep_configs
+        self.assertRaises(exception.InvalidReplicationTarget,
+                          self.common.failover_host, volumes, backend_id)
+        mck_validate.assert_called_once_with(
+            False, backend_id, rep_configs, self.data.array, ['123'])
+
+    @mock.patch.object(common.PowerMaxCommon,
+                       '_update_volume_list_from_sync_vol_list',
+                       return_value={'vol_updates'})
+    @mock.patch.object(common.PowerMaxCommon, '_initial_setup',
+                       return_value=tpd.PowerMaxData.ex_specs_rep_config_sync)
     @mock.patch.object(common.PowerMaxCommon, 'failover_replication',
-                       return_value=({}, {}))
+                       return_value=('grp_updates', {'grp_vol_updates'}))
-    def test_failover_host_groups(self, mock_fg):
+    def test_populate_volume_and_group_update_lists(
+            self, mck_failover_rep, mck_setup, mck_from_sync):
+        test_volume = deepcopy(self.data.test_volume)
+        test_volume.group_id = self.data.test_rep_group.id
+        volumes = [test_volume, self.data.test_rep_volume]
+        groups = [self.data.test_rep_group]
+        group_volumes = [test_volume]
+        volume_updates, group_updates = (
+            self.common._populate_volume_and_group_update_lists(
+                volumes, groups, None))
+        mck_failover_rep.assert_called_once_with(
+            None, groups[0], group_volumes, None, host=True)
+        mck_setup.assert_called_once_with(self.data.test_rep_volume)
+        mck_from_sync.assert_called_once_with(
+            [self.data.test_rep_volume], None)
+        vol_updates_ref = ['grp_vol_updates', 'vol_updates']
+        self.assertEqual(vol_updates_ref, volume_updates)
+        group_updates_ref = [{'group_id': test_volume.group_id,
+                              'updates': 'grp_updates'}]
+        self.assertEqual(group_updates_ref, group_updates)
+
+    def test_failover_replication_empty_group(self):
+        with mock.patch.object(volume_utils, 'is_group_a_type',
+                               return_value=True):
+            model_update, __ = self.common.failover_replication(
+                None, self.data.test_group, [])
+            self.assertEqual({}, model_update)
+
+    @mock.patch.object(rest.PowerMaxRest, 'srdf_failover_group',
+                       return_value=tpd.PowerMaxData.rdf_group_no_1)
+    @mock.patch.object(common.PowerMaxCommon, 'get_rdf_details',
+                       return_value=tpd.PowerMaxData.rdf_group_no_1)
+    @mock.patch.object(common.PowerMaxCommon, '_find_volume_group',
+                       return_value=tpd.PowerMaxData.test_group)
+    def test_failover_replication_failover(self, mck_find_vol_grp,
+                                           mck_get_rdf_grp, mck_failover):
         volumes = [self.data.test_volume_group_member]
-        group1 = self.data.test_group
+        vol_group = self.data.test_group
-        self.common.failover_host(volumes, None, [group1])
+        vol_grp_name = self.data.test_group.name
+        model_update, __ = self.common._failover_replication(
+            volumes, vol_group, vol_grp_name, host=True)
+        self.assertEqual(fields.ReplicationStatus.FAILED_OVER,
+                         model_update['replication_status'])
+
+    @mock.patch.object(rest.PowerMaxRest, 'srdf_failover_group',
+                       return_value=tpd.PowerMaxData.rdf_group_no_1)
+    @mock.patch.object(common.PowerMaxCommon, 'get_rdf_details',
+                       return_value=tpd.PowerMaxData.rdf_group_no_1)
+    @mock.patch.object(common.PowerMaxCommon, '_find_volume_group',
+                       return_value=tpd.PowerMaxData.test_group)
+    def test_failover_replication_failback(self, mck_find_vol_grp,
+                                           mck_get_rdf_grp, mck_failover):
+        volumes = [self.data.test_volume_group_member]
+        vol_group = self.data.test_group
+        vol_grp_name = self.data.test_group.name
+        model_update, __ = self.common._failover_replication(
+            volumes, vol_group, vol_grp_name, host=True,
+            secondary_backend_id='default')
+        self.assertEqual(fields.ReplicationStatus.ENABLED,
+                         model_update['replication_status'])
+
+    @mock.patch.object(common.PowerMaxCommon, 'get_rdf_details',
+                       return_value=None)
+    @mock.patch.object(common.PowerMaxCommon, '_find_volume_group',
+                       return_value=tpd.PowerMaxData.test_group)
+    def test_failover_replication_exception(self, mck_find_vol_grp,
+                                            mck_get_rdf_grp):
+        volumes = [self.data.test_volume_group_member]
+        vol_group = self.data.test_group
+        vol_grp_name = self.data.test_group.name
+        model_update, __ = self.common._failover_replication(
+            volumes, vol_group, vol_grp_name)
+        self.assertEqual(fields.ReplicationStatus.ERROR,
+                         model_update['replication_status'])
+
+    @mock.patch.object(common.PowerMaxCommon, '_failover_replication',
+                       return_value=({}, {}))
+    @mock.patch.object(common.PowerMaxCommon, '_sync_check')
+    @mock.patch.object(rest.PowerMaxRest, 'get_arrays_list',
+                       return_value=['123'])
+    def test_failover_host_async(self, mck_arrays, mck_sync, mock_fg):
+        volumes = [self.data.test_volume]
+        extra_specs = deepcopy(self.extra_specs)
+        extra_specs['rep_mode'] = utils.REP_ASYNC
+        with mock.patch.object(common.PowerMaxCommon, '_initial_setup',
+                               return_value=extra_specs):
+            self.async_driver.common.failover_host(volumes, None, [])
         mock_fg.assert_called_once()
 
     @mock.patch.object(rest.PowerMaxRest,
@@ -444,63 +558,6 @@ class PowerMaxReplicationTest(test.TestCase):
             self.assertEqual(fields.ReplicationStatus.ERROR,
                              model_update['replication_status'])
 
-    def test_failover_replication_empty_group(self):
-        with mock.patch.object(volume_utils, 'is_group_a_type',
-                               return_value=True):
-            model_update, __ = self.common.failover_replication(
-                None, self.data.test_group, [])
-            self.assertEqual({}, model_update)
-
-    @mock.patch.object(rest.PowerMaxRest, 'srdf_failover_group',
-                       return_value=tpd.PowerMaxData.rdf_group_no_1)
-    @mock.patch.object(common.PowerMaxCommon, 'get_rdf_details',
-                       return_value=tpd.PowerMaxData.rdf_group_no_1)
-    @mock.patch.object(common.PowerMaxCommon, '_find_volume_group',
-                       return_value=tpd.PowerMaxData.test_group)
-    def test_failover_replication_failover(self, mck_find_vol_grp,
-                                           mck_get_rdf_grp, mck_failover):
-        volumes = [self.data.test_volume_group_member]
-        vol_group = self.data.test_group
-        vol_grp_name = self.data.test_group.name
-
-        model_update, __ = self.common._failover_replication(
-            volumes, vol_group, vol_grp_name, host=True)
-        self.assertEqual(fields.ReplicationStatus.FAILED_OVER,
-                         model_update['replication_status'])
-
-    @mock.patch.object(rest.PowerMaxRest, 'srdf_failover_group',
-                       return_value=tpd.PowerMaxData.rdf_group_no_1)
-    @mock.patch.object(common.PowerMaxCommon, 'get_rdf_details',
-                       return_value=tpd.PowerMaxData.rdf_group_no_1)
-    @mock.patch.object(common.PowerMaxCommon, '_find_volume_group',
-                       return_value=tpd.PowerMaxData.test_group)
-    def test_failover_replication_failback(self, mck_find_vol_grp,
-                                           mck_get_rdf_grp, mck_failover):
-        volumes = [self.data.test_volume_group_member]
-        vol_group = self.data.test_group
-        vol_grp_name = self.data.test_group.name
-
-        model_update, __ = self.common._failover_replication(
-            volumes, vol_group, vol_grp_name, host=True,
-            secondary_backend_id='default')
-        self.assertEqual(fields.ReplicationStatus.ENABLED,
-                         model_update['replication_status'])
-
-    @mock.patch.object(common.PowerMaxCommon, 'get_rdf_details',
-                       return_value=None)
-    @mock.patch.object(common.PowerMaxCommon, '_find_volume_group',
-                       return_value=tpd.PowerMaxData.test_group)
-    def test_failover_replication_exception(self, mck_find_vol_grp,
-                                            mck_get_rdf_grp):
-        volumes = [self.data.test_volume_group_member]
-        vol_group = self.data.test_group
-        vol_grp_name = self.data.test_group.name
-
-        model_update, __ = self.common._failover_replication(
-            volumes, vol_group, vol_grp_name)
-        self.assertEqual(fields.ReplicationStatus.ERROR,
-                         model_update['replication_status'])
-
     @mock.patch.object(utils.PowerMaxUtils, 'get_volumetype_extra_specs',
                        return_value={utils.REPLICATION_DEVICE_BACKEND_ID:
                                      tpd.PowerMaxData.rep_backend_id_sync})
@@ -552,18 +609,6 @@ class PowerMaxReplicationTest(test.TestCase):
             [self.data.device_id], self.extra_specs, self.data.rep_config_sync)
         mock_rm.assert_called_once()
 
-    @mock.patch.object(common.PowerMaxCommon, '_failover_replication',
-                       return_value=({}, {}))
-    @mock.patch.object(common.PowerMaxCommon, '_sync_check')
-    def test_failover_host_async(self, mck_sync, mock_fg):
-        volumes = [self.data.test_volume]
-        extra_specs = deepcopy(self.extra_specs)
-        extra_specs['rep_mode'] = utils.REP_ASYNC
-        with mock.patch.object(common.PowerMaxCommon, '_initial_setup',
-                               return_value=extra_specs):
-            self.async_driver.common.failover_host(volumes, None, [])
-        mock_fg.assert_called_once()
-
     @mock.patch.object(
         common.PowerMaxCommon, 'get_volume_metadata', return_value={})
     @mock.patch.object(

cinder/tests/unit/volume/drivers/dell_emc/powermax/test_powermax_rest.py

@@ -231,6 +231,20 @@ class PowerMaxRestTest(test.TestCase):
                           self.rest.modify_resource, array, category,
                           resource_type, resource_name)
 
+    def test_get_arrays_list(self):
+        ret_val = {'symmetrixId': tpd.PowerMaxData.array}
+        with mock.patch.object(self.rest, 'get_request',
+                               return_value=ret_val):
+            ref_details = self.data.array
+            array_details = self.rest.get_arrays_list()
+            self.assertEqual(ref_details, array_details)
+
+    def test_get_arrays_list_failed(self):
+        with mock.patch.object(self.rest, 'get_request',
+                               return_value=dict()):
+            array_details = self.rest.get_arrays_list()
+            self.assertEqual(list(), array_details)
+
     def test_get_array_detail(self):
         ref_details = self.data.symmetrix[0]
         array_details = self.rest.get_array_detail(self.data.array)

cinder/tests/unit/volume/drivers/dell_emc/powermax/test_powermax_utils.py

@@ -1367,8 +1367,11 @@ class PowerMaxUtilsTest(test.TestCase):
         is_failed_over = False
         failover_backend_id = self.data.rep_backend_id_sync
         rep_configs = self.data.multi_rep_config_list
+        primary_array = self.data.array
+        array_list = [self.data.array]
         is_valid, msg = self.utils.validate_failover_request(
-            is_failed_over, failover_backend_id, rep_configs)
+            is_failed_over, failover_backend_id, rep_configs,
+            primary_array, array_list)
         self.assertTrue(is_valid)
         self.assertEqual("", msg)
 
@@ -1376,20 +1379,42 @@ class PowerMaxUtilsTest(test.TestCase):
         is_failed_over = True
         failover_backend_id = self.data.rep_backend_id_sync
         rep_configs = self.data.multi_rep_config_list
+        primary_array = self.data.array
+        array_list = [self.data.array]
         is_valid, msg = self.utils.validate_failover_request(
-            is_failed_over, failover_backend_id, rep_configs)
+            is_failed_over, failover_backend_id, rep_configs,
+            primary_array, array_list)
         self.assertFalse(is_valid)
         expected_msg = ('Cannot failover, the backend is already in a failed '
                         'over state, if you meant to failback, please add '
                         '--backend_id default to the command.')
         self.assertEqual(expected_msg, msg)
 
+    def test_validate_failover_request_failback_missing_array(self):
+        is_failed_over = True
+        failover_backend_id = 'default'
+        rep_configs = self.data.multi_rep_config_list
+        primary_array = self.data.array
+        array_list = [self.data.remote_array]
+        is_valid, msg = self.utils.validate_failover_request(
+            is_failed_over, failover_backend_id, rep_configs,
+            primary_array, array_list)
+        self.assertFalse(is_valid)
+        expected_msg = ('Cannot failback, the configured primary array is '
+                        'not currently available to perform failback to. '
+                        'Please ensure array %s is visible in '
+                        'Unisphere.') % primary_array
+        self.assertEqual(expected_msg, msg)
+
     def test_validate_failover_request_invalid_failback(self):
         is_failed_over = False
         failover_backend_id = 'default'
         rep_configs = self.data.multi_rep_config_list
+        primary_array = self.data.array
+        array_list = [self.data.array]
         is_valid, msg = self.utils.validate_failover_request(
-            is_failed_over, failover_backend_id, rep_configs)
+            is_failed_over, failover_backend_id, rep_configs,
+            primary_array, array_list)
         self.assertFalse(is_valid)
         expected_msg = ('Cannot failback, backend is not in a failed over '
                         'state. If you meant to failover, please either omit '
@@ -1401,8 +1426,11 @@ class PowerMaxUtilsTest(test.TestCase):
         is_failed_over = False
         failover_backend_id = None
         rep_configs = self.data.multi_rep_config_list
+        primary_array = self.data.array
+        array_list = [self.data.array]
         is_valid, msg = self.utils.validate_failover_request(
-            is_failed_over, failover_backend_id, rep_configs)
+            is_failed_over, failover_backend_id, rep_configs,
+            primary_array, array_list)
         self.assertFalse(is_valid)
         expected_msg = ('Cannot failover, no backend_id provided while '
                         'multiple replication devices are defined in '
@@ -1415,9 +1443,12 @@ class PowerMaxUtilsTest(test.TestCase):
         is_failed_over = False
         failover_backend_id = 'invalid_id'
         rep_configs = self.data.multi_rep_config_list
+        primary_array = self.data.array
+        array_list = [self.data.array]
         self.assertRaises(exception.InvalidInput,
                           self.utils.validate_failover_request,
-                          is_failed_over, failover_backend_id, rep_configs)
+                          is_failed_over, failover_backend_id, rep_configs,
+                          primary_array, array_list)
 
     def test_validate_replication_group_config_success(self):
         rep_configs = deepcopy(self.data.multi_rep_config_list)

cinder/volume/drivers/dell_emc/powermax/common.py

@@ -1562,11 +1562,12 @@ class PowerMaxCommon(object):
                 backend_id = volume_backend_id
         return backend_id
 
-    def _find_device_on_array(self, volume, extra_specs):
+    def _find_device_on_array(self, volume, extra_specs, remote_device=False):
         """Given the volume get the PowerMax/VMAX device Id.
 
         :param volume: volume object
         :param extra_specs: the extra Specs
+        :param remote_device: find remote device for replicated volumes
         :returns: array, device_id
         """
         founddevice_id = None
@@ -1575,7 +1576,11 @@ class PowerMaxCommon(object):
             name_id = volume._name_id
         except AttributeError:
             name_id = None
-        loc = volume.provider_location
+
+        if remote_device:
+            loc = volume.replication_driver_data
+        else:
+            loc = volume.provider_location
 
         if isinstance(loc, six.string_types):
             name = ast.literal_eval(loc)
@@ -4958,8 +4963,12 @@ class PowerMaxCommon(object):
         :returns: secondary_id, volume_update_list, group_update_list
         :raises: VolumeBackendAPIException
         """
+        primary_array = self._get_configuration_value(
+            utils.VMAX_ARRAY, utils.POWERMAX_ARRAY)
+        array_list = self.rest.get_arrays_list()
         is_valid, msg = self.utils.validate_failover_request(
-            self.failover, secondary_id, self.rep_configs)
+            self.failover, secondary_id, self.rep_configs, primary_array,
+            array_list)
         if not is_valid:
             LOG.error(msg)
             raise exception.InvalidReplicationTarget(msg)
@@ -5012,9 +5021,6 @@ class PowerMaxCommon(object):
             extra_specs = self._initial_setup(volume)
             extra_specs[utils.ARRAY] = array
             if self.utils.is_replication_enabled(extra_specs):
-                device_id = self._find_device_on_array(volume, extra_specs)
-                self._sync_check(array, device_id, extra_specs)
-
                 rep_mode = extra_specs.get(utils.REP_MODE, utils.REP_SYNC)
                 backend_id = self._get_replicated_volume_backend_id(
                     volume)
@@ -5073,21 +5079,23 @@ class PowerMaxCommon(object):
         :returns: vol_updates
         """
         extra_specs = self._initial_setup(sync_vol_list[0])
-        array = ast.literal_eval(
+        replication_details = ast.literal_eval(
-            sync_vol_list[0].provider_location)['array']
+            sync_vol_list[0].replication_driver_data)
-        extra_specs[utils.ARRAY] = array
+        remote_array = replication_details.get(utils.ARRAY)
+        extra_specs[utils.ARRAY] = remote_array
         temp_grp_name = self.utils.get_temp_failover_grp_name(
             extra_specs[utils.REP_CONFIG])
         self.provision.create_volume_group(
-            array, temp_grp_name, extra_specs)
+            remote_array, temp_grp_name, extra_specs)
-        device_ids = self._get_volume_device_ids(sync_vol_list, array)
+        device_ids = self._get_volume_device_ids(
+            sync_vol_list, remote_array, remote_volumes=True)
         self.masking.add_volumes_to_storage_group(
-            array, device_ids, temp_grp_name, extra_specs)
+            remote_array, device_ids, temp_grp_name, extra_specs)
         __, vol_updates = (
             self._failover_replication(
                 sync_vol_list, None, temp_grp_name,
                 secondary_backend_id=group_fo, host=True))
-        self.rest.delete_storage_group(array, temp_grp_name)
+        self.rest.delete_storage_group(remote_array, temp_grp_name)
         return vol_updates
 
     def _get_replication_extra_specs(self, extra_specs, rep_config):
@@ -5711,16 +5719,26 @@ class PowerMaxCommon(object):
             remote_array, remote_device_list, group_name, extra_specs)
         LOG.info("Removed volumes from remote volume group.")
 
-    def _get_volume_device_ids(self, volumes, array):
+    def _get_volume_device_ids(self, volumes, array, remote_volumes=False):
         """Get volume device ids from volume.
 
         :param volumes: volume objects
+        :param array: array id
+        :param remote_volumes: get the remote ids for replicated volumes
         :returns: device_ids
         """
         device_ids = []
         for volume in volumes:
-            specs = {utils.ARRAY: array}
+            if remote_volumes:
-            device_id = self._find_device_on_array(volume, specs)
+                replication_details = ast.literal_eval(
+                    volume.replication_driver_data)
+                remote_array = replication_details.get(utils.ARRAY)
+                specs = {utils.ARRAY: remote_array}
+                device_id = self._find_device_on_array(
+                    volume, specs, remote_volumes)
+            else:
+                specs = {utils.ARRAY: array}
+                device_id = self._find_device_on_array(volume, specs)
             if device_id is None:
                 LOG.error("Volume %(name)s not found on the array.",
                           {'name': volume['name']})
@@ -6131,15 +6149,17 @@ class PowerMaxCommon(object):
             return model_update, vol_model_updates
 
         extra_specs = self._initial_setup(volumes[0])
-        array = ast.literal_eval(volumes[0].provider_location)['array']
+        replication_details = ast.literal_eval(
-        extra_specs[utils.ARRAY] = array
+            volumes[0].replication_driver_data)
+        remote_array = replication_details.get(utils.ARRAY)
+        extra_specs[utils.ARRAY] = remote_array
         failover = False if secondary_backend_id == 'default' else True
 
         try:
             rdf_group_no, __ = self.get_rdf_details(
-                array, extra_specs[utils.REP_CONFIG])
+                remote_array, extra_specs[utils.REP_CONFIG])
             if group:
-                volume_group = self._find_volume_group(array, group)
+                volume_group = self._find_volume_group(remote_array, group)
                 if volume_group:
                     if 'name' in volume_group:
                         vol_grp_name = volume_group['name']
@@ -6149,10 +6169,10 @@ class PowerMaxCommon(object):
             if not is_metro:
                 if failover:
                     self.rest.srdf_failover_group(
-                        array, vol_grp_name, rdf_group_no, extra_specs)
+                        remote_array, vol_grp_name, rdf_group_no, extra_specs)
                 else:
                     self.rest.srdf_failback_group(
-                        array, vol_grp_name, rdf_group_no, extra_specs)
+                        remote_array, vol_grp_name, rdf_group_no, extra_specs)
 
             if failover:
                 model_update.update({
@@ -6183,7 +6203,8 @@ class PowerMaxCommon(object):
                 self.volume_metadata.capture_failover_volume(
                     vol, local['device_id'], local['array'], rdf_group_no,
                     remote['device_id'], remote['array'], extra_specs,
-                    failover, vol_grp_name, vol_rep_status, utils.REP_ASYNC)
+                    failover, vol_grp_name, vol_rep_status,
+                    extra_specs[utils.REP_MODE])
 
             update = {'id': vol.id,
                       'replication_status': vol_rep_status,

cinder/volume/drivers/dell_emc/powermax/fc.py

@@ -128,6 +128,7 @@ class PowerMaxFCDriver(san.SanDriver, driver.FibreChannelDriver):
                 (bp powermax-port-load-balance)
               - Fix to enable legacy volumes to live migrate (#1867163)
               - Use of snap id instead of generation (bp powermax-snapset-ids)
+              - Support for Failover Abilities (bp/powermax-failover-abilities)
     """
 
     VERSION = "4.3.0"

cinder/volume/drivers/dell_emc/powermax/iscsi.py

@@ -134,6 +134,7 @@ class PowerMaxISCSIDriver(san.SanISCSIDriver):
                 (bp powermax-port-load-balance)
               - Fix to enable legacy volumes to live migrate (#1867163)
               - Use of snap id instead of generation (bp powermax-snapset-ids)
+              - Support for Failover Abilities (bp/powermax-failover-abilities)
     """
 
     VERSION = "4.3.0"

cinder/volume/drivers/dell_emc/powermax/rest.py

@@ -692,6 +692,18 @@ class PowerMaxRest(object):
         operation = 'delete %(res)s resource' % {'res': resource_type}
         self.check_status_code_success(operation, status_code, message)
 
+    def get_arrays_list(self):
+        """Get a list of all arrays on U4P instance.
+
+        :returns arrays -- list
+        """
+        target_uri = '/%s/sloprovisioning/symmetrix' % U4V_VERSION
+        array_details = self.get_request(target_uri, 'sloprovisioning')
+        if not array_details:
+            LOG.error("Could not get array details from Unisphere instance.")
+        arrays = array_details.get('symmetrixId', list())
+        return arrays
+
     def get_array_detail(self, array):
         """Get an array from its serial number.
 

cinder/volume/drivers/dell_emc/powermax/utils.py

@@ -1834,7 +1834,7 @@ class PowerMaxUtils(object):
         return list(replication_targets)
 
     def validate_failover_request(self, is_failed_over, failover_backend_id,
-                                  rep_configs):
+                                  rep_configs, primary_array, arrays_list):
         """Validate failover_host request's parameters
 
         Validate that a failover_host operation can be performed with
@@ -1843,6 +1843,8 @@ class PowerMaxUtils(object):
         :param is_failed_over: current failover state
         :param failover_backend_id: backend_id given during failover request
         :param rep_configs: backend rep_configs -- list
+        :param primary_array: configured primary array SID -- string
+        :param arrays_list: list of U4P symmetrix IDs -- list
         :return: (bool, str) is valid, reason on invalid
         """
         is_valid = True
@@ -1853,6 +1855,13 @@ class PowerMaxUtils(object):
                 msg = _('Cannot failover, the backend is already in a failed '
                         'over state, if you meant to failback, please add '
                         '--backend_id default to the command.')
+            elif primary_array not in arrays_list:
+                is_valid = False
+                msg = _('Cannot failback, the configured primary array is '
+                        'not currently available to perform failback to. '
+                        'Please ensure array %s is visible in '
+                        'Unisphere.') % primary_array
+
         else:
             if failover_backend_id == 'default':
                 is_valid = False

releasenotes/notes/powermax-failover-abilities-1fa0a23128f1c00b.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    PowerMax for Cinder driver now supports the ability to transition to a
+    new primary array as part of the failover process if the existing
+    primary array is deemed unrecoverable.