diff --git a/cinder/tests/unit/volume/drivers/dell_emc/unity/test_adapter.py b/cinder/tests/unit/volume/drivers/dell_emc/unity/test_adapter.py index ca9f9e3ce4d..d3d89bdabd5 100644 --- a/cinder/tests/unit/volume/drivers/dell_emc/unity/test_adapter.py +++ b/cinder/tests/unit/volume/drivers/dell_emc/unity/test_adapter.py @@ -118,6 +118,10 @@ class MockClient(object): lun_id += '_low' return test_client.MockResource(_id=lun_id, name=name) + @staticmethod + def lun_has_snapshot(lun): + return lun.name == 'volume_has_snapshot' + @staticmethod def get_lun(name=None, lun_id=None): if lun_id is None: @@ -214,7 +218,9 @@ class MockClient(object): @staticmethod def get_io_limit_policy(specs): - return None + mock_io_policy = (test_client.MockResource(name=specs.get('id')) + if specs else None) + return mock_io_policy @staticmethod def extend_lun(lun_id, size_gib): @@ -257,7 +263,7 @@ class MockClient(object): 'PoolC': 'pool_3'} return pools.get(name, None) - def migrate_lun(self, lun_id, dest_pool_id): + def migrate_lun(self, lun_id, dest_pool_id, provision=None): if dest_pool_id == 'pool_2': return True if dest_pool_id == 'pool_3': @@ -409,6 +415,20 @@ def get_connection_info(adapter, hlu, host, connector): return {} +def get_volume_type_qos_specs(qos_id): + if qos_id == 'qos': + return {'qos_specs': {'id': u'qos_type_id_1', + 'consumer': u'back-end', + u'maxBWS': u'102400', + u'maxIOPS': u'500'}} + if qos_id == 'qos_2': + return {'qos_specs': {'id': u'qos_type_id_2', + 'consumer': u'back-end', + u'maxBWS': u'102402', + u'maxIOPS': u'502'}} + return {'qos_specs': {}} + + def get_volume_type_extra_specs(type_id): if type_id == 'thick': return {'provisioning:type': 'thick', @@ -419,6 +439,9 @@ def get_volume_type_extra_specs(type_id): if type_id == 'tier_lowest': return {'storagetype:tiering': 'LowestAvailable', 'fast_support': ' True'} + if type_id == 'compressed': + return {'provisioning:type': 'compressed', + 'compression_support': ' True'} return {} @@ -439,6 +462,8 @@ def patch_for_unity_adapter(func): new=get_volume_type_extra_specs) @mock.patch('cinder.volume.group_types.get_group_type_specs', new=get_group_type_specs) + @mock.patch('cinder.volume.volume_types.get_volume_type_qos_specs', + new=get_volume_type_qos_specs) @mock.patch('cinder.volume.drivers.dell_emc.unity.utils.' 'get_backend_qos_specs', new=get_backend_qos_specs) @@ -601,6 +626,65 @@ class CommonAdapterTest(test.TestCase): volume = MockOSResource(provider_location='id^lun_4') self.adapter.delete_volume(volume) + @patch_for_unity_adapter + def test_retype_volume_has_snapshot(self): + volume = MockOSResource(name='volume_has_snapshot', size=5, + host='HostA@BackendB#PoolB') + ctxt = None + diff = None + new_type = {'name': u'type01', 'id': 'compressed'} + host = {'host': 'HostA@BackendB#PoolB'} + result = self.adapter.retype(ctxt, volume, new_type, diff, host) + self.assertFalse(result) + + @patch_for_unity_adapter + def test_retype_volume_thick_to_compressed(self): + volume = MockOSResource(name='thick_volume', size=5, + host='HostA@BackendB#PoolA', + provider_location='id^lun_33') + ctxt = None + diff = None + new_type = {'name': u'compressed_type', 'id': 'compressed'} + host = {'host': 'HostA@BackendB#PoolB'} + result = self.adapter.retype(ctxt, volume, new_type, diff, host) + self.assertEqual((True, {}), result) + + @patch_for_unity_adapter + def test_retype_volume_to_compressed(self): + volume = MockOSResource(name='thin_volume', size=5, + host='HostA@BackendB#PoolB') + ctxt = None + diff = None + new_type = {'name': u'compressed_type', 'id': 'compressed'} + host = {'host': 'HostA@BackendB#PoolB'} + result = self.adapter.retype(ctxt, volume, new_type, diff, host) + self.assertTrue(result) + + @patch_for_unity_adapter + def test_retype_volume_to_qos(self): + volume = MockOSResource(name='thin_volume', size=5, + host='HostA@BackendB#PoolB') + ctxt = None + diff = None + new_type = {'name': u'qos_type', 'id': 'qos'} + host = {'host': 'HostA@BackendB#PoolB'} + result = self.adapter.retype(ctxt, volume, new_type, + diff, host) + self.assertTrue(result) + + @patch_for_unity_adapter + def test_retype_volume_revert_qos(self): + volume = MockOSResource(name='qos_volume', size=5, + host='HostA@BackendB#PoolB', + volume_type_id='qos_2') + ctxt = None + diff = None + new_type = {'name': u'no_qos_type', 'id': ''} + host = {'host': 'HostA@BackendB#PoolB'} + result = self.adapter.retype(ctxt, volume, new_type, + diff, host) + self.assertTrue(result) + def test_get_pool_stats(self): stats_list = self.adapter.get_pools_stats() self.assertEqual(1, len(stats_list)) diff --git a/cinder/tests/unit/volume/drivers/dell_emc/unity/test_client.py b/cinder/tests/unit/volume/drivers/dell_emc/unity/test_client.py index e745a62b4d7..55aed654229 100644 --- a/cinder/tests/unit/volume/drivers/dell_emc/unity/test_client.py +++ b/cinder/tests/unit/volume/drivers/dell_emc/unity/test_client.py @@ -59,6 +59,7 @@ class MockResource(object): self.lun = None self.tiering_policy = None self.pool_fast_vp = None + self.snap = True @property def id(self): @@ -199,9 +200,12 @@ class MockResource(object): def storage_resource(self, value): self._storage_resource = value - def modify(self, name=None): + def modify(self, name=None, is_compression=None, io_limit_policy=None): self.name = name + def remove_from_storage(self, lun): + pass + def thin_clone(self, name, io_limit_policy=None, description=None): if name == 'thin_clone_name_in_use': raise ex.UnityLunNameInUseError @@ -213,8 +217,8 @@ class MockResource(object): def restore(self, delete_backup): return MockResource(_id='snap_1', name="internal_snap") - def migrate(self, dest_pool): - if dest_pool.id == 'pool_2': + def migrate(self, dest_pool, **kwargs): + if dest_pool.id == 'fail_migration_pool': return False return True @@ -685,9 +689,17 @@ class ClientTest(unittest.TestCase): self.assertTrue(ret) def test_migrate_lun_failed(self): - ret = self.client.migrate_lun('lun_0', 'pool_2') + ret = self.client.migrate_lun('lun_0', 'fail_migration_pool') self.assertFalse(ret) + def test_migrate_lun_thick(self): + ret = self.client.migrate_lun('lun_thick', 'pool_2', 'thick') + self.assertTrue(ret) + + def test_migrate_lun_compressed(self): + ret = self.client.migrate_lun('lun_compressed', 'pool_2', 'compressed') + self.assertTrue(ret) + def test_get_pool_id_by_name(self): self.assertEqual('pool_3', self.client.get_pool_id_by_name('Pool 3')) diff --git a/cinder/tests/unit/volume/drivers/dell_emc/unity/test_driver.py b/cinder/tests/unit/volume/drivers/dell_emc/unity/test_driver.py index f171d2448d4..7d33f3ed7ca 100644 --- a/cinder/tests/unit/volume/drivers/dell_emc/unity/test_driver.py +++ b/cinder/tests/unit/volume/drivers/dell_emc/unity/test_driver.py @@ -175,6 +175,9 @@ class MockAdapter(object): return group_update, volumes_update return group_update, None + def retype(self, ctxt, volume, new_type, diff, host): + return True + class MockReplicationManager(object): def __init__(self): @@ -318,6 +321,17 @@ class UnityDriverTest(unittest.TestCase): 'HostA@BackendB#PoolC') self.assertEqual((True, {}), ret) + def test_retype_volume(self): + volume = self.get_volume() + new_type = {'name': u'type01', 'qos_specs_id': 'test_qos_id', + 'extra_specs': {}, + 'id': u'd67c4480-a61b-44c0-a58b-24c0357cadeb'} + diff = None + ret = self.driver.retype(self.get_context(), + volume, new_type, diff, + 'HostA@BackendB#PoolC') + self.assertTrue(ret) + def test_create_snapshot(self): snapshot = self.get_snapshot() self.driver.create_snapshot(snapshot) diff --git a/cinder/tests/unit/volume/drivers/dell_emc/unity/test_utils.py b/cinder/tests/unit/volume/drivers/dell_emc/unity/test_utils.py index f7e52c2f386..3e9054b91bd 100644 --- a/cinder/tests/unit/volume/drivers/dell_emc/unity/test_utils.py +++ b/cinder/tests/unit/volume/drivers/dell_emc/unity/test_utils.py @@ -329,3 +329,52 @@ class UnityUtilsTest(unittest.TestCase): cg = test_driver.UnityDriverTest.get_cg() result = utils.get_group_specs(cg, 'test_key') self.assertIsNone(result) + + @patch_volume_types + def test_retype_no_need_migration_when_same_host(self): + volume = test_adapter.MockOSResource(volume_type_id='host_1', + host='host_1') + new_host = {'name': 'new_name', 'host': 'host_1'} + ret = utils.retype_need_migration(volume, None, None, new_host) + self.assertFalse(ret) + + @patch_volume_types + def test_retype_need_migration_when_diff_host(self): + volume = test_adapter.MockOSResource(volume_type_id='host_1', + host='host_1') + new_host = {'name': 'new_name', 'host': 'new_host'} + ret = utils.retype_need_migration(volume, None, None, new_host) + self.assertTrue(ret) + + @patch_volume_types + def test_retype_no_need_migration_thin_to_compressed(self): + volume = test_adapter.MockOSResource(volume_type_id='host_1', + host='host_1') + new_host = {'name': 'new_name', 'host': 'host_1'} + old_provision = '' + new_provision = 'compressed' + ret = utils.retype_need_migration(volume, old_provision, + new_provision, new_host) + self.assertFalse(ret) + + @patch_volume_types + def test_retype_no_need_migration_compressed_to_thin(self): + volume = test_adapter.MockOSResource(volume_type_id='host_1', + host='host_1') + new_host = {'name': 'new_name', 'host': 'host_1'} + old_provision = 'compressed' + new_provision = '' + ret = utils.retype_need_migration(volume, old_provision, + new_provision, new_host) + self.assertFalse(ret) + + @patch_volume_types + def test_retype_need_migration_thin_to_thick(self): + volume = test_adapter.MockOSResource(volume_type_id='host_1', + host='host_1') + new_host = {'name': 'new_name', 'host': 'host_1'} + old_provision = '' + new_provision = 'thick' + ret = utils.retype_need_migration(volume, old_provision, + new_provision, new_host) + self.assertTrue(ret) diff --git a/cinder/volume/drivers/dell_emc/unity/adapter.py b/cinder/volume/drivers/dell_emc/unity/adapter.py index a6ae7f95417..2dab0d1ff57 100644 --- a/cinder/volume/drivers/dell_emc/unity/adapter.py +++ b/cinder/volume/drivers/dell_emc/unity/adapter.py @@ -30,6 +30,7 @@ from cinder.objects import fields from cinder import utils as cinder_utils from cinder.volume.drivers.dell_emc.unity import client from cinder.volume.drivers.dell_emc.unity import utils +from cinder.volume import volume_types from cinder.volume import volume_utils storops = importutils.try_import('storops') @@ -120,7 +121,8 @@ class VolumeParams(object): @property def is_thick(self): if self._is_thick is None: - provision = utils.get_extra_spec(self._volume, 'provisioning:type') + provision = utils.get_extra_spec(self._volume, + utils.PROVISIONING_TYPE) support = utils.get_extra_spec(self._volume, 'thick_provisioning_support') self._is_thick = (provision == 'thick' and support == ' True') @@ -129,10 +131,12 @@ class VolumeParams(object): @property def is_compressed(self): if self._is_compressed is None: - provision = utils.get_extra_spec(self._volume, 'provisioning:type') + provision = utils.get_extra_spec(self._volume, + utils.PROVISIONING_TYPE) compression = utils.get_extra_spec(self._volume, 'compression_support') - if provision == 'compressed' and compression == ' True': + if (provision == utils.PROVISIONING_COMPRESSED and + compression == ' True'): self._is_compressed = True return self._is_compressed @@ -443,6 +447,60 @@ class CommonAdapter(object): else: self.client.delete_lun(lun_id) + def retype(self, ctxt, volume, new_type, diff, host): + """Changes volume from one type to another.""" + old_qos_specs = {utils.QOS_SPECS: None} + old_provision = None + new_specs = volume_types.get_volume_type_extra_specs( + new_type.get(utils.QOS_ID)) + new_qos_specs = volume_types.get_volume_type_qos_specs( + new_type.get(utils.QOS_ID)) + lun = self.client.get_lun(name=volume.name) + volume_type_id = volume.volume_type_id + if volume_type_id: + old_provision = utils.get_extra_spec(volume, + utils.PROVISIONING_TYPE) + old_qos_specs = volume_types.get_volume_type_qos_specs( + volume_type_id) + + need_migration = utils.retype_need_migration( + volume, old_provision, + new_specs.get(utils.PROVISIONING_TYPE), host) + need_change_compress = utils.retype_need_change_compression( + old_provision, new_specs.get(utils.PROVISIONING_TYPE)) + need_change_qos = utils.retype_need_change_qos( + old_qos_specs, new_qos_specs) + + if need_migration or need_change_compress[0] or need_change_qos: + if self.client.lun_has_snapshot(lun): + LOG.warning('Driver is not able to do retype because ' + 'the volume %s has snapshot(s).', + volume.id) + return False + + new_qos_dict = new_qos_specs.get(utils.QOS_SPECS) + if need_change_qos: + new_io_policy = (self.client.get_io_limit_policy(new_qos_dict) + if need_change_qos else None) + # Modify lun to change qos settings + if new_io_policy: + lun.modify(io_limit_policy=new_io_policy) + else: + # remove current qos settings + old_qos_dict = old_qos_specs.get(utils.QOS_SPECS) + old_io_policy = self.client.get_io_limit_policy(old_qos_dict) + old_io_policy.remove_from_storage(lun) + + if need_migration: + LOG.debug('Driver needs to use storage-assisted migration ' + 'to retype the volume.') + return self.migrate_volume(volume, host, new_specs) + + if need_change_compress[0]: + # Modify lun to change compression + lun.modify(is_compression=need_change_compress[1]) + return True + def _create_host_and_attach(self, host_name, lun_or_snap): @utils.lock_if(self.to_lock_host, '{lock_name}') def _lock_helper(lock_name): @@ -908,18 +966,22 @@ class CommonAdapter(object): def restore_snapshot(self, volume, snapshot): return self.client.restore_snapshot(snapshot.name) - def migrate_volume(self, volume, host): + def migrate_volume(self, volume, host, extra_specs=None): """Leverage the Unity move session functionality. This method is invoked at the source backend. + + :param extra_specs: Instance of ExtraSpecs. The new volume will be + changed to align with the new extra specs. """ log_params = { 'name': volume.name, 'src_host': volume.host, - 'dest_host': host['host'] + 'dest_host': host['host'], + 'extra_specs': extra_specs, } LOG.info('Migrate Volume: %(name)s, host: %(src_host)s, destination: ' - '%(dest_host)s', log_params) + '%(dest_host)s, extra_specs: %(extra_specs)s', log_params) src_backend = utils.get_backend_name_from_volume(volume) dest_backend = utils.get_backend_name_from_host(host) @@ -930,10 +992,12 @@ class CommonAdapter(object): return False, None lun_id = self.get_lun_id(volume) + provision = None + if extra_specs: + provision = extra_specs.get(utils.PROVISIONING_TYPE) dest_pool_name = utils.get_pool_name_from_host(host) dest_pool_id = self.get_pool_id_by_name(dest_pool_name) - - if self.client.migrate_lun(lun_id, dest_pool_id): + if self.client.migrate_lun(lun_id, dest_pool_id, provision): LOG.debug('Volume migrated successfully.') model_update = {} return True, model_update diff --git a/cinder/volume/drivers/dell_emc/unity/client.py b/cinder/volume/drivers/dell_emc/unity/client.py index 612d8d2407e..cb891282975 100644 --- a/cinder/volume/drivers/dell_emc/unity/client.py +++ b/cinder/volume/drivers/dell_emc/unity/client.py @@ -177,10 +177,23 @@ class UnityClient(object): lun_id) return lun - def migrate_lun(self, lun_id, dest_pool_id): + def migrate_lun(self, lun_id, dest_pool_id, dest_provision=None): + # dest_provision possible value ('thin', 'thick', 'compressed') lun = self.system.get_lun(lun_id) dest_pool = self.system.get_pool(dest_pool_id) - return lun.migrate(dest_pool) + is_thin = True if dest_provision == 'thin' else None + if dest_provision == 'compressed': + # compressed needs work with thin + is_compressed = True + is_thin = True + else: + is_compressed = False + if dest_provision == 'thick': + # thick needs work with uncompressed + is_thin = False + is_compressed = False + return lun.migrate(dest_pool, is_compressed=is_compressed, + is_thin=is_thin) def get_pools(self): """Gets all storage pools on the Unity system. @@ -236,6 +249,10 @@ class UnityClient(object): {'name': name, 'err': err}) return None + def lun_has_snapshot(self, lun): + snaps = lun.snapshots + return len(snaps) != 0 + @coordination.synchronized('{self.host}-{name}') def create_host(self, name): return self.create_host_wo_lock(name) diff --git a/cinder/volume/drivers/dell_emc/unity/driver.py b/cinder/volume/drivers/dell_emc/unity/driver.py index 3b7464eb533..827644a40de 100644 --- a/cinder/volume/drivers/dell_emc/unity/driver.py +++ b/cinder/volume/drivers/dell_emc/unity/driver.py @@ -83,9 +83,10 @@ class UnityDriver(driver.ManageableVD, 6.1.0 - Support volume replication 7.0.0 - Support tiering policy 7.1.0 - Support consistency group replication + 7.2.0 - Support retype volume """ - VERSION = '07.01.00' + VERSION = '07.02.00' VENDOR = 'Dell EMC' # ThirdPartySystems wiki page CI_WIKI_NAME = "EMC_UNITY_CI" @@ -142,6 +143,10 @@ class UnityDriver(driver.ManageableVD, """Migrates a volume.""" return self.adapter.migrate_volume(volume, host) + def retype(self, ctxt, volume, new_type, diff, host): + """Convert the volume to be of the new type.""" + return self.adapter.retype(ctxt, volume, new_type, diff, host) + def create_snapshot(self, snapshot): """Creates a snapshot.""" self.adapter.create_snapshot(snapshot) diff --git a/cinder/volume/drivers/dell_emc/unity/utils.py b/cinder/volume/drivers/dell_emc/unity/utils.py index a3b83642e5d..d924f6ed1fa 100644 --- a/cinder/volume/drivers/dell_emc/unity/utils.py +++ b/cinder/volume/drivers/dell_emc/unity/utils.py @@ -38,6 +38,11 @@ LOG = logging.getLogger(__name__) BACKEND_QOS_CONSUMERS = frozenset(['back-end', 'both']) QOS_MAX_IOPS = 'maxIOPS' QOS_MAX_BWS = 'maxBWS' +PROVISIONING_TYPE = 'provisioning:type' +PROVISIONING_COMPRESSED = 'compressed' +QOS_SPECS = 'qos_specs' +SPECS_OF_QOS = 'specs' +QOS_ID = 'id' def dump_provider_location(location_dict): @@ -113,6 +118,36 @@ def validate_pool_names(conf_pools, array_pools): return existed +def retype_need_migration(volume, old_provision, new_provision, host): + if volume['host'] != host['host']: + return True + + if old_provision != new_provision: + if retype_need_change_compression(old_provision, new_provision)[0]: + return False + else: + return True + return False + + +def retype_need_change_compression(old_provision, new_provision): + """:return: whether need change compression and the new value""" + if ((not old_provision or old_provision == 'thin') and + new_provision == PROVISIONING_COMPRESSED): + return True, True + elif (old_provision == PROVISIONING_COMPRESSED and + (not new_provision or old_provision == 'thin')): + return True, False + # no need change compression + return False, None + + +def retype_need_change_qos(old_qos=None, new_qos=None): + old = old_qos.get(QOS_SPECS).get(QOS_ID) if old_qos.get(QOS_SPECS) else '' + new = new_qos.get(QOS_SPECS).get(QOS_ID) if new_qos.get(QOS_SPECS) else '' + return old != new + + def extract_iscsi_uids(connector): if 'initiator' not in connector: msg = _("Host %s doesn't have iSCSI initiator.") % connector['host'] @@ -281,7 +316,7 @@ def get_backend_qos_specs(volume): if qos_specs is None: return None - qos_specs = qos_specs['qos_specs'] + qos_specs = qos_specs[QOS_SPECS] if qos_specs is None: return None @@ -290,8 +325,8 @@ def get_backend_qos_specs(volume): if consumer not in BACKEND_QOS_CONSUMERS: return None - max_iops = qos_specs['specs'].get(QOS_MAX_IOPS) - max_bws = qos_specs['specs'].get(QOS_MAX_BWS) + max_iops = qos_specs[SPECS_OF_QOS].get(QOS_MAX_IOPS) + max_bws = qos_specs[SPECS_OF_QOS].get(QOS_MAX_BWS) if max_iops is None and max_bws is None: return None diff --git a/doc/source/configuration/block-storage/drivers/dell-emc-unity-driver.rst b/doc/source/configuration/block-storage/drivers/dell-emc-unity-driver.rst index 0c837d49988..1c77ffaaa44 100644 --- a/doc/source/configuration/block-storage/drivers/dell-emc-unity-driver.rst +++ b/doc/source/configuration/block-storage/drivers/dell-emc-unity-driver.rst @@ -296,12 +296,29 @@ triggered. Instead, host-assisted volume migration will be triggered: the storage-assisted volume migration of vol_2 will not be triggered. +Retype volume support +~~~~~~~~~~~~~~~~~~~~~ + +Unity driver supports to change a volume's type after its creation. + +.. code-block:: console + + $ cinder retype [--migration-policy ] + +The --migration-policy is not enabled by default. +Some retype operations will require migration based on back-end support. +In these cases, the storage-assisted migration will be triggered regardless +the --migration-policy. For examples: retype between 'thin' and 'thick', retype +between 'thick' and 'compressed', retype to type(s) current host doesn't +support. + + QoS support ~~~~~~~~~~~ Unity driver supports ``maxBWS`` and ``maxIOPS`` specs for the back-end -consumer type. ``maxBWS`` represents the ``Maximum IO/S`` absolute limit, -``maxIOPS`` represents the ``Maximum Bandwidth (KBPS)`` absolute limit on the +consumer type. ``maxBWS`` represents the ``Maximum Bandwidth (KBPS)`` absolute +limit, ``maxIOPS`` represents the ``Maximum IO/S`` absolute limit on the Unity respectively. @@ -655,3 +672,4 @@ to track specific Block Storage command logs. # grep "req-3a459e0e-871a-49f9-9796-b63cc48b5015" cinder-volume.log + diff --git a/releasenotes/notes/unity-retype-volume-support-773ae17b8811fb3f.yaml b/releasenotes/notes/unity-retype-volume-support-773ae17b8811fb3f.yaml new file mode 100644 index 00000000000..6554aafdc14 --- /dev/null +++ b/releasenotes/notes/unity-retype-volume-support-773ae17b8811fb3f.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Dell EMC Unity driver: Add efficient retype support when new type uses the same Unity device.