Merge "[Unity] Retype volume support"

cinder/tests/unit/volume/drivers/dell_emc/unity/test_adapter.py

@@ -118,6 +118,10 @@ class MockClient(object):
                 lun_id += '_low'
         return test_client.MockResource(_id=lun_id, name=name)
 
+    @staticmethod
+    def lun_has_snapshot(lun):
+        return lun.name == 'volume_has_snapshot'
+
     @staticmethod
     def get_lun(name=None, lun_id=None):
         if lun_id is None:
@@ -214,7 +218,9 @@ class MockClient(object):
 
     @staticmethod
     def get_io_limit_policy(specs):
-        return None
+        mock_io_policy = (test_client.MockResource(name=specs.get('id'))
+                          if specs else None)
+        return mock_io_policy
 
     @staticmethod
     def extend_lun(lun_id, size_gib):
@@ -257,7 +263,7 @@ class MockClient(object):
                  'PoolC': 'pool_3'}
         return pools.get(name, None)
 
-    def migrate_lun(self, lun_id, dest_pool_id):
+    def migrate_lun(self, lun_id, dest_pool_id, provision=None):
         if dest_pool_id == 'pool_2':
             return True
         if dest_pool_id == 'pool_3':
@@ -409,6 +415,20 @@ def get_connection_info(adapter, hlu, host, connector):
     return {}
 
 
+def get_volume_type_qos_specs(qos_id):
+    if qos_id == 'qos':
+        return {'qos_specs': {'id': u'qos_type_id_1',
+                              'consumer': u'back-end',
+                              u'maxBWS': u'102400',
+                              u'maxIOPS': u'500'}}
+    if qos_id == 'qos_2':
+        return {'qos_specs': {'id': u'qos_type_id_2',
+                              'consumer': u'back-end',
+                              u'maxBWS': u'102402',
+                              u'maxIOPS': u'502'}}
+    return {'qos_specs': {}}
+
+
 def get_volume_type_extra_specs(type_id):
     if type_id == 'thick':
         return {'provisioning:type': 'thick',
@@ -419,6 +439,9 @@ def get_volume_type_extra_specs(type_id):
     if type_id == 'tier_lowest':
         return {'storagetype:tiering': 'LowestAvailable',
                 'fast_support': '<is> True'}
+    if type_id == 'compressed':
+        return {'provisioning:type': 'compressed',
+                'compression_support': '<is> True'}
     return {}
 
 
@@ -439,6 +462,8 @@ def patch_for_unity_adapter(func):
                 new=get_volume_type_extra_specs)
     @mock.patch('cinder.volume.group_types.get_group_type_specs',
                 new=get_group_type_specs)
+    @mock.patch('cinder.volume.volume_types.get_volume_type_qos_specs',
+                new=get_volume_type_qos_specs)
     @mock.patch('cinder.volume.drivers.dell_emc.unity.utils.'
                 'get_backend_qos_specs',
                 new=get_backend_qos_specs)
@@ -601,6 +626,65 @@ class CommonAdapterTest(test.TestCase):
         volume = MockOSResource(provider_location='id^lun_4')
         self.adapter.delete_volume(volume)
 
+    @patch_for_unity_adapter
+    def test_retype_volume_has_snapshot(self):
+        volume = MockOSResource(name='volume_has_snapshot', size=5,
+                                host='HostA@BackendB#PoolB')
+        ctxt = None
+        diff = None
+        new_type = {'name': u'type01', 'id': 'compressed'}
+        host = {'host': 'HostA@BackendB#PoolB'}
+        result = self.adapter.retype(ctxt, volume, new_type, diff, host)
+        self.assertFalse(result)
+
+    @patch_for_unity_adapter
+    def test_retype_volume_thick_to_compressed(self):
+        volume = MockOSResource(name='thick_volume', size=5,
+                                host='HostA@BackendB#PoolA',
+                                provider_location='id^lun_33')
+        ctxt = None
+        diff = None
+        new_type = {'name': u'compressed_type', 'id': 'compressed'}
+        host = {'host': 'HostA@BackendB#PoolB'}
+        result = self.adapter.retype(ctxt, volume, new_type, diff, host)
+        self.assertEqual((True, {}), result)
+
+    @patch_for_unity_adapter
+    def test_retype_volume_to_compressed(self):
+        volume = MockOSResource(name='thin_volume', size=5,
+                                host='HostA@BackendB#PoolB')
+        ctxt = None
+        diff = None
+        new_type = {'name': u'compressed_type', 'id': 'compressed'}
+        host = {'host': 'HostA@BackendB#PoolB'}
+        result = self.adapter.retype(ctxt, volume, new_type, diff, host)
+        self.assertTrue(result)
+
+    @patch_for_unity_adapter
+    def test_retype_volume_to_qos(self):
+        volume = MockOSResource(name='thin_volume', size=5,
+                                host='HostA@BackendB#PoolB')
+        ctxt = None
+        diff = None
+        new_type = {'name': u'qos_type', 'id': 'qos'}
+        host = {'host': 'HostA@BackendB#PoolB'}
+        result = self.adapter.retype(ctxt, volume, new_type,
+                                     diff, host)
+        self.assertTrue(result)
+
+    @patch_for_unity_adapter
+    def test_retype_volume_revert_qos(self):
+        volume = MockOSResource(name='qos_volume', size=5,
+                                host='HostA@BackendB#PoolB',
+                                volume_type_id='qos_2')
+        ctxt = None
+        diff = None
+        new_type = {'name': u'no_qos_type', 'id': ''}
+        host = {'host': 'HostA@BackendB#PoolB'}
+        result = self.adapter.retype(ctxt, volume, new_type,
+                                     diff, host)
+        self.assertTrue(result)
+
     def test_get_pool_stats(self):
         stats_list = self.adapter.get_pools_stats()
         self.assertEqual(1, len(stats_list))

cinder/tests/unit/volume/drivers/dell_emc/unity/test_client.py

@@ -59,6 +59,7 @@ class MockResource(object):
         self.lun = None
         self.tiering_policy = None
         self.pool_fast_vp = None
+        self.snap = True
 
     @property
     def id(self):
@@ -199,9 +200,12 @@ class MockResource(object):
     def storage_resource(self, value):
         self._storage_resource = value
 
-    def modify(self, name=None):
+    def modify(self, name=None, is_compression=None, io_limit_policy=None):
         self.name = name
 
+    def remove_from_storage(self, lun):
+        pass
+
     def thin_clone(self, name, io_limit_policy=None, description=None):
         if name == 'thin_clone_name_in_use':
             raise ex.UnityLunNameInUseError
@@ -213,8 +217,8 @@ class MockResource(object):
     def restore(self, delete_backup):
         return MockResource(_id='snap_1', name="internal_snap")
 
-    def migrate(self, dest_pool):
+    def migrate(self, dest_pool, **kwargs):
-        if dest_pool.id == 'pool_2':
+        if dest_pool.id == 'fail_migration_pool':
             return False
         return True
 
@@ -685,9 +689,17 @@ class ClientTest(unittest.TestCase):
         self.assertTrue(ret)
 
     def test_migrate_lun_failed(self):
-        ret = self.client.migrate_lun('lun_0', 'pool_2')
+        ret = self.client.migrate_lun('lun_0', 'fail_migration_pool')
         self.assertFalse(ret)
 
+    def test_migrate_lun_thick(self):
+        ret = self.client.migrate_lun('lun_thick', 'pool_2', 'thick')
+        self.assertTrue(ret)
+
+    def test_migrate_lun_compressed(self):
+        ret = self.client.migrate_lun('lun_compressed', 'pool_2', 'compressed')
+        self.assertTrue(ret)
+
     def test_get_pool_id_by_name(self):
         self.assertEqual('pool_3', self.client.get_pool_id_by_name('Pool 3'))
 

cinder/tests/unit/volume/drivers/dell_emc/unity/test_driver.py

@@ -175,6 +175,9 @@ class MockAdapter(object):
             return group_update, volumes_update
         return group_update, None
 
+    def retype(self, ctxt, volume, new_type, diff, host):
+        return True
+
 
 class MockReplicationManager(object):
     def __init__(self):
@@ -318,6 +321,17 @@ class UnityDriverTest(unittest.TestCase):
                                          'HostA@BackendB#PoolC')
         self.assertEqual((True, {}), ret)
 
+    def test_retype_volume(self):
+        volume = self.get_volume()
+        new_type = {'name': u'type01', 'qos_specs_id': 'test_qos_id',
+                    'extra_specs': {},
+                    'id': u'd67c4480-a61b-44c0-a58b-24c0357cadeb'}
+        diff = None
+        ret = self.driver.retype(self.get_context(),
+                                 volume, new_type, diff,
+                                 'HostA@BackendB#PoolC')
+        self.assertTrue(ret)
+
     def test_create_snapshot(self):
         snapshot = self.get_snapshot()
         self.driver.create_snapshot(snapshot)

cinder/tests/unit/volume/drivers/dell_emc/unity/test_utils.py

@@ -329,3 +329,52 @@ class UnityUtilsTest(unittest.TestCase):
         cg = test_driver.UnityDriverTest.get_cg()
         result = utils.get_group_specs(cg, 'test_key')
         self.assertIsNone(result)
+
+    @patch_volume_types
+    def test_retype_no_need_migration_when_same_host(self):
+        volume = test_adapter.MockOSResource(volume_type_id='host_1',
+                                             host='host_1')
+        new_host = {'name': 'new_name', 'host': 'host_1'}
+        ret = utils.retype_need_migration(volume, None, None, new_host)
+        self.assertFalse(ret)
+
+    @patch_volume_types
+    def test_retype_need_migration_when_diff_host(self):
+        volume = test_adapter.MockOSResource(volume_type_id='host_1',
+                                             host='host_1')
+        new_host = {'name': 'new_name', 'host': 'new_host'}
+        ret = utils.retype_need_migration(volume, None, None, new_host)
+        self.assertTrue(ret)
+
+    @patch_volume_types
+    def test_retype_no_need_migration_thin_to_compressed(self):
+        volume = test_adapter.MockOSResource(volume_type_id='host_1',
+                                             host='host_1')
+        new_host = {'name': 'new_name', 'host': 'host_1'}
+        old_provision = ''
+        new_provision = 'compressed'
+        ret = utils.retype_need_migration(volume, old_provision,
+                                          new_provision, new_host)
+        self.assertFalse(ret)
+
+    @patch_volume_types
+    def test_retype_no_need_migration_compressed_to_thin(self):
+        volume = test_adapter.MockOSResource(volume_type_id='host_1',
+                                             host='host_1')
+        new_host = {'name': 'new_name', 'host': 'host_1'}
+        old_provision = 'compressed'
+        new_provision = ''
+        ret = utils.retype_need_migration(volume, old_provision,
+                                          new_provision, new_host)
+        self.assertFalse(ret)
+
+    @patch_volume_types
+    def test_retype_need_migration_thin_to_thick(self):
+        volume = test_adapter.MockOSResource(volume_type_id='host_1',
+                                             host='host_1')
+        new_host = {'name': 'new_name', 'host': 'host_1'}
+        old_provision = ''
+        new_provision = 'thick'
+        ret = utils.retype_need_migration(volume, old_provision,
+                                          new_provision, new_host)
+        self.assertTrue(ret)

cinder/volume/drivers/dell_emc/unity/adapter.py

@@ -30,6 +30,7 @@ from cinder.objects import fields
 from cinder import utils as cinder_utils
 from cinder.volume.drivers.dell_emc.unity import client
 from cinder.volume.drivers.dell_emc.unity import utils
+from cinder.volume import volume_types
 from cinder.volume import volume_utils
 
 storops = importutils.try_import('storops')
@@ -120,7 +121,8 @@ class VolumeParams(object):
     @property
     def is_thick(self):
         if self._is_thick is None:
-            provision = utils.get_extra_spec(self._volume, 'provisioning:type')
+            provision = utils.get_extra_spec(self._volume,
+                                             utils.PROVISIONING_TYPE)
             support = utils.get_extra_spec(self._volume,
                                            'thick_provisioning_support')
             self._is_thick = (provision == 'thick' and support == '<is> True')
@@ -129,10 +131,12 @@ class VolumeParams(object):
     @property
     def is_compressed(self):
         if self._is_compressed is None:
-            provision = utils.get_extra_spec(self._volume, 'provisioning:type')
+            provision = utils.get_extra_spec(self._volume,
+                                             utils.PROVISIONING_TYPE)
             compression = utils.get_extra_spec(self._volume,
                                                'compression_support')
-            if provision == 'compressed' and compression == '<is> True':
+            if (provision == utils.PROVISIONING_COMPRESSED and
+                    compression == '<is> True'):
                 self._is_compressed = True
         return self._is_compressed
 
@@ -443,6 +447,60 @@ class CommonAdapter(object):
         else:
             self.client.delete_lun(lun_id)
 
+    def retype(self, ctxt, volume, new_type, diff, host):
+        """Changes volume from one type to another."""
+        old_qos_specs = {utils.QOS_SPECS: None}
+        old_provision = None
+        new_specs = volume_types.get_volume_type_extra_specs(
+            new_type.get(utils.QOS_ID))
+        new_qos_specs = volume_types.get_volume_type_qos_specs(
+            new_type.get(utils.QOS_ID))
+        lun = self.client.get_lun(name=volume.name)
+        volume_type_id = volume.volume_type_id
+        if volume_type_id:
+            old_provision = utils.get_extra_spec(volume,
+                                                 utils.PROVISIONING_TYPE)
+            old_qos_specs = volume_types.get_volume_type_qos_specs(
+                volume_type_id)
+
+        need_migration = utils.retype_need_migration(
+            volume, old_provision,
+            new_specs.get(utils.PROVISIONING_TYPE), host)
+        need_change_compress = utils.retype_need_change_compression(
+            old_provision, new_specs.get(utils.PROVISIONING_TYPE))
+        need_change_qos = utils.retype_need_change_qos(
+            old_qos_specs, new_qos_specs)
+
+        if need_migration or need_change_compress[0] or need_change_qos:
+            if self.client.lun_has_snapshot(lun):
+                LOG.warning('Driver is not able to do retype because '
+                            'the volume %s has snapshot(s).',
+                            volume.id)
+                return False
+
+        new_qos_dict = new_qos_specs.get(utils.QOS_SPECS)
+        if need_change_qos:
+            new_io_policy = (self.client.get_io_limit_policy(new_qos_dict)
+                             if need_change_qos else None)
+            # Modify lun to change qos settings
+            if new_io_policy:
+                lun.modify(io_limit_policy=new_io_policy)
+            else:
+                # remove current qos settings
+                old_qos_dict = old_qos_specs.get(utils.QOS_SPECS)
+                old_io_policy = self.client.get_io_limit_policy(old_qos_dict)
+                old_io_policy.remove_from_storage(lun)
+
+        if need_migration:
+            LOG.debug('Driver needs to use storage-assisted migration '
+                      'to retype the volume.')
+            return self.migrate_volume(volume, host, new_specs)
+
+        if need_change_compress[0]:
+            # Modify lun to change compression
+            lun.modify(is_compression=need_change_compress[1])
+        return True
+
     def _create_host_and_attach(self, host_name, lun_or_snap):
         @utils.lock_if(self.to_lock_host, '{lock_name}')
         def _lock_helper(lock_name):
@@ -908,18 +966,22 @@ class CommonAdapter(object):
     def restore_snapshot(self, volume, snapshot):
         return self.client.restore_snapshot(snapshot.name)
 
-    def migrate_volume(self, volume, host):
+    def migrate_volume(self, volume, host, extra_specs=None):
         """Leverage the Unity move session functionality.
 
         This method is invoked at the source backend.
+
+        :param extra_specs: Instance of ExtraSpecs. The new volume will be
+            changed to align with the new extra specs.
         """
         log_params = {
             'name': volume.name,
             'src_host': volume.host,
-            'dest_host': host['host']
+            'dest_host': host['host'],
+            'extra_specs': extra_specs,
         }
         LOG.info('Migrate Volume: %(name)s, host: %(src_host)s, destination: '
-                 '%(dest_host)s', log_params)
+                 '%(dest_host)s, extra_specs: %(extra_specs)s', log_params)
 
         src_backend = utils.get_backend_name_from_volume(volume)
         dest_backend = utils.get_backend_name_from_host(host)
@@ -930,10 +992,12 @@ class CommonAdapter(object):
             return False, None
 
         lun_id = self.get_lun_id(volume)
+        provision = None
+        if extra_specs:
+            provision = extra_specs.get(utils.PROVISIONING_TYPE)
         dest_pool_name = utils.get_pool_name_from_host(host)
         dest_pool_id = self.get_pool_id_by_name(dest_pool_name)
-
+        if self.client.migrate_lun(lun_id, dest_pool_id, provision):
-        if self.client.migrate_lun(lun_id, dest_pool_id):
             LOG.debug('Volume migrated successfully.')
             model_update = {}
             return True, model_update

cinder/volume/drivers/dell_emc/unity/client.py

@@ -177,10 +177,23 @@ class UnityClient(object):
                       lun_id)
         return lun
 
-    def migrate_lun(self, lun_id, dest_pool_id):
+    def migrate_lun(self, lun_id, dest_pool_id, dest_provision=None):
+        # dest_provision possible value ('thin', 'thick', 'compressed')
         lun = self.system.get_lun(lun_id)
         dest_pool = self.system.get_pool(dest_pool_id)
-        return lun.migrate(dest_pool)
+        is_thin = True if dest_provision == 'thin' else None
+        if dest_provision == 'compressed':
+            # compressed needs work with thin
+            is_compressed = True
+            is_thin = True
+        else:
+            is_compressed = False
+        if dest_provision == 'thick':
+            # thick needs work with uncompressed
+            is_thin = False
+            is_compressed = False
+        return lun.migrate(dest_pool, is_compressed=is_compressed,
+                           is_thin=is_thin)
 
     def get_pools(self):
         """Gets all storage pools on the Unity system.
@@ -236,6 +249,10 @@ class UnityClient(object):
                         {'name': name, 'err': err})
         return None
 
+    def lun_has_snapshot(self, lun):
+        snaps = lun.snapshots
+        return len(snaps) != 0
+
     @coordination.synchronized('{self.host}-{name}')
     def create_host(self, name):
         return self.create_host_wo_lock(name)

cinder/volume/drivers/dell_emc/unity/driver.py

@@ -83,9 +83,10 @@ class UnityDriver(driver.ManageableVD,
         6.1.0 - Support volume replication
         7.0.0 - Support tiering policy
         7.1.0 - Support consistency group replication
+        7.2.0 - Support retype volume
     """
 
-    VERSION = '07.01.00'
+    VERSION = '07.02.00'
     VENDOR = 'Dell EMC'
     # ThirdPartySystems wiki page
     CI_WIKI_NAME = "EMC_UNITY_CI"
@@ -142,6 +143,10 @@ class UnityDriver(driver.ManageableVD,
         """Migrates a volume."""
         return self.adapter.migrate_volume(volume, host)
 
+    def retype(self, ctxt, volume, new_type, diff, host):
+        """Convert the volume to be of the new type."""
+        return self.adapter.retype(ctxt, volume, new_type, diff, host)
+
     def create_snapshot(self, snapshot):
         """Creates a snapshot."""
         self.adapter.create_snapshot(snapshot)

cinder/volume/drivers/dell_emc/unity/utils.py

@@ -38,6 +38,11 @@ LOG = logging.getLogger(__name__)
 BACKEND_QOS_CONSUMERS = frozenset(['back-end', 'both'])
 QOS_MAX_IOPS = 'maxIOPS'
 QOS_MAX_BWS = 'maxBWS'
+PROVISIONING_TYPE = 'provisioning:type'
+PROVISIONING_COMPRESSED = 'compressed'
+QOS_SPECS = 'qos_specs'
+SPECS_OF_QOS = 'specs'
+QOS_ID = 'id'
 
 
 def dump_provider_location(location_dict):
@@ -113,6 +118,36 @@ def validate_pool_names(conf_pools, array_pools):
     return existed
 
 
+def retype_need_migration(volume, old_provision, new_provision, host):
+    if volume['host'] != host['host']:
+        return True
+
+    if old_provision != new_provision:
+        if retype_need_change_compression(old_provision, new_provision)[0]:
+            return False
+        else:
+            return True
+    return False
+
+
+def retype_need_change_compression(old_provision, new_provision):
+    """:return: whether need change compression and the new value"""
+    if ((not old_provision or old_provision == 'thin') and
+            new_provision == PROVISIONING_COMPRESSED):
+        return True, True
+    elif (old_provision == PROVISIONING_COMPRESSED and
+          (not new_provision or old_provision == 'thin')):
+        return True, False
+    # no need change compression
+    return False, None
+
+
+def retype_need_change_qos(old_qos=None, new_qos=None):
+    old = old_qos.get(QOS_SPECS).get(QOS_ID) if old_qos.get(QOS_SPECS) else ''
+    new = new_qos.get(QOS_SPECS).get(QOS_ID) if new_qos.get(QOS_SPECS) else ''
+    return old != new
+
+
 def extract_iscsi_uids(connector):
     if 'initiator' not in connector:
         msg = _("Host %s doesn't have iSCSI initiator.") % connector['host']
@@ -281,7 +316,7 @@ def get_backend_qos_specs(volume):
     if qos_specs is None:
         return None
 
-    qos_specs = qos_specs['qos_specs']
+    qos_specs = qos_specs[QOS_SPECS]
     if qos_specs is None:
         return None
 
@@ -290,8 +325,8 @@ def get_backend_qos_specs(volume):
     if consumer not in BACKEND_QOS_CONSUMERS:
         return None
 
-    max_iops = qos_specs['specs'].get(QOS_MAX_IOPS)
+    max_iops = qos_specs[SPECS_OF_QOS].get(QOS_MAX_IOPS)
-    max_bws = qos_specs['specs'].get(QOS_MAX_BWS)
+    max_bws = qos_specs[SPECS_OF_QOS].get(QOS_MAX_BWS)
     if max_iops is None and max_bws is None:
         return None
 

doc/source/configuration/block-storage/drivers/dell-emc-unity-driver.rst

@@ -296,12 +296,29 @@ triggered. Instead, host-assisted volume migration will be triggered:
   the storage-assisted volume migration of vol_2 will not be triggered.
 
 
+Retype volume support
+~~~~~~~~~~~~~~~~~~~~~
+
+Unity driver supports to change a volume's type after its creation.
+
+.. code-block:: console
+
+   $ cinder retype [--migration-policy <never|on-demand>] <volume> <volume-type>
+
+The --migration-policy is not enabled by default.
+Some retype operations will require migration based on back-end support.
+In these cases, the storage-assisted migration will be triggered regardless
+the --migration-policy. For examples: retype between 'thin' and 'thick', retype
+between 'thick' and 'compressed', retype to type(s) current host doesn't
+support.
+
+
 QoS support
 ~~~~~~~~~~~
 
 Unity driver supports ``maxBWS`` and ``maxIOPS`` specs for the back-end
-consumer type. ``maxBWS`` represents the ``Maximum IO/S`` absolute limit,
+consumer type. ``maxBWS`` represents the ``Maximum Bandwidth (KBPS)`` absolute
-``maxIOPS`` represents the ``Maximum Bandwidth (KBPS)`` absolute limit on the
+limit, ``maxIOPS`` represents the ``Maximum IO/S`` absolute limit on the
 Unity respectively.
 
 
@@ -655,3 +672,4 @@ to track specific Block Storage command logs.
 
       # grep "req-3a459e0e-871a-49f9-9796-b63cc48b5015" cinder-volume.log
 
+

releasenotes/notes/unity-retype-volume-support-773ae17b8811fb3f.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    Dell EMC Unity driver: Add efficient retype support when new type uses the same Unity device.