openstack/cinder
Code Issues Proposed changes

Merge "[Unity] Retype volume support"

Browse Source
This commit is contained in:
Zuul 2020-04-14 15:49:51 +00:00 committed by Gerrit Code Review
commit ad73a1e0f6
10 changed files with 324 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
cinder/tests/unit/volume/drivers/dell_emc/unity/test_adapter.py
View File

@ -118,6 +118,10 @@ class MockClient(object):
lun_id += '_low'
return test_client.MockResource(_id=lun_id, name=name)
@staticmethod
def lun_has_snapshot(lun):
return lun.name == 'volume_has_snapshot'
@staticmethod
def get_lun(name=None, lun_id=None):
if lun_id is None:
@ -214,7 +218,9 @@ class MockClient(object):
@staticmethod
def get_io_limit_policy(specs):
return None
mock_io_policy = (test_client.MockResource(name=specs.get('id'))
if specs else None)
return mock_io_policy
@staticmethod
def extend_lun(lun_id, size_gib):
@ -257,7 +263,7 @@ class MockClient(object):
'PoolC': 'pool_3'}
return pools.get(name, None)
def migrate_lun(self, lun_id, dest_pool_id):
def migrate_lun(self, lun_id, dest_pool_id, provision=None):
if dest_pool_id == 'pool_2':
return True
if dest_pool_id == 'pool_3':
@ -409,6 +415,20 @@ def get_connection_info(adapter, hlu, host, connector):
return {}
def get_volume_type_qos_specs(qos_id):
if qos_id == 'qos':
return {'qos_specs': {'id': u'qos_type_id_1',
'consumer': u'back-end',
u'maxBWS': u'102400',
u'maxIOPS': u'500'}}
if qos_id == 'qos_2':
return {'qos_specs': {'id': u'qos_type_id_2',
'consumer': u'back-end',
u'maxBWS': u'102402',
u'maxIOPS': u'502'}}
return {'qos_specs': {}}
def get_volume_type_extra_specs(type_id):
if type_id == 'thick':
return {'provisioning:type': 'thick',
@ -419,6 +439,9 @@ def get_volume_type_extra_specs(type_id):
if type_id == 'tier_lowest':
return {'storagetype:tiering': 'LowestAvailable',
'fast_support': '<is> True'}
if type_id == 'compressed':
return {'provisioning:type': 'compressed',
'compression_support': '<is> True'}
return {}
@ -439,6 +462,8 @@ def patch_for_unity_adapter(func):
new=get_volume_type_extra_specs)
@mock.patch('cinder.volume.group_types.get_group_type_specs',
new=get_group_type_specs)
@mock.patch('cinder.volume.volume_types.get_volume_type_qos_specs',
new=get_volume_type_qos_specs)
@mock.patch('cinder.volume.drivers.dell_emc.unity.utils.'
'get_backend_qos_specs',
new=get_backend_qos_specs)
@ -601,6 +626,65 @@ class CommonAdapterTest(test.TestCase):
volume = MockOSResource(provider_location='id^lun_4')
self.adapter.delete_volume(volume)
@patch_for_unity_adapter
def test_retype_volume_has_snapshot(self):
volume = MockOSResource(name='volume_has_snapshot', size=5,
host='HostA@BackendB#PoolB')
ctxt = None
diff = None
new_type = {'name': u'type01', 'id': 'compressed'}
host = {'host': 'HostA@BackendB#PoolB'}
result = self.adapter.retype(ctxt, volume, new_type, diff, host)
self.assertFalse(result)
@patch_for_unity_adapter
def test_retype_volume_thick_to_compressed(self):
volume = MockOSResource(name='thick_volume', size=5,
host='HostA@BackendB#PoolA',
provider_location='id^lun_33')
ctxt = None
diff = None
new_type = {'name': u'compressed_type', 'id': 'compressed'}
host = {'host': 'HostA@BackendB#PoolB'}
result = self.adapter.retype(ctxt, volume, new_type, diff, host)
self.assertEqual((True, {}), result)
@patch_for_unity_adapter
def test_retype_volume_to_compressed(self):
volume = MockOSResource(name='thin_volume', size=5,
host='HostA@BackendB#PoolB')
ctxt = None
diff = None
new_type = {'name': u'compressed_type', 'id': 'compressed'}
host = {'host': 'HostA@BackendB#PoolB'}
result = self.adapter.retype(ctxt, volume, new_type, diff, host)
self.assertTrue(result)
@patch_for_unity_adapter
def test_retype_volume_to_qos(self):
volume = MockOSResource(name='thin_volume', size=5,
host='HostA@BackendB#PoolB')
ctxt = None
diff = None
new_type = {'name': u'qos_type', 'id': 'qos'}
host = {'host': 'HostA@BackendB#PoolB'}
result = self.adapter.retype(ctxt, volume, new_type,
diff, host)
self.assertTrue(result)
@patch_for_unity_adapter
def test_retype_volume_revert_qos(self):
volume = MockOSResource(name='qos_volume', size=5,
host='HostA@BackendB#PoolB',
volume_type_id='qos_2')
ctxt = None
diff = None
new_type = {'name': u'no_qos_type', 'id': ''}
host = {'host': 'HostA@BackendB#PoolB'}
result = self.adapter.retype(ctxt, volume, new_type,
diff, host)
self.assertTrue(result)
def test_get_pool_stats(self):
stats_list = self.adapter.get_pools_stats()
self.assertEqual(1, len(stats_list))

20
cinder/tests/unit/volume/drivers/dell_emc/unity/test_client.py
View File

@ -59,6 +59,7 @@ class MockResource(object):
self.lun = None
self.tiering_policy = None
self.pool_fast_vp = None
self.snap = True
@property
def id(self):
@ -199,9 +200,12 @@ class MockResource(object):
def storage_resource(self, value):
self._storage_resource = value
def modify(self, name=None):
def modify(self, name=None, is_compression=None, io_limit_policy=None):
self.name = name
def remove_from_storage(self, lun):
pass
def thin_clone(self, name, io_limit_policy=None, description=None):
if name == 'thin_clone_name_in_use':
raise ex.UnityLunNameInUseError
@ -213,8 +217,8 @@ class MockResource(object):
def restore(self, delete_backup):
return MockResource(_id='snap_1', name="internal_snap")
def migrate(self, dest_pool):
if dest_pool.id == 'pool_2':
def migrate(self, dest_pool, **kwargs):
if dest_pool.id == 'fail_migration_pool':
return False
return True
@ -685,9 +689,17 @@ class ClientTest(unittest.TestCase):
self.assertTrue(ret)
def test_migrate_lun_failed(self):
ret = self.client.migrate_lun('lun_0', 'pool_2')
ret = self.client.migrate_lun('lun_0', 'fail_migration_pool')
self.assertFalse(ret)
def test_migrate_lun_thick(self):
ret = self.client.migrate_lun('lun_thick', 'pool_2', 'thick')
self.assertTrue(ret)
def test_migrate_lun_compressed(self):
ret = self.client.migrate_lun('lun_compressed', 'pool_2', 'compressed')
self.assertTrue(ret)
def test_get_pool_id_by_name(self):
self.assertEqual('pool_3', self.client.get_pool_id_by_name('Pool 3'))

14
cinder/tests/unit/volume/drivers/dell_emc/unity/test_driver.py
View File

@ -175,6 +175,9 @@ class MockAdapter(object):
return group_update, volumes_update
return group_update, None
def retype(self, ctxt, volume, new_type, diff, host):
return True
class MockReplicationManager(object):
def __init__(self):
@ -318,6 +321,17 @@ class UnityDriverTest(unittest.TestCase):
'HostA@BackendB#PoolC')
self.assertEqual((True, {}), ret)
def test_retype_volume(self):
volume = self.get_volume()
new_type = {'name': u'type01', 'qos_specs_id': 'test_qos_id',
'extra_specs': {},
'id': u'd67c4480-a61b-44c0-a58b-24c0357cadeb'}
diff = None
ret = self.driver.retype(self.get_context(),
volume, new_type, diff,
'HostA@BackendB#PoolC')
self.assertTrue(ret)
def test_create_snapshot(self):
snapshot = self.get_snapshot()
self.driver.create_snapshot(snapshot)

49
cinder/tests/unit/volume/drivers/dell_emc/unity/test_utils.py
View File

@ -329,3 +329,52 @@ class UnityUtilsTest(unittest.TestCase):
cg = test_driver.UnityDriverTest.get_cg()
result = utils.get_group_specs(cg, 'test_key')
self.assertIsNone(result)
@patch_volume_types
def test_retype_no_need_migration_when_same_host(self):
volume = test_adapter.MockOSResource(volume_type_id='host_1',
host='host_1')
new_host = {'name': 'new_name', 'host': 'host_1'}
ret = utils.retype_need_migration(volume, None, None, new_host)
self.assertFalse(ret)
@patch_volume_types
def test_retype_need_migration_when_diff_host(self):
volume = test_adapter.MockOSResource(volume_type_id='host_1',
host='host_1')
new_host = {'name': 'new_name', 'host': 'new_host'}
ret = utils.retype_need_migration(volume, None, None, new_host)
self.assertTrue(ret)
@patch_volume_types
def test_retype_no_need_migration_thin_to_compressed(self):
volume = test_adapter.MockOSResource(volume_type_id='host_1',
host='host_1')
new_host = {'name': 'new_name', 'host': 'host_1'}
old_provision = ''
new_provision = 'compressed'
ret = utils.retype_need_migration(volume, old_provision,
new_provision, new_host)
self.assertFalse(ret)
@patch_volume_types
def test_retype_no_need_migration_compressed_to_thin(self):
volume = test_adapter.MockOSResource(volume_type_id='host_1',
host='host_1')
new_host = {'name': 'new_name', 'host': 'host_1'}
old_provision = 'compressed'
new_provision = ''
ret = utils.retype_need_migration(volume, old_provision,
new_provision, new_host)
self.assertFalse(ret)
@patch_volume_types
def test_retype_need_migration_thin_to_thick(self):
volume = test_adapter.MockOSResource(volume_type_id='host_1',
host='host_1')
new_host = {'name': 'new_name', 'host': 'host_1'}
old_provision = ''
new_provision = 'thick'
ret = utils.retype_need_migration(volume, old_provision,
new_provision, new_host)
self.assertTrue(ret)

80
cinder/volume/drivers/dell_emc/unity/adapter.py
View File

@ -30,6 +30,7 @@ from cinder.objects import fields
from cinder import utils as cinder_utils
from cinder.volume.drivers.dell_emc.unity import client
from cinder.volume.drivers.dell_emc.unity import utils
from cinder.volume import volume_types
from cinder.volume import volume_utils
storops = importutils.try_import('storops')
@ -120,7 +121,8 @@ class VolumeParams(object):
@property
def is_thick(self):
if self._is_thick is None:
provision = utils.get_extra_spec(self._volume, 'provisioning:type')
provision = utils.get_extra_spec(self._volume,
utils.PROVISIONING_TYPE)
support = utils.get_extra_spec(self._volume,
'thick_provisioning_support')
self._is_thick = (provision == 'thick' and support == '<is> True')
@ -129,10 +131,12 @@ class VolumeParams(object):
@property
def is_compressed(self):
if self._is_compressed is None:
provision = utils.get_extra_spec(self._volume, 'provisioning:type')
provision = utils.get_extra_spec(self._volume,
utils.PROVISIONING_TYPE)
compression = utils.get_extra_spec(self._volume,
'compression_support')
if provision == 'compressed' and compression == '<is> True':
if (provision == utils.PROVISIONING_COMPRESSED and
compression == '<is> True'):
self._is_compressed = True
return self._is_compressed
@ -443,6 +447,60 @@ class CommonAdapter(object):
else:
self.client.delete_lun(lun_id)
def retype(self, ctxt, volume, new_type, diff, host):
"""Changes volume from one type to another."""
old_qos_specs = {utils.QOS_SPECS: None}
old_provision = None
new_specs = volume_types.get_volume_type_extra_specs(
new_type.get(utils.QOS_ID))
new_qos_specs = volume_types.get_volume_type_qos_specs(
new_type.get(utils.QOS_ID))
lun = self.client.get_lun(name=volume.name)
volume_type_id = volume.volume_type_id
if volume_type_id:
old_provision = utils.get_extra_spec(volume,
utils.PROVISIONING_TYPE)
old_qos_specs = volume_types.get_volume_type_qos_specs(
volume_type_id)
need_migration = utils.retype_need_migration(
volume, old_provision,
new_specs.get(utils.PROVISIONING_TYPE), host)
need_change_compress = utils.retype_need_change_compression(
old_provision, new_specs.get(utils.PROVISIONING_TYPE))
need_change_qos = utils.retype_need_change_qos(
old_qos_specs, new_qos_specs)
if need_migration or need_change_compress[0] or need_change_qos:
if self.client.lun_has_snapshot(lun):
LOG.warning('Driver is not able to do retype because '
'the volume %s has snapshot(s).',
volume.id)
return False
new_qos_dict = new_qos_specs.get(utils.QOS_SPECS)
if need_change_qos:
new_io_policy = (self.client.get_io_limit_policy(new_qos_dict)
if need_change_qos else None)
# Modify lun to change qos settings
if new_io_policy:
lun.modify(io_limit_policy=new_io_policy)
else:
# remove current qos settings
old_qos_dict = old_qos_specs.get(utils.QOS_SPECS)
old_io_policy = self.client.get_io_limit_policy(old_qos_dict)
old_io_policy.remove_from_storage(lun)
if need_migration:
LOG.debug('Driver needs to use storage-assisted migration '
'to retype the volume.')
return self.migrate_volume(volume, host, new_specs)
if need_change_compress[0]:
# Modify lun to change compression
lun.modify(is_compression=need_change_compress[1])
return True
def _create_host_and_attach(self, host_name, lun_or_snap):
@utils.lock_if(self.to_lock_host, '{lock_name}')
def _lock_helper(lock_name):
@ -908,18 +966,22 @@ class CommonAdapter(object):
def restore_snapshot(self, volume, snapshot):
return self.client.restore_snapshot(snapshot.name)
def migrate_volume(self, volume, host):
def migrate_volume(self, volume, host, extra_specs=None):
"""Leverage the Unity move session functionality.
This method is invoked at the source backend.
:param extra_specs: Instance of ExtraSpecs. The new volume will be
changed to align with the new extra specs.
"""
log_params = {
'name': volume.name,
'src_host': volume.host,
'dest_host': host['host']
'dest_host': host['host'],
'extra_specs': extra_specs,
}
LOG.info('Migrate Volume: %(name)s, host: %(src_host)s, destination: '
'%(dest_host)s', log_params)
'%(dest_host)s, extra_specs: %(extra_specs)s', log_params)
src_backend = utils.get_backend_name_from_volume(volume)
dest_backend = utils.get_backend_name_from_host(host)
@ -930,10 +992,12 @@ class CommonAdapter(object):
return False, None
lun_id = self.get_lun_id(volume)
provision = None
if extra_specs:
provision = extra_specs.get(utils.PROVISIONING_TYPE)
dest_pool_name = utils.get_pool_name_from_host(host)
dest_pool_id = self.get_pool_id_by_name(dest_pool_name)
if self.client.migrate_lun(lun_id, dest_pool_id):
if self.client.migrate_lun(lun_id, dest_pool_id, provision):
LOG.debug('Volume migrated successfully.')
model_update = {}
return True, model_update

21
cinder/volume/drivers/dell_emc/unity/client.py
View File

@ -177,10 +177,23 @@ class UnityClient(object):
lun_id)
return lun
def migrate_lun(self, lun_id, dest_pool_id):
def migrate_lun(self, lun_id, dest_pool_id, dest_provision=None):
# dest_provision possible value ('thin', 'thick', 'compressed')
lun = self.system.get_lun(lun_id)
dest_pool = self.system.get_pool(dest_pool_id)
return lun.migrate(dest_pool)
is_thin = True if dest_provision == 'thin' else None
if dest_provision == 'compressed':
# compressed needs work with thin
is_compressed = True
is_thin = True
else:
is_compressed = False
if dest_provision == 'thick':
# thick needs work with uncompressed
is_thin = False
is_compressed = False
return lun.migrate(dest_pool, is_compressed=is_compressed,
is_thin=is_thin)
def get_pools(self):
"""Gets all storage pools on the Unity system.
@ -236,6 +249,10 @@ class UnityClient(object):
{'name': name, 'err': err})
return None
def lun_has_snapshot(self, lun):
snaps = lun.snapshots
return len(snaps) != 0
@coordination.synchronized('{self.host}-{name}')
def create_host(self, name):
return self.create_host_wo_lock(name)

7
cinder/volume/drivers/dell_emc/unity/driver.py
View File

@ -83,9 +83,10 @@ class UnityDriver(driver.ManageableVD,
6.1.0 - Support volume replication
7.0.0 - Support tiering policy
7.1.0 - Support consistency group replication
7.2.0 - Support retype volume
"""
VERSION = '07.01.00'
VERSION = '07.02.00'
VENDOR = 'Dell EMC'
# ThirdPartySystems wiki page
CI_WIKI_NAME = "EMC_UNITY_CI"
@ -142,6 +143,10 @@ class UnityDriver(driver.ManageableVD,
"""Migrates a volume."""
return self.adapter.migrate_volume(volume, host)
def retype(self, ctxt, volume, new_type, diff, host):
"""Convert the volume to be of the new type."""
return self.adapter.retype(ctxt, volume, new_type, diff, host)
def create_snapshot(self, snapshot):
"""Creates a snapshot."""
self.adapter.create_snapshot(snapshot)

41
cinder/volume/drivers/dell_emc/unity/utils.py
View File

@ -38,6 +38,11 @@ LOG = logging.getLogger(__name__)
BACKEND_QOS_CONSUMERS = frozenset(['back-end', 'both'])
QOS_MAX_IOPS = 'maxIOPS'
QOS_MAX_BWS = 'maxBWS'
PROVISIONING_TYPE = 'provisioning:type'
PROVISIONING_COMPRESSED = 'compressed'
QOS_SPECS = 'qos_specs'
SPECS_OF_QOS = 'specs'
QOS_ID = 'id'
def dump_provider_location(location_dict):
@ -113,6 +118,36 @@ def validate_pool_names(conf_pools, array_pools):
return existed
def retype_need_migration(volume, old_provision, new_provision, host):
if volume['host'] != host['host']:
return True
if old_provision != new_provision:
if retype_need_change_compression(old_provision, new_provision)[0]:
return False
else:
return True
return False
def retype_need_change_compression(old_provision, new_provision):
""":return: whether need change compression and the new value"""
if ((not old_provision or old_provision == 'thin') and
new_provision == PROVISIONING_COMPRESSED):
return True, True
elif (old_provision == PROVISIONING_COMPRESSED and
(not new_provision or old_provision == 'thin')):
return True, False
# no need change compression
return False, None
def retype_need_change_qos(old_qos=None, new_qos=None):
old = old_qos.get(QOS_SPECS).get(QOS_ID) if old_qos.get(QOS_SPECS) else ''
new = new_qos.get(QOS_SPECS).get(QOS_ID) if new_qos.get(QOS_SPECS) else ''
return old != new
def extract_iscsi_uids(connector):
if 'initiator' not in connector:
msg = _("Host %s doesn't have iSCSI initiator.") % connector['host']
@ -281,7 +316,7 @@ def get_backend_qos_specs(volume):
if qos_specs is None:
return None
qos_specs = qos_specs['qos_specs']
qos_specs = qos_specs[QOS_SPECS]
if qos_specs is None:
return None
@ -290,8 +325,8 @@ def get_backend_qos_specs(volume):
if consumer not in BACKEND_QOS_CONSUMERS:
return None
max_iops = qos_specs['specs'].get(QOS_MAX_IOPS)
max_bws = qos_specs['specs'].get(QOS_MAX_BWS)
max_iops = qos_specs[SPECS_OF_QOS].get(QOS_MAX_IOPS)
max_bws = qos_specs[SPECS_OF_QOS].get(QOS_MAX_BWS)
if max_iops is None and max_bws is None:
return None

22
doc/source/configuration/block-storage/drivers/dell-emc-unity-driver.rst
View File

@ -296,12 +296,29 @@ triggered. Instead, host-assisted volume migration will be triggered:
the storage-assisted volume migration of vol_2 will not be triggered.
Retype volume support
~~~~~~~~~~~~~~~~~~~~~
Unity driver supports to change a volume's type after its creation.
.. code-block:: console
$ cinder retype [--migration-policy <never|on-demand>] <volume> <volume-type>
The --migration-policy is not enabled by default.
Some retype operations will require migration based on back-end support.
In these cases, the storage-assisted migration will be triggered regardless
the --migration-policy. For examples: retype between 'thin' and 'thick', retype
between 'thick' and 'compressed', retype to type(s) current host doesn't
support.
QoS support
~~~~~~~~~~~
Unity driver supports ``maxBWS`` and ``maxIOPS`` specs for the back-end
consumer type. ``maxBWS`` represents the ``Maximum IO/S`` absolute limit,
``maxIOPS`` represents the ``Maximum Bandwidth (KBPS)`` absolute limit on the
consumer type. ``maxBWS`` represents the ``Maximum Bandwidth (KBPS)`` absolute
limit, ``maxIOPS`` represents the ``Maximum IO/S`` absolute limit on the
Unity respectively.
@ -655,3 +672,4 @@ to track specific Block Storage command logs.
# grep "req-3a459e0e-871a-49f9-9796-b63cc48b5015" cinder-volume.log

4
releasenotes/notes/unity-retype-volume-support-773ae17b8811fb3f.yaml Normal file
View File

@ -0,0 +1,4 @@
---
features:
- |
Dell EMC Unity driver: Add efficient retype support when new type uses the same Unity device.