openstack/cinder
Code Issues Proposed changes

Unity: Add consistent group support

Browse Source 
Users could create a group type supporting consistent groups with
specification `'consistent_group_snapshot_enabled': <is> True`, then any
groups created of that group type are consistent groups, otherwise they
are generic groups.

The supported operations are:

- Create/delete consistent groups
- Add volumes to and remove volumes from consistent groups
- Create/delete consistent group snapshots
- Create consistent groups from snapshots
- Clone consistent groups

This change also does some refactor and puts extra capabilities report
together in `utils.py`, including the existing
`thin_provisioning_support`, `thick_provisioning_support` and the newly
one added for cg named `consistent_group_snapshot_enabled`.

Implements: blueprint unity-consistent-group-support

Change-Id: I0ef2ec959f892acb79d8d08a31d9a8ad47c4350f
This commit is contained in:
Ryan Liang 2018-05-30 16:19:12 +08:00
parent fbd7c0377d
commit 661a4f1212
11 changed files with 883 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cinder/tests/unit/volume/drivers/dell_emc/unity/fake_exception.py
View File

@ -92,3 +92,7 @@ class UnityThinCloneNotAllowedError(StoropsException):
class SystemAPINotSupported(StoropsException):
pass
class UnityConsistencyGroupNameInUseError(StoropsException):
pass

297
cinder/tests/unit/volume/drivers/dell_emc/unity/test_adapter.py
View File

@ -16,6 +16,7 @@
import contextlib
import functools
import ddt
import mock
from oslo_utils import units
@ -375,7 +376,10 @@ class IdMatcher(object):
#
########################
@ddt.ddt
@mock.patch.object(adapter, 'storops_ex', new=ex)
@mock.patch.object(adapter.vol_utils, 'is_group_a_cg_snapshot_type',
new=lambda x: True)
class CommonAdapterTest(test.TestCase):
def setUp(self):
super(CommonAdapterTest, self).setUp()
@ -389,7 +393,8 @@ class CommonAdapterTest(test.TestCase):
@patch_for_unity_adapter
def test_create_volume(self):
volume = MockOSResource(name='lun_3', size=5, host='unity#pool1')
volume = MockOSResource(name='lun_3', size=5, host='unity#pool1',
group=None)
ret = self.adapter.create_volume(volume)
expected = get_lun_pl('lun_3')
self.assertEqual(expected, ret['provider_location'])
@ -397,7 +402,7 @@ class CommonAdapterTest(test.TestCase):
@patch_for_unity_adapter
def test_create_volume_thick(self):
volume = MockOSResource(name='lun_3', size=5, host='unity#pool1',
volume_type_id='thick')
group=None, volume_type_id='thick')
ret = self.adapter.create_volume(volume)
expected = get_lun_pl('lun_3_thick')
@ -408,7 +413,7 @@ class CommonAdapterTest(test.TestCase):
volume_type = MockOSResource(
extra_specs={'compression_support': '<is> True'})
volume = MockOSResource(name='lun_3', size=5, host='unity#pool1',
volume_type=volume_type)
group=None, volume_type=volume_type)
ret = self.adapter.create_volume(volume)
expected = get_lun_pl('lun_3')
self.assertEqual(expected, ret['provider_location'])
@ -454,6 +459,7 @@ class CommonAdapterTest(test.TestCase):
self.assertTrue(stats['thick_provisioning_support'])
self.assertTrue(stats['thin_provisioning_support'])
self.assertTrue(stats['compression_support'])
self.assertTrue(stats['consistent_group_snapshot_enabled'])
def test_update_volume_stats(self):
stats = self.adapter.update_volume_stats()
@ -461,6 +467,7 @@ class CommonAdapterTest(test.TestCase):
self.assertEqual('unknown', stats['storage_protocol'])
self.assertTrue(stats['thin_provisioning_support'])
self.assertTrue(stats['thick_provisioning_support'])
self.assertTrue(stats['consistent_group_snapshot_enabled'])
self.assertEqual(1, len(stats['pools']))
def test_serial_number(self):
@ -678,6 +685,7 @@ class CommonAdapterTest(test.TestCase):
volume = MockOSResource(name=lun_id, id=lun_id, host='unity#pool1',
provider_location=get_lun_pl(lun_id))
src_snap = test_client.MockResource(name=src_snap_id, _id=src_snap_id)
src_snap.size = 5 * units.Gi
src_snap.storage_resource = test_client.MockResource(name=src_lun_id,
_id=src_lun_id)
with patch_copy_volume() as copy_volume:
@ -841,6 +849,289 @@ class CommonAdapterTest(test.TestCase):
ret = self.adapter.migrate_volume(volume, host)
self.assertEqual((False, None), ret)
@ddt.unpack
@ddt.data((('group-1', 'group-1_name', 'group-1_description'),
('group-1', 'group-1_description')),
(('group-2', 'group-2_name', None), ('group-2', 'group-2_name')),
(('group-3', 'group-3_name', ''), ('group-3', 'group-3_name')))
def test_create_group(self, inputs, expected):
cg_id, cg_name, cg_description = inputs
cg = MockOSResource(id=cg_id, name=cg_name, description=cg_description)
with mock.patch.object(self.adapter.client, 'create_cg',
create=True) as mocked:
model_update = self.adapter.create_group(cg)
self.assertEqual('available', model_update['status'])
mocked.assert_called_once_with(expected[0],
description=expected[1])
def test_delete_group(self):
cg = MockOSResource(id='group-1')
with mock.patch.object(self.adapter.client, 'delete_cg',
create=True) as mocked:
ret = self.adapter.delete_group(cg)
self.assertIsNone(ret[0])
self.assertIsNone(ret[1])
mocked.assert_called_once_with('group-1')
def test_update_group(self):
cg = MockOSResource(id='group-1')
add_volumes = [MockOSResource(id=vol_id,
provider_location=get_lun_pl(lun_id))
for vol_id, lun_id in (('volume-1', 'sv_1'),
('volume-2', 'sv_2'))]
remove_volumes = [MockOSResource(
id='volume-3', provider_location=get_lun_pl('sv_3'))]
with mock.patch.object(self.adapter.client, 'update_cg',
create=True) as mocked:
ret = self.adapter.update_group(cg, add_volumes, remove_volumes)
self.assertEqual('available', ret[0]['status'])
self.assertIsNone(ret[1])
self.assertIsNone(ret[2])
mocked.assert_called_once_with('group-1', {'sv_1', 'sv_2'},
{'sv_3'})
def test_update_group_add_volumes_none(self):
cg = MockOSResource(id='group-1')
remove_volumes = [MockOSResource(
id='volume-3', provider_location=get_lun_pl('sv_3'))]
with mock.patch.object(self.adapter.client, 'update_cg',
create=True) as mocked:
ret = self.adapter.update_group(cg, None, remove_volumes)
self.assertEqual('available', ret[0]['status'])
self.assertIsNone(ret[1])
self.assertIsNone(ret[2])
mocked.assert_called_once_with('group-1', set(), {'sv_3'})
def test_update_group_remove_volumes_none(self):
cg = MockOSResource(id='group-1')
add_volumes = [MockOSResource(id=vol_id,
provider_location=get_lun_pl(lun_id))
for vol_id, lun_id in (('volume-1', 'sv_1'),
('volume-2', 'sv_2'))]
with mock.patch.object(self.adapter.client, 'update_cg',
create=True) as mocked:
ret = self.adapter.update_group(cg, add_volumes, None)
self.assertEqual('available', ret[0]['status'])
self.assertIsNone(ret[1])
self.assertIsNone(ret[2])
mocked.assert_called_once_with('group-1', {'sv_1', 'sv_2'}, set())
def test_update_group_add_remove_volumes_none(self):
cg = MockOSResource(id='group-1')
with mock.patch.object(self.adapter.client, 'update_cg',
create=True) as mocked:
ret = self.adapter.update_group(cg, None, None)
self.assertEqual('available', ret[0]['status'])
self.assertIsNone(ret[1])
self.assertIsNone(ret[2])
mocked.assert_called_once_with('group-1', set(), set())
@patch_for_unity_adapter
def test_copy_luns_in_group(self):
cg = MockOSResource(id='group-1')
volumes = [MockOSResource(id=vol_id,
provider_location=get_lun_pl(lun_id))
for vol_id, lun_id in (('volume-3', 'sv_3'),
('volume-4', 'sv_4'))]
src_cg_snap = test_client.MockResource(_id='id_src_cg_snap')
src_volumes = [MockOSResource(id=vol_id,
provider_location=get_lun_pl(lun_id))
for vol_id, lun_id in (('volume-1', 'sv_1'),
('volume-2', 'sv_2'))]
copied_luns = [test_client.MockResource(_id=lun_id)
for lun_id in ('sv_3', 'sv_4')]
def _prepare_lun_snaps(lun_id):
lun_snap = test_client.MockResource(_id='snap_{}'.format(lun_id))
lun_snap.lun = test_client.MockResource(_id=lun_id)
return lun_snap
lun_snaps = list(map(_prepare_lun_snaps, ('sv_1', 'sv_2')))
with mock.patch.object(self.adapter.client, 'filter_snaps_in_cg_snap',
create=True) as mocked_filter, \
mock.patch.object(self.adapter.client, 'create_cg',
create=True) as mocked_create_cg, \
patch_dd_copy(None) as mocked_dd:
mocked_filter.return_value = lun_snaps
mocked_dd.side_effect = copied_luns
ret = self.adapter.copy_luns_in_group(cg, volumes, src_cg_snap,
src_volumes)
mocked_filter.assert_called_once_with('id_src_cg_snap')
dd_args = zip([adapter.VolumeParams(self.adapter, vol)
for vol in volumes],
lun_snaps)
mocked_dd.assert_has_calls([mock.call(*args) for args in dd_args])
mocked_create_cg.assert_called_once_with('group-1',
lun_add=copied_luns)
self.assertEqual('available', ret[0]['status'])
self.assertEqual(2, len(ret[1]))
for vol_id in ('volume-3', 'volume-4'):
self.assertIn({'id': vol_id, 'status': 'available'}, ret[1])
def test_create_group_from_snap(self):
cg = MockOSResource(id='group-2')
volumes = [MockOSResource(id=vol_id,
provider_location=get_lun_pl(lun_id))
for vol_id, lun_id in (('volume-3', 'sv_3'),
('volume-4', 'sv_4'))]
cg_snap = MockOSResource(id='snap-group-1')
vol_1 = MockOSResource(id='volume-1')
vol_2 = MockOSResource(id='volume-2')
vol_snaps = [MockOSResource(id='snap-volume-1', volume=vol_1),
MockOSResource(id='snap-volume-2', volume=vol_2)]
src_cg_snap = test_client.MockResource(_id='id_src_cg_snap')
with mock.patch.object(self.adapter.client, 'get_snap',
create=True, return_value=src_cg_snap), \
mock.patch.object(self.adapter, 'copy_luns_in_group',
create=True) as mocked_copy:
mocked_copy.return_value = ({'status': 'available'},
[{'id': 'volume-3',
'status': 'available'},
{'id': 'volume-4',
'status': 'available'}])
ret = self.adapter.create_group_from_snap(cg, volumes, cg_snap,
vol_snaps)
mocked_copy.assert_called_once_with(cg, volumes, src_cg_snap,
[vol_1, vol_2])
self.assertEqual('available', ret[0]['status'])
self.assertEqual(2, len(ret[1]))
for vol_id in ('volume-3', 'volume-4'):
self.assertIn({'id': vol_id, 'status': 'available'}, ret[1])
def test_create_group_from_snap_none_snapshots(self):
cg = MockOSResource(id='group-2')
volumes = [MockOSResource(id=vol_id,
provider_location=get_lun_pl(lun_id))
for vol_id, lun_id in (('volume-3', 'sv_3'),
('volume-4', 'sv_4'))]
cg_snap = MockOSResource(id='snap-group-1')
src_cg_snap = test_client.MockResource(_id='id_src_cg_snap')
with mock.patch.object(self.adapter.client, 'get_snap',
create=True, return_value=src_cg_snap), \
mock.patch.object(self.adapter, 'copy_luns_in_group',
create=True) as mocked_copy:
mocked_copy.return_value = ({'status': 'available'},
[{'id': 'volume-3',
'status': 'available'},
{'id': 'volume-4',
'status': 'available'}])
ret = self.adapter.create_group_from_snap(cg, volumes, cg_snap,
None)
mocked_copy.assert_called_once_with(cg, volumes, src_cg_snap, [])
self.assertEqual('available', ret[0]['status'])
self.assertEqual(2, len(ret[1]))
for vol_id in ('volume-3', 'volume-4'):
self.assertIn({'id': vol_id, 'status': 'available'}, ret[1])
def test_create_cloned_group(self):
cg = MockOSResource(id='group-2')
volumes = [MockOSResource(id=vol_id,
provider_location=get_lun_pl(lun_id))
for vol_id, lun_id in (('volume-3', 'sv_3'),
('volume-4', 'sv_4'))]
src_cg = MockOSResource(id='group-1')
vol_1 = MockOSResource(id='volume-1')
vol_2 = MockOSResource(id='volume-2')
src_vols = [vol_1, vol_2]
src_cg_snap = test_client.MockResource(_id='id_src_cg_snap')
with mock.patch.object(self.adapter.client, 'create_cg_snap',
create=True,
return_value=src_cg_snap) as mocked_create, \
mock.patch.object(self.adapter, 'copy_luns_in_group',
create=True) as mocked_copy:
mocked_create.__name__ = 'create_cg_snap'
mocked_copy.return_value = ({'status': 'available'},
[{'id': 'volume-3',
'status': 'available'},
{'id': 'volume-4',
'status': 'available'}])
ret = self.adapter.create_cloned_group(cg, volumes, src_cg,
src_vols)
mocked_create.assert_called_once_with('group-1',
'snap_clone_group_group-1')
mocked_copy.assert_called_once_with(cg, volumes, src_cg_snap,
[vol_1, vol_2])
self.assertEqual('available', ret[0]['status'])
self.assertEqual(2, len(ret[1]))
for vol_id in ('volume-3', 'volume-4'):
self.assertIn({'id': vol_id, 'status': 'available'}, ret[1])
def test_create_cloned_group_none_source_vols(self):
cg = MockOSResource(id='group-2')
volumes = [MockOSResource(id=vol_id,
provider_location=get_lun_pl(lun_id))
for vol_id, lun_id in (('volume-3', 'sv_3'),
('volume-4', 'sv_4'))]
src_cg = MockOSResource(id='group-1')
src_cg_snap = test_client.MockResource(_id='id_src_cg_snap')
with mock.patch.object(self.adapter.client, 'create_cg_snap',
create=True,
return_value=src_cg_snap) as mocked_create, \
mock.patch.object(self.adapter, 'copy_luns_in_group',
create=True) as mocked_copy:
mocked_create.__name__ = 'create_cg_snap'
mocked_copy.return_value = ({'status': 'available'},
[{'id': 'volume-3',
'status': 'available'},
{'id': 'volume-4',
'status': 'available'}])
ret = self.adapter.create_cloned_group(cg, volumes, src_cg,
None)
mocked_create.assert_called_once_with('group-1',
'snap_clone_group_group-1')
mocked_copy.assert_called_once_with(cg, volumes, src_cg_snap, [])
self.assertEqual('available', ret[0]['status'])
self.assertEqual(2, len(ret[1]))
for vol_id in ('volume-3', 'volume-4'):
self.assertIn({'id': vol_id, 'status': 'available'}, ret[1])
def test_create_group_snapshot(self):
cg_snap = MockOSResource(id='snap-group-1', group_id='group-1')
vol_1 = MockOSResource(id='volume-1')
vol_2 = MockOSResource(id='volume-2')
vol_snaps = [MockOSResource(id='snap-volume-1', volume=vol_1),
MockOSResource(id='snap-volume-2', volume=vol_2)]
with mock.patch.object(self.adapter.client, 'create_cg_snap',
create=True) as mocked_create:
mocked_create.return_value = ({'status': 'available'},
[{'id': 'snap-volume-1',
'status': 'available'},
{'id': 'snap-volume-2',
'status': 'available'}])
ret = self.adapter.create_group_snapshot(cg_snap, vol_snaps)
mocked_create.assert_called_once_with('group-1',
snap_name='snap-group-1')
self.assertEqual({'status': 'available'}, ret[0])
self.assertEqual(2, len(ret[1]))
for snap_id in ('snap-volume-1', 'snap-volume-2'):
self.assertIn({'id': snap_id, 'status': 'available'}, ret[1])
def test_delete_group_snapshot(self):
group_snap = MockOSResource(id='snap-group-1')
cg_snap = test_client.MockResource(_id='snap_cg_1')
with mock.patch.object(self.adapter.client, 'get_snap',
create=True,
return_value=cg_snap) as mocked_get, \
mock.patch.object(self.adapter.client, 'delete_snap',
create=True) as mocked_delete:
ret = self.adapter.delete_group_snapshot(group_snap)
mocked_get.assert_called_once_with('snap-group-1')
mocked_delete.assert_called_once_with(cg_snap)
self.assertEqual((None, None), ret)
class FCAdapterTest(test.TestCase):
def setUp(self):

157
cinder/tests/unit/volume/drivers/dell_emc/unity/test_client.py
View File

@ -14,6 +14,7 @@
# under the License.
import unittest
import ddt
from mock import mock
from oslo_utils import units
@ -22,6 +23,7 @@ from cinder.tests.unit.volume.drivers.dell_emc.unity \
import fake_exception as ex
from cinder.volume.drivers.dell_emc.unity import client
########################
#
# Start of Mocks
@ -50,6 +52,9 @@ class MockResource(object):
self.host_cache = []
self.is_thin = None
self.is_all_flash = True
self.description = None
self.luns = None
self.lun = None
@property
def id(self):
@ -220,6 +225,14 @@ class MockResourceList(object):
def name(self):
return map(lambda i: i.name, self.resources)
@property
def list(self):
return self.resources
@list.setter
def list(self, value):
self.resources = []
def __iter__(self):
return self.resources.__iter__()
@ -327,6 +340,7 @@ def get_client():
# Start of Tests
#
########################
@ddt.ddt
@mock.patch.object(client, 'storops_ex', new=ex)
class ClientTest(unittest.TestCase):
def setUp(self):
@ -591,3 +605,146 @@ class ClientTest(unittest.TestCase):
self.client.host_cache['empty-host-in-cache'] = host
self.client.delete_host_wo_lock(host)
self.assertNotIn(host.name, self.client.host_cache)
@ddt.data(('cg_1', 'cg_1_description', [MockResource(_id='sv_1')]),
('cg_2', None, None),
('cg_3', None, [MockResource(_id='sv_2')]),
('cg_4', 'cg_4_description', None))
@ddt.unpack
def test_create_cg(self, cg_name, cg_description, lun_add):
created_cg = MockResource(_id='cg_1')
with mock.patch.object(self.client.system, 'create_cg',
create=True, return_value=created_cg
) as mocked_create:
ret = self.client.create_cg(cg_name, description=cg_description,
lun_add=lun_add)
mocked_create.assert_called_once_with(cg_name,
description=cg_description,
lun_add=lun_add)
self.assertEqual(created_cg, ret)
def test_create_cg_existing_name(self):
existing_cg = MockResource(_id='cg_1')
with mock.patch.object(
self.client.system, 'create_cg',
side_effect=ex.UnityConsistencyGroupNameInUseError,
create=True) as mocked_create, \
mock.patch.object(self.client.system, 'get_cg',
create=True,
return_value=existing_cg) as mocked_get:
ret = self.client.create_cg('existing_name')
mocked_create.assert_called_once_with('existing_name',
description=None,
lun_add=None)
mocked_get.assert_called_once_with(name='existing_name')
self.assertEqual(existing_cg, ret)
def test_get_cg(self):
existing_cg = MockResource(_id='cg_1')
with mock.patch.object(self.client.system, 'get_cg',
create=True,
return_value=existing_cg) as mocked_get:
ret = self.client.get_cg('existing_name')
mocked_get.assert_called_once_with(name='existing_name')
self.assertEqual(existing_cg, ret)
def test_get_cg_not_found(self):
with mock.patch.object(self.client.system, 'get_cg',
create=True,
side_effect=ex.UnityResourceNotFoundError
) as mocked_get:
ret = self.client.get_cg('not_found_name')
mocked_get.assert_called_once_with(name='not_found_name')
self.assertIsNone(ret)
def test_delete_cg(self):
existing_cg = MockResource(_id='cg_1')
with mock.patch.object(existing_cg, 'delete', create=True
) as mocked_delete, \
mock.patch.object(self.client, 'get_cg',
create=True,
return_value=existing_cg) as mocked_get:
ret = self.client.delete_cg('cg_1_name')
mocked_get.assert_called_once_with('cg_1_name')
mocked_delete.assert_called_once()
self.assertIsNone(ret)
def test_update_cg(self):
existing_cg = MockResource(_id='cg_1')
lun_1 = MockResource(_id='sv_1')
lun_2 = MockResource(_id='sv_2')
lun_3 = MockResource(_id='sv_3')
def _mocked_get_lun(lun_id):
if lun_id == 'sv_1':
return lun_1
if lun_id == 'sv_2':
return lun_2
if lun_id == 'sv_3':
return lun_3
with mock.patch.object(existing_cg, 'update_lun', create=True
) as mocked_update, \
mock.patch.object(self.client, 'get_cg',
create=True,
return_value=existing_cg) as mocked_get, \
mock.patch.object(self.client, 'get_lun',
side_effect=_mocked_get_lun):
ret = self.client.update_cg('cg_1_name', ['sv_1', 'sv_2'],
['sv_3'])
mocked_get.assert_called_once_with('cg_1_name')
mocked_update.assert_called_once_with(add_luns=[lun_1, lun_2],
remove_luns=[lun_3])
self.assertIsNone(ret)
def test_update_cg_empty_lun_ids(self):
existing_cg = MockResource(_id='cg_1')
with mock.patch.object(existing_cg, 'update_lun', create=True
) as mocked_update, \
mock.patch.object(self.client, 'get_cg',
create=True,
return_value=existing_cg) as mocked_get:
ret = self.client.update_cg('cg_1_name', set(), set())
mocked_get.assert_called_once_with('cg_1_name')
mocked_update.assert_called_once_with(add_luns=[], remove_luns=[])
self.assertIsNone(ret)
def test_create_cg_group(self):
existing_cg = MockResource(_id='cg_1')
created_snap = MockResource(_id='snap_cg_1', name='snap_name_cg_1')
with mock.patch.object(existing_cg, 'create_snap', create=True,
return_value=created_snap) as mocked_create, \
mock.patch.object(self.client, 'get_cg',
create=True,
return_value=existing_cg) as mocked_get:
ret = self.client.create_cg_snap('cg_1_name',
snap_name='snap_name_cg_1')
mocked_get.assert_called_once_with('cg_1_name')
mocked_create.assert_called_once_with(name='snap_name_cg_1',
is_auto_delete=False)
self.assertEqual(created_snap, ret)
def test_create_cg_group_none_name(self):
existing_cg = MockResource(_id='cg_1')
created_snap = MockResource(_id='snap_cg_1')
with mock.patch.object(existing_cg, 'create_snap', create=True,
return_value=created_snap) as mocked_create, \
mock.patch.object(self.client, 'get_cg',
create=True,
return_value=existing_cg) as mocked_get:
ret = self.client.create_cg_snap('cg_1_name')
mocked_get.assert_called_once_with('cg_1_name')
mocked_create.assert_called_once_with(name=None,
is_auto_delete=False)
self.assertEqual(created_snap, ret)
def test_filter_snaps_in_cg_snap(self):
snaps = [MockResource(_id='snap_{}'.format(n)) for n in (1, 2)]
snap_list = mock.MagicMock()
snap_list.list = snaps
with mock.patch.object(self.client.system, 'get_snap',
create=True,
return_value=snap_list) as mocked_get:
ret = self.client.filter_snaps_in_cg_snap('snap_cg_1')
mocked_get.assert_called_once_with(snap_group='snap_cg_1')
self.assertEqual(snaps, ret)

143
cinder/tests/unit/volume/drivers/dell_emc/unity/test_driver.py
View File

@ -13,8 +13,11 @@
# License for the specific language governing permissions and limitations
# under the License.
import functools
import unittest
import mock
from cinder.tests.unit.volume.drivers.dell_emc.unity \
import fake_exception as ex
from cinder.tests.unit.volume.drivers.dell_emc.unity import test_adapter
@ -104,6 +107,34 @@ class MockAdapter(object):
def migrate_volume(volume, host):
return True, {}
@staticmethod
def create_group(group):
return group
@staticmethod
def delete_group(group):
return group
@staticmethod
def update_group(group, add_volumes, remove_volumes):
return group, add_volumes, remove_volumes
@staticmethod
def create_group_from_snap(group, volumes, group_snapshot, snapshots):
return group, volumes, group_snapshot, snapshots
@staticmethod
def create_cloned_group(group, volumes, source_group, source_vols):
return group, volumes, source_group, source_vols
@staticmethod
def create_group_snapshot(group_snapshot, snapshots):
return group_snapshot, snapshots
@staticmethod
def delete_group_snapshot(group_snapshot):
return group_snapshot
########################
#
@ -112,16 +143,41 @@ class MockAdapter(object):
########################
patch_check_cg = mock.patch(
'cinder.volume.utils.is_group_a_cg_snapshot_type',
side_effect=lambda g: not g.id.endswith('_generic'))
class UnityDriverTest(unittest.TestCase):
@staticmethod
def get_volume():
return test_adapter.MockOSResource(provider_location='id^lun_43',
id='id_43')
@staticmethod
def get_generic_group():
return test_adapter.MockOSResource(name='group_name_generic',
id='group_id_generic')
@staticmethod
def get_cg():
return test_adapter.MockOSResource(name='group_name_cg',
id='group_id_cg')
@classmethod
def get_snapshot(cls):
return test_adapter.MockOSResource(volume=cls.get_volume())
@classmethod
def get_generic_group_snapshot(cls):
return test_adapter.MockOSResource(group=cls.get_generic_group(),
id='group_snapshot_id_generic')
@classmethod
def get_cg_group_snapshot(cls):
return test_adapter.MockOSResource(group=cls.get_cg(),
id='group_snapshot_id_cg')
@staticmethod
def get_context():
return None
@ -279,3 +335,90 @@ class UnityDriverTest(unittest.TestCase):
volume = self.get_volume()
r = self.driver.revert_to_snapshot(None, volume, snapshot)
self.assertTrue(r)
@patch_check_cg
def test_operate_generic_group_not_implemented(self, _):
group = self.get_generic_group()
context = self.get_context()
for func in (self.driver.create_group, self.driver.update_group):
self.assertRaises(NotImplementedError,
functools.partial(func, context, group))
volumes = [self.get_volume()]
for func in (self.driver.delete_group,
self.driver.create_group_from_src):
self.assertRaises(NotImplementedError,
functools.partial(func, context, group, volumes))
group_snap = self.get_generic_group_snapshot()
volume_snaps = [self.get_snapshot()]
for func in (self.driver.create_group_snapshot,
self.driver.delete_group_snapshot):
self.assertRaises(NotImplementedError,
functools.partial(func, context, group_snap,
volume_snaps))
@patch_check_cg
def test_create_group_cg(self, _):
cg = self.get_cg()
ret = self.driver.create_group(self.get_context(), cg)
self.assertEqual(ret, cg)
@patch_check_cg
def test_delete_group_cg(self, _):
cg = self.get_cg()
volumes = [self.get_volume()]
ret = self.driver.delete_group(self.get_context(), cg, volumes)
self.assertEqual(ret, cg)
@patch_check_cg
def test_update_group_cg(self, _):
cg = self.get_cg()
volumes = [self.get_volume()]
ret = self.driver.update_group(self.get_context(), cg,
add_volumes=volumes)
self.assertEqual(ret[0], cg)
self.assertListEqual(ret[1], volumes)
self.assertIsNone(ret[2])
@patch_check_cg
def test_create_group_from_src_group(self, _):
cg = self.get_cg()
volumes = [self.get_volume()]
source_group = cg
ret = self.driver.create_group_from_src(self.get_context(), cg,
volumes,
source_group=source_group)
self.assertEqual(ret[0], cg)
self.assertListEqual(ret[1], volumes)
self.assertEqual(ret[2], source_group)
self.assertIsNone(ret[3])
@patch_check_cg
def test_create_group_from_src_group_snapshot(self, _):
cg = self.get_cg()
volumes = [self.get_volume()]
cg_snap = self.get_cg_group_snapshot()
ret = self.driver.create_group_from_src(self.get_context(), cg,
volumes,
group_snapshot=cg_snap)
self.assertEqual(ret[0], cg)
self.assertListEqual(ret[1], volumes)
self.assertEqual(ret[2], cg_snap)
self.assertIsNone(ret[3])
@patch_check_cg
def test_create_group_snapshot_cg(self, _):
cg_snap = self.get_cg_group_snapshot()
ret = self.driver.create_group_snapshot(self.get_context(), cg_snap,
None)
self.assertEqual(ret[0], cg_snap)
self.assertIsNone(ret[1])
@patch_check_cg
def test_delete_group_snapshot_cg(self, _):
cg_snap = self.get_cg_group_snapshot()
ret = self.driver.delete_group_snapshot(self.get_context(), cg_snap,
None)
self.assertEqual(ret, cg_snap)

137
cinder/volume/drivers/dell_emc/unity/adapter.py
View File

@ -26,6 +26,7 @@ from oslo_utils import importutils
from cinder import exception
from cinder.i18n import _
from cinder.objects import fields
from cinder import utils as cinder_utils
from cinder.volume.drivers.dell_emc.unity import client
from cinder.volume.drivers.dell_emc.unity import utils
@ -59,6 +60,7 @@ class VolumeParams(object):
self._io_limit_policy = None
self._is_thick = None
self._is_compressed = None
self._is_in_cg = None
@property
def volume_id(self):
@ -133,13 +135,29 @@ class VolumeParams(object):
def is_compressed(self, value):
self._is_compressed = value
@property
def is_in_cg(self):
if self._is_in_cg is None:
self._is_in_cg = (self._volume.group and
vol_utils.is_group_a_cg_snapshot_type(
self._volume.group))
return self._is_in_cg
@property
def cg_id(self):
if self.is_in_cg:
return self._volume.group_id
return None
def __eq__(self, other):
return (self.volume_id == other.volume_id and
self.name == other.name and
self.size == other.size and
self.io_limit_policy == other.io_limit_policy and
self.is_thick == other.is_thick and
self.is_compressed == other.is_compressed)
self.is_compressed == other.is_compressed and
self.is_in_cg == other.is_in_cg and
self.cg_id == other.cg_id)
class CommonAdapter(object):
@ -302,22 +320,30 @@ class CommonAdapter(object):
'pool': params.pool,
'io_limit_policy': params.io_limit_policy,
'is_thick': params.is_thick,
'is_compressed': params.is_compressed
'is_compressed': params.is_compressed,
'cg_id': params.cg_id
}
LOG.info('Create Volume: %(name)s, size: %(size)s, description: '
'%(description)s, pool: %(pool)s, io limit policy: '
'%(io_limit_policy)s, thick: %(is_thick)s, '
'%(is_compressed)s.', log_params)
'compressed: %(is_compressed)s, cg_group: %(cg_id)s.',
log_params)
return self.makeup_model(
self.client.create_lun(name=params.name,
lun = self.client.create_lun(
name=params.name,
size=params.size,
pool=params.pool,
description=params.description,
io_limit_policy=params.io_limit_policy,
is_thin=False if params.is_thick else None,
is_compressed=params.is_compressed))
is_compressed=params.is_compressed)
if params.cg_id:
LOG.debug('Adding lun %(lun)s to cg %(cg)s.',
{'lun': lun.get_id(), 'cg': params.cg_id})
self.client.update_cg(params.cg_id, [lun.get_id()], ())
return self.makeup_model(lun)
def delete_volume(self, volume):
lun_id = self.get_lun_id(volume)
@ -442,12 +468,11 @@ class CommonAdapter(object):
lun_id=lun_id,
version=self.version)
@utils.append_capabilities
def update_volume_stats(self):
return {
'volume_backend_name': self.volume_backend_name,
'storage_protocol': self.protocol,
'thin_provisioning_support': True,
'thick_provisioning_support': True,
'pools': self.get_pools_stats(),
}
@ -459,6 +484,7 @@ class CommonAdapter(object):
def pools(self):
return self.storage_pools_map.values()
@utils.append_capabilities
def _get_pool_stats(self, pool):
return {
'pool_name': pool.name,
@ -470,8 +496,6 @@ class CommonAdapter(object):
'location_info': ('%(pool_name)s|%(array_serial)s' %
{'pool_name': pool.name,
'array_serial': self.serial_number}),
'thin_provisioning_support': True,
'thick_provisioning_support': True,
'compression_support': pool.is_all_flash,
'max_over_subscription_ratio': (
self.max_over_subscription_ratio),
@ -643,9 +667,7 @@ class CommonAdapter(object):
if src_lun is None:
# If size is not specified, need to get the size from LUN
# of snapshot.
lun = self.client.get_lun(
lun_id=src_snap.storage_resource.get_id())
size_in_m = utils.byte_to_mib(lun.size_total)
size_in_m = utils.byte_to_mib(src_snap.size)
else:
size_in_m = utils.byte_to_mib(src_lun.size_total)
vol_utils.copy_volume(
@ -814,6 +836,95 @@ class CommonAdapter(object):
'host-assisted migration.')
return False, None
def create_group(self, group):
"""Creates a generic group.
:param group: group information
"""
cg_name = group.id
description = group.description if group.description else group.name
LOG.info('Create group: %(name)s, description: %(description)s',
{'name': cg_name, 'description': description})
self.client.create_cg(cg_name, description=description)
return {'status': fields.GroupStatus.AVAILABLE}
def delete_group(self, group):
"""Deletes the generic group.
:param group: the group to delete
"""
# Deleting cg will also delete all the luns in it.
self.client.delete_cg(group.id)
return None, None
def update_group(self, group, add_volumes, remove_volumes):
add_lun_ids = (set(map(self.get_lun_id, add_volumes)) if add_volumes
else set())
remove_lun_ids = (set(map(self.get_lun_id, remove_volumes))
if remove_volumes else set())
self.client.update_cg(group.id, add_lun_ids, remove_lun_ids)
return {'status': fields.GroupStatus.AVAILABLE}, None, None
def copy_luns_in_group(self, group, volumes, src_cg_snap, src_volumes):
# Use dd to copy data here. The reason why not using thinclone is:
# 1. Cannot use cg thinclone due to the tight couple between source
# group and cloned one.
# 2. Cannot use lun thinclone due to clone lun in cg is not supported.
lun_snaps = self.client.filter_snaps_in_cg_snap(src_cg_snap.id)
# Make sure the `lun_snaps` is as order of `src_volumes`
src_lun_ids = [self.get_lun_id(volume) for volume in src_volumes]
lun_snaps.sort(key=lambda snap: src_lun_ids.index(snap.lun.id))
dest_luns = [self._dd_copy(VolumeParams(self, dest_volume), lun_snap)
for dest_volume, lun_snap in zip(volumes, lun_snaps)]
self.client.create_cg(group.id, lun_add=dest_luns)
return ({'status': fields.GroupStatus.AVAILABLE},
[{'id': dest_volume.id, 'status': fields.GroupStatus.AVAILABLE}
for dest_volume in volumes])
def create_group_from_snap(self, group, volumes,
group_snapshot, snapshots):
src_cg_snap = self.client.get_snap(group_snapshot.id)
src_vols = ([snap.volume for snap in snapshots] if snapshots else [])
return self.copy_luns_in_group(group, volumes, src_cg_snap, src_vols)
def create_cloned_group(self, group, volumes, source_group, source_vols):
src_group_snap_name = 'snap_clone_group_{}'.format(source_group.id)
create_snap_func = functools.partial(self.client.create_cg_snap,
source_group.id,
src_group_snap_name)
with utils.assure_cleanup(create_snap_func,
self.client.delete_snap,
True) as src_cg_snap:
LOG.debug('Internal group snapshot for clone is created, '
'name: %(name)s, id: %(id)s.',
{'name': src_group_snap_name,
'id': src_cg_snap.get_id()})
source_vols = source_vols if source_vols else []
return self.copy_luns_in_group(group, volumes, src_cg_snap,
source_vols)
def create_group_snapshot(self, group_snapshot, snapshots):
self.client.create_cg_snap(group_snapshot.group_id,
snap_name=group_snapshot.id)
model_update = {'status': fields.GroupStatus.AVAILABLE}
snapshots_model_update = [{'id': snapshot.id,
'status': fields.SnapshotStatus.AVAILABLE}
for snapshot in snapshots]
return model_update, snapshots_model_update
def delete_group_snapshot(self, group_snapshot):
cg_snap = self.client.get_snap(group_snapshot.id)
self.client.delete_snap(cg_snap)
return None, None
class ISCSIAdapter(CommonAdapter):
protocol = PROTOCOL_ISCSI

41
cinder/volume/drivers/dell_emc/unity/client.py
View File

@ -58,7 +58,7 @@ class UnityClient(object):
def create_lun(self, name, size, pool, description=None,
io_limit_policy=None, is_thin=None,
is_compressed=None):
is_compressed=None, cg_name=None):
"""Creates LUN on the Unity system.
:param name: lun name
@ -68,6 +68,7 @@ class UnityClient(object):
:param io_limit_policy: io limit on the LUN
:param is_thin: if False, a thick LUN will be created
:param is_compressed: is compressed LUN enabled
:param cg_name: the name of cg to join if any
:return: UnityLun object
"""
try:
@ -349,3 +350,41 @@ class UnityClient(object):
def restore_snapshot(self, snap_name):
snap = self.get_snap(snap_name)
return snap.restore(delete_backup=True)
def create_cg(self, name, description=None, lun_add=None):
try:
cg = self.system.create_cg(name, description=description,
lun_add=lun_add)
except storops_ex.UnityConsistencyGroupNameInUseError:
LOG.debug('CG %s already exists. Return the existing one.', name)
cg = self.system.get_cg(name=name)
return cg
def get_cg(self, name):
try:
cg = self.system.get_cg(name=name)
except storops_ex.UnityResourceNotFoundError:
LOG.info('CG %s not found.', name)
return None
else:
return cg
def delete_cg(self, name):
cg = self.get_cg(name)
if cg:
cg.delete() # Deleting cg will also delete the luns in it
def update_cg(self, name, add_lun_ids, remove_lun_ids):
cg = self.get_cg(name)
cg.update_lun(add_luns=[self.get_lun(lun_id=lun_id)
for lun_id in add_lun_ids],
remove_luns=[self.get_lun(lun_id=lun_id)
for lun_id in remove_lun_ids])
def create_cg_snap(self, cg_name, snap_name=None):
cg = self.get_cg(cg_name)
# Creating snap of cg will create corresponding snaps of luns in it
return cg.create_snap(name=snap_name, is_auto_delete=False)
def filter_snaps_in_cg_snap(self, cg_snap_id):
return self.system.get_snap(snap_group=cg_snap_id).list

60
cinder/volume/drivers/dell_emc/unity/driver.py
View File

@ -18,11 +18,14 @@
from oslo_config import cfg
from oslo_log import log as logging
import six
from cinder import interface
from cinder.volume import configuration
from cinder.volume import driver
from cinder.volume.drivers.dell_emc.unity import adapter
from cinder.volume.drivers.san.san import san_opts
from cinder.volume import utils
from cinder.zonemanager import utils as zm_utils
LOG = logging.getLogger(__name__)
@ -46,6 +49,20 @@ UNITY_OPTS = [
CONF.register_opts(UNITY_OPTS, group=configuration.SHARED_CONF_GROUP)
def skip_if_not_cg(func):
@six.wraps(func)
def inner(self, *args, **kwargs):
# Only used to decorating the second argument is `group`
if utils.is_group_a_cg_snapshot_type(args[1]):
return func(self, *args, **kwargs)
LOG.debug('Group is not a consistency group. Unity driver does '
'nothing.')
# This exception will let cinder handle it as a generic group
raise NotImplementedError()
return inner
@interface.volumedriver
class UnityDriver(driver.ManageableVD,
driver.ManageableSnapshotsVD,
@ -60,9 +77,10 @@ class UnityDriver(driver.ManageableVD,
4.0.0 - Support remove empty host
4.2.0 - Support compressed volume
5.0.0 - Support storage assisted volume migration
6.0.0 - Support generic group and consistent group
"""
VERSION = '05.00.00'
VERSION = '06.00.00'
VENDOR = 'Dell EMC'
# ThirdPartySystems wiki page
CI_WIKI_NAME = "EMC_UNITY_CI"
@ -252,3 +270,43 @@ class UnityDriver(driver.ManageableVD,
def revert_to_snapshot(self, context, volume, snapshot):
"""Reverts a volume to a snapshot."""
return self.adapter.restore_snapshot(volume, snapshot)
@skip_if_not_cg
def create_group(self, context, group):
"""Creates a consistency group."""
return self.adapter.create_group(group)
@skip_if_not_cg
def delete_group(self, context, group, volumes):
"""Deletes a consistency group."""
return self.adapter.delete_group(group)
@skip_if_not_cg
def update_group(self, context, group, add_volumes=None,
remove_volumes=None):
"""Updates a consistency group, i.e. add/remove luns to/from it."""
# TODO(Ryan L) update other information (like description) of group
return self.adapter.update_group(group, add_volumes, remove_volumes)
@skip_if_not_cg
def create_group_from_src(self, context, group, volumes,
group_snapshot=None, snapshots=None,
source_group=None, source_vols=None):
"""Creates a consistency group from another group or group snapshot."""
if group_snapshot:
return self.adapter.create_group_from_snap(group, volumes,
group_snapshot,
snapshots)
elif source_group:
return self.adapter.create_cloned_group(group, volumes,
source_group, source_vols)
@skip_if_not_cg
def create_group_snapshot(self, context, group_snapshot, snapshots):
"""Creates a snapshot of consistency group."""
return self.adapter.create_group_snapshot(group_snapshot, snapshots)
@skip_if_not_cg
def delete_group_snapshot(self, context, group_snapshot, snapshots):
"""Deletes a snapshot of consistency group."""
return self.adapter.delete_group_snapshot(group_snapshot)

16
cinder/volume/drivers/dell_emc/unity/utils.py
View File

@ -319,6 +319,22 @@ def lock_if(condition, lock_name):
return functools.partial
def append_capabilities(func):
capabilities = {
'thin_provisioning_support': True,
'thick_provisioning_support': True,
'consistent_group_snapshot_enabled': True
}
@six.wraps(func)
def _inner(*args, **kwargs):
output = func(*args, **kwargs)
output.update(capabilities)
return output
return _inner
def is_multiattach_to_host(volume_attachment, host_name):
# When multiattach is enabled, a volume could be attached to two or more
# instances which are hosted on one nova host.

25
doc/source/configuration/block-storage/drivers/dell-emc-unity-driver.rst
View File

@ -35,6 +35,11 @@ Supported operations
- Efficient non-disruptive volume backup.
- Revert a volume to a snapshot.
- Create thick volumes.
- Create and delete consistent groups.
- Add/remove volumes to/from a consistent group.
- Create and delete consistent group snapshots.
- Clone a consistent group.
- Create a consistent group from a snapshot.
- Attach a volume to multiple servers simultaneously (multiattach).
Driver configuration
@ -382,6 +387,26 @@ attached hosts.
For more detail, please refer to
https://developer.openstack.org/api-ref/block-storage/v2/?expanded=force-detach-volume-detail#force-detach-volume
Consistent group support
~~~~~~~~~~~~~~~~~~~~~~~~
For a group to support consistent group snapshot, the group specs in the
corresponding group type should have the following entry:
.. code-block:: ini
{'consistent_group_snapshot_enabled': <is> True}
Similarly, for a volume to be in a group that supports consistent group
snapshots, the volume type extra specs would also have the following entry:
.. code-block:: ini
{'consistent_group_snapshot_enabled': <is> True}
Refer to https://docs.openstack.org/cinder/latest/admin/blockstorage-groups.html
for command lines detail.
Troubleshooting
~~~~~~~~~~~~~~~

2
doc/source/reference/support-matrix.ini
View File

@ -554,7 +554,7 @@ driver.datera=missing
driver.dell_emc_powermax=complete
driver.dell_emc_ps=missing
driver.dell_emc_sc=complete
driver.dell_emc_unity=missing
driver.dell_emc_unity=complete
driver.dell_emc_vmax_af=complete
driver.dell_emc_vmax_3=complete
driver.dell_emc_vnx=complete

11
releasenotes/notes/support-cg-2b55da0bd9f69c7d.yaml Normal file
View File

@ -0,0 +1,11 @@
---
features:
- |
Dell EMC Unity driver: Add consistent group support. Users could create a
group type supporting consistent groups with specification
`'consistent_group_snapshot_enabled': <is> True`, then any groups created
of that group type are consistent groups, otherwise they are generic
groups. The supported operations are: create/delete consistent groups, add
volumes to and remove volumes from consistent groups, create/delete
consistent group snapshots, create consistent groups from snapshots, clone
consistent groups.