openstack/manila
Code Issues Proposed changes

Merge "Consistency Group Support for the Generic Driver"

Browse Source
This commit is contained in:
Jenkins 2015-09-07 06:52:35 +00:00 committed by Gerrit Code Review
commit c4dbdb414b
4 changed files with 461 additions and 4 deletions
contrib/ci
post_test_hook.sh
manila
share
drivers
generic.py
manager.py
tests/share/drivers
test_generic.py

2
contrib/ci/post_test_hook.sh

@ -88,7 +88,7 @@ fi
set +o errexit
cd $BASE/new/tempest
export MANILA_TEMPEST_CONCURRENCY=${MANILA_TEMPEST_CONCURRENCY:-12}
export MANILA_TEMPEST_CONCURRENCY=${MANILA_TEMPEST_CONCURRENCY:-6}
export MANILA_TESTS=${MANILA_TESTS:-'manila_tempest_tests.tests.api'}
if [[ "$JOB_NAME" =~ "scenario" ]]; then

180
manila/share/drivers/generic.py

@ -108,7 +108,7 @@ def ensure_server(f):
raise exception.ManilaException(
_("Share server handling is not available. "
"But 'share_server' was provided. '%s'. "
"Share network should not be used.") % server['id'])
"Share network should not be used.") % server.get('id'))
elif not server:
raise exception.ManilaException(
_("Share server handling is enabled. But 'share_server' "
@ -566,7 +566,9 @@ class GenericShareDriver(driver.ExecuteMixin, driver.ShareDriver):
data = dict(
share_backend_name=self.backend_name,
storage_protocol='NFS_CIFS',
reserved_percentage=(self.configuration.reserved_share_percentage))
reserved_percentage=self.configuration.reserved_share_percentage,
consistency_group_support='pool',
)
super(GenericShareDriver, self)._update_share_stats(data)
@ensure_server
@ -952,6 +954,180 @@ class GenericShareDriver(driver.ExecuteMixin, driver.ShareDriver):
return size
@ensure_server
def create_consistency_group(self, context, cg_dict, share_server=None):
"""Creates a consistency group.
Since we are faking the CG object, apart from verifying if the
share_server is valid, we do nothing else here.
"""
LOG.debug('Created a Consistency Group with ID: %s.', cg_dict['id'])
msg = _LW('The Generic driver has no means to guarantee consistency '
'group snapshots are actually consistent. This '
'implementation is for reference and testing purposes only.')
LOG.warning(msg)
def delete_consistency_group(self, context, cg_dict, share_server=None):
"""Deletes a consistency group.
Since we are faking the CG object we do nothing here.
"""
LOG.debug('Deleted the consistency group with ID %s.', cg_dict['id'])
def _cleanup_cg_share_snapshot(self, context, share_snapshot,
share_server):
"""Deletes the snapshot of a share belonging to a consistency group."""
try:
self.delete_snapshot(context, share_snapshot, share_server)
except exception.ManilaException:
msg = _LE('Could not delete CG Snapshot %(snap)s '
'for share %(share)s.')
LOG.error(msg % {
'snap': share_snapshot['id'],
'share': share_snapshot['share_id'],
})
raise
@ensure_server
def create_cgsnapshot(self, context, snap_dict, share_server=None):
"""Creates a consistency group snapshot one or more shares."""
LOG.debug('Attempting to create a CG snapshot %s.' % snap_dict['id'])
msg = _LW('The Consistency Group Snapshot being created is '
'not expected to be consistent. This implementation is '
'for reference and testing purposes only.')
LOG.warning(msg)
cg_members = snap_dict.get('cgsnapshot_members', [])
if not cg_members:
LOG.warning(_LW('No shares in Consistency Group to Create CG '
'snapshot.'))
else:
share_snapshots = []
for member in cg_members:
share_snapshot = {
'share_id': member['share_id'],
'id': member['id'],
}
try:
self.create_snapshot(context, share_snapshot, share_server)
share_snapshots.append(share_snapshot)
except exception.ManilaException as e:
msg = _LE('Could not create CG Snapshot. Failed '
'to create share snapshot %(snap)s for '
'share %(share)s.')
LOG.exception(msg % {
'snap': share_snapshot['id'],
'share': share_snapshot['share_id']
})
# clean up any share snapshots previously created
LOG.debug('Attempting to clean up snapshots due to '
'failure...')
for share_snapshot in share_snapshots:
self._cleanup_cg_share_snapshot(context,
share_snapshot,
share_server)
raise e
LOG.debug('Successfully created CG snapshot %s.' % snap_dict['id'])
return None, None
@ensure_server
def delete_cgsnapshot(self, context, snap_dict, share_server=None):
"""Deletes a consistency group snapshot."""
cg_members = snap_dict.get('cgsnapshot_members', [])
LOG.debug('Deleting CG snapshot %s.' % snap_dict['id'])
for member in cg_members:
share_snapshot = {
'share_id': member['share_id'],
'id': member['id'],
}
self._cleanup_cg_share_snapshot(context,
share_snapshot,
share_server)
LOG.debug('Deleted CG snapshot %s.' % snap_dict['id'])
return None, None
@ensure_server
def create_consistency_group_from_cgsnapshot(self, context, cg_dict,
cgsnapshot_dict,
share_server=None):
"""Creates a consistency group from an existing CG snapshot."""
# Ensure that the consistency group snapshot has members
if not cgsnapshot_dict['cgsnapshot_members']:
return None, None
clone_list = self._collate_cg_snapshot_info(cg_dict, cgsnapshot_dict)
share_update_list = list()
LOG.debug('Creating consistency group from CG snapshot %s.',
cgsnapshot_dict['id'])
for clone in clone_list:
kwargs = {}
if self.driver_handles_share_servers:
kwargs['share_server'] = share_server
export_location = (
self.create_share_from_snapshot(
context,
clone['share'],
clone['snapshot'],
**kwargs))
share_update_list.append({
'id': clone['share']['id'],
'export_locations': export_location,
})
return None, share_update_list
def _collate_cg_snapshot_info(self, cg_dict, cgsnapshot_dict):
"""Collate the data for a clone of the CG snapshot.
Given two data structures, a CG snapshot (cgsnapshot_dict) and a new
CG to be cloned from the snapshot (cg_dict), match up both
structures into a list of dicts (share & snapshot) suitable for use
by existing method that clones individual share snapshots.
"""
clone_list = list()
for share in cg_dict['shares']:
clone_info = {'share': share}
for cgsnapshot_member in cgsnapshot_dict['cgsnapshot_members']:
if (share['source_cgsnapshot_member_id'] ==
cgsnapshot_member['id']):
clone_info['snapshot'] = {
'id': cgsnapshot_member['id'],
}
break
if len(clone_info) != 2:
msg = _("Invalid data supplied for creating consistency "
"group from CG snapshot %s.") % cgsnapshot_dict['id']
raise exception.InvalidConsistencyGroup(reason=msg)
clone_list.append(clone_info)
return clone_list
class NASHelperBase(object):
"""Interface to work with share."""

2
manila/share/manager.py

@ -1431,7 +1431,7 @@ class ShareManager(manager.SchedulerDependentManager):
share_network_id = group_ref.get('share_network_id', None)
share_server = None
if parent_share_server_id:
if parent_share_server_id and self.driver.driver_handles_share_servers:
share_server = self.db.share_server_get(context,
parent_share_server_id)
share_network_id = share_server['share_network_id']

281
manila/tests/share/drivers/test_generic.py

@ -57,6 +57,96 @@ def get_fake_manage_share():
}
def get_fake_snap_dict():
snap_dict = {
'status': 'available',
'project_id': '13c0be6290934bd98596cfa004650049',
'user_id': 'a0314a441ca842019b0952224aa39192',
'description': None,
'deleted': '0',
'created_at': '2015-08-10 00:05:58',
'updated_at': '2015-08-10 00:05:58',
'consistency_group_id': '4b04fdc3-00b9-4909-ba1a-06e9b3f88b67',
'cgsnapshot_members': [
{
'status': 'available',
'share_type_id': '1a9ed31e-ee70-483d-93ba-89690e028d7f',
'share_id': 'e14b5174-e534-4f35-bc4f-fe81c1575d6f',
'user_id': 'a0314a441ca842019b0952224aa39192',
'deleted': 'False',
'created_at': '2015-08-10 00:05:58',
'share': {
'id': '03e2f06e-14f2-45a5-9631-0949d1937bd8',
'deleted': False,
},
'updated_at': '2015-08-10 00:05:58',
'share_proto': 'NFS',
'project_id': '13c0be6290934bd98596cfa004650049',
'cgsnapshot_id': 'f6aa3b59-57eb-421e-965c-4e182538e36a',
'deleted_at': None,
'id': '03e2f06e-14f2-45a5-9631-0949d1937bd8',
'size': 1,
},
],
'deleted_at': None,
'id': 'f6aa3b59-57eb-421e-965c-4e182538e36a',
'name': None,
}
return snap_dict
def get_fake_cg_dict():
cg_dict = {
'status': 'creating',
'project_id': '13c0be6290934bd98596cfa004650049',
'user_id': 'a0314a441ca842019b0952224aa39192',
'description': None,
'deleted': 'False',
'created_at': '2015-08-10 00:07:58',
'updated_at': None,
'source_cgsnapshot_id': 'f6aa3b59-57eb-421e-965c-4e182538e36a',
'host': 'openstack2@cmodeSSVMNFS',
'deleted_at': None,
'shares': [
{
'id': '02a32f06e-14f2-45a5-9631-7483f1937bd8',
'deleted': False,
'source_cgsnapshot_member_id':
'03e2f06e-14f2-45a5-9631-0949d1937bd8',
},
],
'share_types': [
{
'id': 'f6aa3b56-45a5-9631-02a32f06e1937b',
'deleted': False,
'consistency_group_id': '4b04fdc3-00b9-4909-ba1a-06e9b3f88b67',
'share_type_id': '1a9ed31e-ee70-483d-93ba-89690e028d7f',
},
],
'id': 'eda52174-0442-476d-9694-a58327466c14',
'name': None
}
return cg_dict
def get_fake_collated_cg_snap_info():
fake_collated_cg_snap_info = [
{
'share': {
'id': '02a32f06e-14f2-45a5-9631-7483f1937bd8',
'deleted': False,
'source_cgsnapshot_member_id':
'03e2f06e-14f2-45a5-9631-0949d1937bd8',
},
'snapshot': {
'id': '03e2f06e-14f2-45a5-9631-0949d1937bd8',
},
},
]
return fake_collated_cg_snap_info
@ddt.ddt
class GenericShareDriverTestCase(test.TestCase):
"""Tests GenericShareDriver."""
@ -129,6 +219,10 @@ class GenericShareDriverTestCase(test.TestCase):
self.access = fake_share.fake_access()
self.snapshot = fake_share.fake_snapshot()
self.mock_object(time, 'sleep')
self.mock_debug_log = self.mock_object(generic.LOG, 'debug')
self.mock_warning_log = self.mock_object(generic.LOG, 'warning')
self.mock_error_log = self.mock_object(generic.LOG, 'error')
self.mock_exception_log = self.mock_object(generic.LOG, 'exception')
def test_do_setup(self):
self.mock_object(volume, 'API')
@ -1670,6 +1764,193 @@ class GenericShareDriverTestCase(test.TestCase):
self.assertEqual(result, actual_result)
def test_create_consistency_group(self):
FAKE_SNAP_DICT = get_fake_snap_dict()
result = self._driver.create_consistency_group(
self._context, FAKE_SNAP_DICT, share_server=self.server)
self.assertEqual(1, self.mock_debug_log.call_count)
self.assertEqual(1, self.mock_warning_log.call_count)
self.assertIsNone(result)
def test_delete_consistency_group(self):
FAKE_SNAP_DICT = get_fake_snap_dict()
result = self._driver.delete_consistency_group(
self._context, FAKE_SNAP_DICT, share_server=self.server)
self.assertEqual(1, self.mock_debug_log.call_count)
self.assertIsNone(result)
def test_create_cgsnapshot_no_cg_members(self):
FAKE_SNAP_DICT = dict(get_fake_snap_dict(), cgsnapshot_members=[])
mock_snapshot_creation = self.mock_object(generic.GenericShareDriver,
'create_snapshot')
result = self._driver.create_cgsnapshot(
self._context, FAKE_SNAP_DICT, share_server=self.server)
self.assertEqual(1, self.mock_debug_log.call_count)
self.assertEqual(2, self.mock_warning_log.call_count)
self.assertFalse(mock_snapshot_creation.called)
self.assertEqual((None, None), result)
@ddt.data(
{
'delete_snap_side_effect': None,
'expected_error_log_call_count': 0,
},
{
'delete_snap_side_effect': exception.ManilaException,
'expected_error_log_call_count': 1,
}
)
@ddt.unpack
def test_create_cgsnapshot_manila_exception_on_create_and_delete(
self, delete_snap_side_effect, expected_error_log_call_count):
FAKE_SNAP_DICT = get_fake_snap_dict()
# Append another fake share
FAKE_SHARE = dict(FAKE_SNAP_DICT['cgsnapshot_members'][0])
FAKE_SNAP_DICT['cgsnapshot_members'].append(FAKE_SHARE)
self.mock_object(generic.GenericShareDriver,
'create_snapshot',
mock.Mock(side_effect=[
None,
exception.ManilaException,
]))
self.mock_object(generic.GenericShareDriver,
'delete_snapshot',
mock.Mock(side_effect=delete_snap_side_effect))
self.assertRaises(exception.ManilaException,
self._driver.create_cgsnapshot,
self._context, FAKE_SNAP_DICT,
share_server=self.server)
self.assertEqual(2, self.mock_debug_log.call_count)
self.assertEqual(1, self.mock_warning_log.call_count)
self.assertEqual(1, self.mock_exception_log.call_count)
self.assertEqual(expected_error_log_call_count,
self.mock_error_log.call_count)
def test_create_cgsnapshot(self):
FAKE_SNAP_DICT = get_fake_snap_dict()
FAKE_SHARE_SNAPSHOT = {
'share_id': 'e14b5174-e534-4f35-bc4f-fe81c1575d6f',
'id': '03e2f06e-14f2-45a5-9631-0949d1937bd8',
}
mock_snapshot_creation = self.mock_object(generic.GenericShareDriver,
'create_snapshot')
result = self._driver.create_cgsnapshot(
self._context, FAKE_SNAP_DICT, share_server=self.server)
mock_snapshot_creation.assert_called_once_with(self._context,
FAKE_SHARE_SNAPSHOT,
self.server)
self.assertEqual(2, self.mock_debug_log.call_count)
self.assertEqual(1, self.mock_warning_log.call_count)
self.assertFalse(self.mock_error_log.called)
self.assertEqual((None, None), result)
def test_delete_cgsnapshot_manila_exception(self):
FAKE_SNAP_DICT = get_fake_snap_dict()
self.mock_object(generic.GenericShareDriver,
'delete_snapshot',
mock.Mock(side_effect=exception.ManilaException))
self.assertRaises(exception.ManilaException,
self._driver.delete_cgsnapshot,
self._context, FAKE_SNAP_DICT,
share_server=self.server)
self.assertEqual(1, self.mock_error_log.call_count)
def test_delete_cgsnapshot(self):
FAKE_SNAP_DICT = get_fake_snap_dict()
FAKE_SHARE_SNAPSHOT = {
'share_id': 'e14b5174-e534-4f35-bc4f-fe81c1575d6f',
'id': '03e2f06e-14f2-45a5-9631-0949d1937bd8',
}
mock_snapshot_creation = self.mock_object(generic.GenericShareDriver,
'delete_snapshot')
result = self._driver.delete_cgsnapshot(
self._context, FAKE_SNAP_DICT, share_server=self.server)
mock_snapshot_creation.assert_called_once_with(self._context,
FAKE_SHARE_SNAPSHOT,
self.server)
self.assertEqual(2, self.mock_debug_log.call_count)
self.assertEqual((None, None), result)
def test_create_consistency_group_from_cgsnapshot_no_members(self):
FAKE_CG_DICT = get_fake_cg_dict()
FAKE_CGSNAP_DICT = dict(get_fake_snap_dict(), cgsnapshot_members=[])
mock_share_creation = self.mock_object(generic.GenericShareDriver,
'create_share_from_snapshot')
result = self._driver.create_consistency_group_from_cgsnapshot(
self._context, FAKE_CG_DICT, FAKE_CGSNAP_DICT,
share_server=self.server)
self.assertFalse(self.mock_debug_log.called)
self.assertFalse(mock_share_creation.called)
self.assertEqual((None, None), result)
def test_create_consistency_group_from_cgsnapshot(self):
FAKE_CG_DICT = get_fake_cg_dict()
FAKE_CGSNAP_DICT = get_fake_snap_dict()
FAKE_COLLATED_INFO = get_fake_collated_cg_snap_info()
FAKE_SHARE_UPDATE_LIST = [
{
'id': '02a32f06e-14f2-45a5-9631-7483f1937bd8',
'export_locations': 'xyzzy',
}
]
self.mock_object(generic.GenericShareDriver,
'_collate_cg_snapshot_info',
mock.Mock(return_value=FAKE_COLLATED_INFO))
mock_share_creation = self.mock_object(generic.GenericShareDriver,
'create_share_from_snapshot',
mock.Mock(return_value='xyzzy'))
result = self._driver.create_consistency_group_from_cgsnapshot(
self._context, FAKE_CG_DICT, FAKE_CGSNAP_DICT,
share_server=self.server)
self.assertEqual((None, FAKE_SHARE_UPDATE_LIST), result)
self.assertEqual(1, self.mock_debug_log.call_count)
mock_share_creation.assert_called_once_with(
self._context,
FAKE_COLLATED_INFO[0]['share'],
FAKE_COLLATED_INFO[0]['snapshot'],
share_server=self.server
)
def test_collate_cg_snapshot_info_invalid_cg(self):
FAKE_CG_DICT = get_fake_cg_dict()
FAKE_CGSNAP_DICT = dict(get_fake_snap_dict(), cgsnapshot_members=[])
self.assertRaises(exception.InvalidConsistencyGroup,
self._driver._collate_cg_snapshot_info,
FAKE_CG_DICT,
FAKE_CGSNAP_DICT)
def test_collate_cg_snapshot(self):
FAKE_CG_DICT = get_fake_cg_dict()
FAKE_CGSNAP_DICT = get_fake_snap_dict()
FAKE_COLLATED_INFO = get_fake_collated_cg_snap_info()
result = self._driver._collate_cg_snapshot_info(
FAKE_CG_DICT, FAKE_CGSNAP_DICT)
self.assertEqual(FAKE_COLLATED_INFO, result)
@generic.ensure_server
def fake(driver_instance, context, share_server=None):