NetApp NFS: Clone image using copy file operation

The copy offload tool is no longer available available, so this patch adds
an alternative approach using file copy operations to the scenarios where
the tool was required. During the clone image operation, if the source
volume is not available in the same pool or cache, the copy file operation
is called to perform the action.

The copy offload tool will keeping working and it will be removed in the
next release (Antelope). There patch [1] deprecates the tool.

[1] https://review.opendev.org/c/openstack/cinder/+/847733

Implements: blueprint netapp-nfs-copy-offload-image
Change-Id: I4e2163661b913800a5a1179a5899ec1384f8c167
This commit is contained in:
Felipe Rodrigues 2022-04-26 11:31:10 -03:00
parent f97f7ff514
commit 147637b63f
4 changed files with 215 additions and 140 deletions

View File

@ -924,7 +924,8 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase):
drv._construct_image_nfs_url = mock.Mock(return_value=["nfs://1"]) drv._construct_image_nfs_url = mock.Mock(return_value=["nfs://1"])
drv._check_get_nfs_path_segs = mock.Mock( drv._check_get_nfs_path_segs = mock.Mock(
return_value=("test:test", "dr")) return_value=("test:test", "dr"))
drv._get_ip_verify_on_cluster = mock.Mock(return_value="192.128.1.1") drv._get_ip_verify_on_cluster = mock.Mock(return_value=("192.128.1.1",
"vserver"))
drv._get_mount_point_for_share = mock.Mock(return_value='mnt_point') drv._get_mount_point_for_share = mock.Mock(return_value='mnt_point')
drv._check_share_can_hold_size = mock.Mock() drv._check_share_can_hold_size = mock.Mock()
# Raise error as if the copyoffload file can not be found # Raise error as if the copyoffload file can not be found
@ -937,9 +938,9 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase):
drv._discover_file_till_timeout.assert_not_called() drv._discover_file_till_timeout.assert_not_called()
@mock.patch.object(image_utils, 'qemu_img_info') @ddt.data(True, False)
def test_copy_from_img_service_raw_copyoffload_workflow_success( def test_copy_from_img_service_raw_copyoffload_workflow_success(
self, mock_qemu_img_info): self, use_tool):
drv = self.driver drv = self.driver
volume = {'id': 'vol_id', 'name': 'name', 'size': 1, volume = {'id': 'vol_id', 'name': 'name', 'size': 1,
'host': 'openstack@nfscmode#ip1:/mnt_point'} 'host': 'openstack@nfscmode#ip1:/mnt_point'}
@ -952,16 +953,18 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase):
drv._check_get_nfs_path_segs =\ drv._check_get_nfs_path_segs =\
mock.Mock(return_value=('ip1', '/openstack')) mock.Mock(return_value=('ip1', '/openstack'))
drv._get_ip_verify_on_cluster = mock.Mock(return_value='ip1') drv._get_ip_verify_on_cluster = mock.Mock(return_value=('ip1',
'vserver'))
drv._get_host_ip = mock.Mock(return_value='ip2') drv._get_host_ip = mock.Mock(return_value='ip2')
drv._get_export_path = mock.Mock(return_value='/exp_path') drv._get_export_path = mock.Mock(return_value='/exp_path')
drv._get_provider_location = mock.Mock(return_value='share') drv._get_provider_location = mock.Mock(return_value='share')
drv._execute = mock.Mock() drv._execute = mock.Mock()
drv._copy_file = mock.Mock()
drv._get_mount_point_for_share = mock.Mock(return_value='mnt_point') drv._get_mount_point_for_share = mock.Mock(return_value='mnt_point')
drv._discover_file_till_timeout = mock.Mock(return_value=True) drv._discover_file_till_timeout = mock.Mock(return_value=True)
img_inf = mock.Mock() img_inf = mock.Mock()
img_inf.file_format = 'raw' img_inf.file_format = 'raw'
mock_qemu_img_info.return_value = img_inf image_utils.qemu_img_info.return_value = img_inf
drv._check_share_can_hold_size = mock.Mock() drv._check_share_can_hold_size = mock.Mock()
drv._move_nfs_file = mock.Mock(return_value=True) drv._move_nfs_file = mock.Mock(return_value=True)
drv._delete_file_at_path = mock.Mock() drv._delete_file_at_path = mock.Mock()
@ -969,13 +972,19 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase):
drv._post_clone_image = mock.Mock() drv._post_clone_image = mock.Mock()
retval = drv._copy_from_img_service( retval = drv._copy_from_img_service(
context, volume, image_service, image_id) context, volume, image_service, image_id,
use_copyoffload_tool=use_tool)
self.assertTrue(retval) self.assertTrue(retval)
drv._get_ip_verify_on_cluster.assert_any_call('ip1') drv._get_ip_verify_on_cluster.assert_any_call('ip1')
drv._check_share_can_hold_size.assert_called_with( drv._check_share_can_hold_size.assert_called_with(
'ip1:/mnt_point', 1) 'ip1:/mnt_point', 1)
if use_tool:
self.assertEqual(1, drv._execute.call_count) self.assertEqual(1, drv._execute.call_count)
self.assertEqual(0, drv._copy_file.call_count)
else:
self.assertEqual(1, drv._copy_file.call_count)
self.assertEqual(0, drv._execute.call_count)
@mock.patch.object(image_utils, 'convert_image') @mock.patch.object(image_utils, 'convert_image')
@mock.patch.object(image_utils, 'qemu_img_info') @mock.patch.object(image_utils, 'qemu_img_info')
@ -1007,7 +1016,8 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase):
mock.Mock(return_value=('203.0.113.122', '/openstack')) mock.Mock(return_value=('203.0.113.122', '/openstack'))
) )
drv._get_ip_verify_on_cluster = mock.Mock(return_value='203.0.113.122') drv._get_ip_verify_on_cluster = mock.Mock(
return_value=('203.0.113.122', 'vserver'))
drv._execute = mock.Mock() drv._execute = mock.Mock()
drv._execute_as_root = False drv._execute_as_root = False
drv._get_mount_point_for_share = mock.Mock( drv._get_mount_point_for_share = mock.Mock(
@ -1054,7 +1064,8 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase):
'host': 'openstack@nfscmode#192.128.1.1:/exp_path'} 'host': 'openstack@nfscmode#192.128.1.1:/exp_path'}
image_id = 'image_id' image_id = 'image_id'
cache_result = [('ip1:/openstack', 'img-cache-imgid')] cache_result = [('ip1:/openstack', 'img-cache-imgid')]
drv._get_ip_verify_on_cluster = mock.Mock(return_value='ip1') drv._get_ip_verify_on_cluster = mock.Mock(return_value=('ip1',
'vserver'))
drv._execute = mock.Mock() drv._execute = mock.Mock()
drv._register_image_in_cache = mock.Mock() drv._register_image_in_cache = mock.Mock()
drv._post_clone_image = mock.Mock() drv._post_clone_image = mock.Mock()
@ -1241,25 +1252,29 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase):
@ddt.unpack @ddt.unpack
def test_get_source_ip_and_path(self, share, ip): def test_get_source_ip_and_path(self, share, ip):
self.driver._get_ip_verify_on_cluster = mock.Mock( self.driver._get_ip_verify_on_cluster = mock.Mock(
return_value=ip) return_value=(ip, fake.VSERVER_NAME))
src_ip, src_path = self.driver._get_source_ip_and_path( src_ip, src_vserver, src_share, src_path = (
share, fake.IMAGE_FILE_ID) self.driver._get_source_ip_and_path(
share, fake.IMAGE_FILE_ID))
self.assertEqual(ip, src_ip) self.assertEqual(ip, src_ip)
self.assertEqual(fake.VSERVER_NAME, src_vserver)
self.assertEqual(fake.EXPORT_PATH, src_share)
assert_path = fake.EXPORT_PATH + '/' + fake.IMAGE_FILE_ID assert_path = fake.EXPORT_PATH + '/' + fake.IMAGE_FILE_ID
self.assertEqual(assert_path, src_path) self.assertEqual(assert_path, src_path)
self.driver._get_ip_verify_on_cluster.assert_called_once_with(ip) self.driver._get_ip_verify_on_cluster.assert_called_once_with(ip)
def test_get_destination_ip_and_path(self): def test_get_destination_ip_and_path(self):
self.driver._get_ip_verify_on_cluster = mock.Mock( self.driver._get_ip_verify_on_cluster = mock.Mock(
return_value=fake.SHARE_IP) return_value=(fake.SHARE_IP, fake.VSERVER_NAME))
mock_extract_host = self.mock_object(volume_utils, 'extract_host') mock_extract_host = self.mock_object(volume_utils, 'extract_host')
mock_extract_host.return_value = fake.NFS_SHARE mock_extract_host.return_value = fake.NFS_SHARE
dest_ip, dest_path = self.driver._get_destination_ip_and_path( dest_ip, dest_vserver, dest_path = (
fake.VOLUME) self.driver._get_destination_ip_and_path(fake.VOLUME))
self.assertEqual(fake.VSERVER_NAME, dest_vserver)
self.assertEqual(fake.SHARE_IP, dest_ip) self.assertEqual(fake.SHARE_IP, dest_ip)
assert_path = fake.EXPORT_PATH + '/' + fake.LUN_NAME assert_path = fake.EXPORT_PATH + '/' + fake.LUN_NAME
self.assertEqual(assert_path, dest_path) self.assertEqual(assert_path, dest_path)
@ -1309,7 +1324,8 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase):
self.driver._is_flexgroup.assert_called_once_with(host=volume['host']) self.driver._is_flexgroup.assert_called_once_with(host=volume['host'])
mock_clone_file.assert_called_once_with() mock_clone_file.assert_called_once_with()
def test_clone_image_copyoffload_from_img_service(self): @ddt.data(True, False)
def test_clone_image_from_img_service(self, use_tool):
drv = self.driver drv = self.driver
context = object() context = object()
volume = {'id': 'vol_id', 'name': 'name', volume = {'id': 'vol_id', 'name': 'name',
@ -1330,6 +1346,8 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase):
drv._copy_from_img_service = mock.Mock(return_value=True) drv._copy_from_img_service = mock.Mock(return_value=True)
drv._is_flexgroup = mock.Mock(return_value=False) drv._is_flexgroup = mock.Mock(return_value=False)
drv._is_flexgroup_clone_file_supported = mock.Mock(return_value=True) drv._is_flexgroup_clone_file_supported = mock.Mock(return_value=True)
if not use_tool:
drv.configuration.netapp_copyoffload_tool_path = None
retval = drv.clone_image( retval = drv.clone_image(
context, volume, image_location, image_meta, image_service) context, volume, image_location, image_meta, image_service)
@ -1338,7 +1356,8 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase):
{'provider_location': '192.128.1.1:/mnt_point', {'provider_location': '192.128.1.1:/mnt_point',
'bootable': True}, True)) 'bootable': True}, True))
drv._copy_from_img_service.assert_called_once_with( drv._copy_from_img_service.assert_called_once_with(
context, volume, image_service, image_id) context, volume, image_service, image_id,
use_copyoffload_tool=use_tool)
def test_clone_image_copyoffload_failure(self): def test_clone_image_copyoffload_failure(self):
mock_log = self.mock_object(nfs_cmode, 'LOG') mock_log = self.mock_object(nfs_cmode, 'LOG')
@ -1365,51 +1384,57 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase):
self.assertEqual(retval, ({'bootable': False, self.assertEqual(retval, ({'bootable': False,
'provider_location': None}, False)) 'provider_location': None}, False))
drv._copy_from_img_service.assert_called_once_with( drv._copy_from_img_service.assert_called_once_with(
context, volume, image_service, image_id) context, volume, image_service, image_id,
use_copyoffload_tool=True)
mock_log.info.assert_not_called() mock_log.info.assert_not_called()
def test_copy_from_remote_cache(self): @ddt.data(True, False)
def test_copy_from_remote_cache(self, use_tool):
source_ip = '192.0.1.1' source_ip = '192.0.1.1'
source_path = '/openstack/img-cache-imgid' source_path = '/openstack/img-cache-imgid'
source_vserver = 'fake_vserver'
source_share = 'vol_fake'
cache_copy = ('192.0.1.1:/openstack', fake.IMAGE_FILE_ID) cache_copy = ('192.0.1.1:/openstack', fake.IMAGE_FILE_ID)
dest_vserver = 'fake_dest_vserver'
dest_path = fake.EXPORT_PATH + '/' + fake.VOLUME['name'] dest_path = fake.EXPORT_PATH + '/' + fake.VOLUME['name']
self.driver._execute = mock.Mock() self.driver._execute = mock.Mock()
self.driver._copy_file = mock.Mock()
self.driver._get_source_ip_and_path = mock.Mock( self.driver._get_source_ip_and_path = mock.Mock(
return_value=(source_ip, source_path)) return_value=(
source_ip, source_vserver, source_share, source_path))
self.driver._get_destination_ip_and_path = mock.Mock( self.driver._get_destination_ip_and_path = mock.Mock(
return_value=(fake.SHARE_IP, dest_path)) return_value=(fake.SHARE_IP, dest_vserver, dest_path))
self.driver._register_image_in_cache = mock.Mock() self.driver._register_image_in_cache = mock.Mock()
ctxt = mock.Mock()
vol_fields = {'id': fake.VOLUME_ID, 'name': fake.VOLUME_NAME}
fake_vol = fake_volume.fake_volume_obj(ctxt, **vol_fields)
self.driver._copy_from_remote_cache( self.driver._copy_from_remote_cache(
fake.VOLUME, fake.IMAGE_FILE_ID, cache_copy) fake_vol, fake.IMAGE_FILE_ID, cache_copy,
use_copyoffload_tool=use_tool)
if use_tool:
self.driver._execute.assert_called_once_with( self.driver._execute.assert_called_once_with(
'copyoffload_tool_path', source_ip, fake.SHARE_IP, 'copyoffload_tool_path', source_ip, fake.SHARE_IP,
source_path, dest_path, run_as_root=False, check_exit_code=0) source_path, dest_path, run_as_root=False, check_exit_code=0)
self.driver._copy_file.assert_not_called()
else:
dest_share_path = dest_path.rsplit("/", 1)[0]
self.driver._copy_file.assert_called_once_with(
fake.IMAGE_FILE_ID, fake.IMAGE_FILE_ID, source_share,
source_vserver, dest_share_path, dest_vserver,
dest_backend_name=self.driver.backend_name,
dest_file_name=fake_vol.name)
self.driver._execute.assert_not_called()
self.driver._get_source_ip_and_path.assert_called_once_with( self.driver._get_source_ip_and_path.assert_called_once_with(
cache_copy[0], fake.IMAGE_FILE_ID) cache_copy[0], fake.IMAGE_FILE_ID)
self.driver._get_destination_ip_and_path.assert_called_once_with( self.driver._get_destination_ip_and_path.assert_called_once_with(
fake.VOLUME) fake_vol)
self.driver._register_image_in_cache.assert_called_once_with( self.driver._register_image_in_cache.assert_called_once_with(
fake.VOLUME, fake.IMAGE_FILE_ID) fake_vol, fake.IMAGE_FILE_ID)
def test_copy_from_cache_workflow_remote_location(self): @ddt.data(True, False)
cache_result = [('ip1:/openstack', fake.IMAGE_FILE_ID), def test_copy_from_cache_workflow_remote_location(self, use_tool):
('ip2:/openstack', fake.IMAGE_FILE_ID),
('ip3:/openstack', fake.IMAGE_FILE_ID)]
self.driver._find_image_location = mock.Mock(return_value=[
cache_result[0], False])
self.driver._copy_from_remote_cache = mock.Mock()
self.driver._post_clone_image = mock.Mock()
copied = self.driver._copy_from_cache(
fake.VOLUME, fake.IMAGE_FILE_ID, cache_result)
self.assertTrue(copied)
self.driver._copy_from_remote_cache.assert_called_once_with(
fake.VOLUME, fake.IMAGE_FILE_ID, cache_result[0])
def test_copy_from_cache_workflow_remote_location_no_copyoffload(self):
cache_result = [('ip1:/openstack', fake.IMAGE_FILE_ID), cache_result = [('ip1:/openstack', fake.IMAGE_FILE_ID),
('ip2:/openstack', fake.IMAGE_FILE_ID), ('ip2:/openstack', fake.IMAGE_FILE_ID),
('ip3:/openstack', fake.IMAGE_FILE_ID)] ('ip3:/openstack', fake.IMAGE_FILE_ID)]
@ -1417,13 +1442,20 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase):
cache_result[0], False]) cache_result[0], False])
self.driver._copy_from_remote_cache = mock.Mock() self.driver._copy_from_remote_cache = mock.Mock()
self.driver._post_clone_image = mock.Mock() self.driver._post_clone_image = mock.Mock()
if not use_tool:
self.driver.configuration.netapp_copyoffload_tool_path = None self.driver.configuration.netapp_copyoffload_tool_path = None
copied = self.driver._copy_from_cache( copied = self.driver._copy_from_cache(
fake.VOLUME, fake.IMAGE_FILE_ID, cache_result) fake.VOLUME, fake.IMAGE_FILE_ID, cache_result)
self.assertFalse(copied) self.assertTrue(copied)
self.driver._copy_from_remote_cache.assert_not_called() if use_tool:
self.driver._copy_from_remote_cache.assert_called_once_with(
fake.VOLUME, fake.IMAGE_FILE_ID, cache_result[0])
else:
self.driver._copy_from_remote_cache.assert_called_once_with(
fake.VOLUME, fake.IMAGE_FILE_ID, cache_result[0],
use_copyoffload_tool=False)
def test_copy_from_cache_workflow_local_location(self): def test_copy_from_cache_workflow_local_location(self):
local_share = '/share' local_share = '/share'
@ -1456,20 +1488,28 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase):
self.assertFalse(copied) self.assertFalse(copied)
def test_copy_from_cache_workflow_exception(self): @ddt.data(True, False)
def test_copy_from_cache_workflow_exception(self, use_tool):
cache_result = [('ip1:/openstack', fake.IMAGE_FILE_ID)] cache_result = [('ip1:/openstack', fake.IMAGE_FILE_ID)]
self.driver._find_image_location = mock.Mock(return_value=[ self.driver._find_image_location = mock.Mock(return_value=[
cache_result[0], False]) cache_result[0], False])
self.driver._copy_from_remote_cache = mock.Mock( self.driver._copy_from_remote_cache = mock.Mock(
side_effect=Exception) side_effect=Exception)
self.driver._post_clone_image = mock.Mock() self.driver._post_clone_image = mock.Mock()
if not use_tool:
self.driver.configuration.netapp_copyoffload_tool_path = None
copied = self.driver._copy_from_cache( copied = self.driver._copy_from_cache(
fake.VOLUME, fake.IMAGE_FILE_ID, cache_result) fake.VOLUME, fake.IMAGE_FILE_ID, cache_result)
self.assertFalse(copied) self.assertFalse(copied)
if use_tool:
self.driver._copy_from_remote_cache.assert_called_once_with( self.driver._copy_from_remote_cache.assert_called_once_with(
fake.VOLUME, fake.IMAGE_FILE_ID, cache_result[0]) fake.VOLUME, fake.IMAGE_FILE_ID, cache_result[0])
else:
self.driver._copy_from_remote_cache.assert_called_once_with(
fake.VOLUME, fake.IMAGE_FILE_ID, cache_result[0],
use_copyoffload_tool=False)
self.assertFalse(self.driver._post_clone_image.called) self.assertFalse(self.driver._post_clone_image.called)
@ddt.data({'secondary_id': 'dev0', 'configured_targets': ['dev1']}, @ddt.data({'secondary_id': 'dev0', 'configured_targets': ['dev1']},
@ -1902,21 +1942,15 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase):
return_value=fake_job_status) return_value=fake_job_status)
mock_cancel_file_copy = self.mock_object( mock_cancel_file_copy = self.mock_object(
self.driver, '_cancel_file_copy') self.driver, '_cancel_file_copy')
ctxt = mock.Mock()
vol_fields = {
'id': fake.VOLUME_ID,
'name': fake.VOLUME_NAME,
'status': fields.VolumeStatus.AVAILABLE
}
fake_vol = fake_volume.fake_volume_obj(ctxt, **vol_fields)
result = self.driver._copy_file( result = self.driver._copy_file(
fake_vol, fake.POOL_NAME, fake.VSERVER_NAME, fake.DEST_POOL_NAME, fake.VOLUME_NAME, fake.VOLUME_ID, fake.POOL_NAME,
fake.DEST_VSERVER_NAME, dest_file_name=fake.VOLUME_NAME, fake.VSERVER_NAME, fake.DEST_POOL_NAME, fake.DEST_VSERVER_NAME,
dest_file_name=fake.VOLUME_NAME,
dest_backend_name=fake.DEST_BACKEND_NAME, cancel_on_error=True) dest_backend_name=fake.DEST_BACKEND_NAME, cancel_on_error=True)
mock_start_file_copy.assert_called_with( mock_start_file_copy.assert_called_with(
fake_vol.name, fake.DEST_POOL_NAME, fake.VOLUME_NAME, fake.DEST_POOL_NAME,
src_ontap_volume=fake.POOL_NAME, src_ontap_volume=fake.POOL_NAME,
dest_file_name=fake.VOLUME_NAME) dest_file_name=fake.VOLUME_NAME)
mock_get_file_copy_status.assert_called_with(fake.JOB_UUID) mock_get_file_copy_status.assert_called_with(fake.JOB_UUID)
@ -1943,29 +1977,23 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase):
return_value=fake_job_status) return_value=fake_job_status)
mock_cancel_file_copy = self.mock_object( mock_cancel_file_copy = self.mock_object(
self.driver, '_cancel_file_copy') self.driver, '_cancel_file_copy')
ctxt = mock.Mock()
vol_fields = {
'id': fake.VOLUME_ID,
'name': fake.VOLUME_NAME,
'status': fields.VolumeStatus.AVAILABLE
}
fake_vol = fake_volume.fake_volume_obj(ctxt, **vol_fields)
self.assertRaises(copy_exception, self.assertRaises(copy_exception,
self.driver._copy_file, self.driver._copy_file,
fake_vol, fake.POOL_NAME, fake.VSERVER_NAME, fake.VOLUME_NAME, fake.VOLUME_ID, fake.POOL_NAME,
fake.DEST_POOL_NAME, fake.DEST_VSERVER_NAME, fake.VSERVER_NAME, fake.DEST_POOL_NAME,
fake.DEST_VSERVER_NAME,
dest_file_name=fake.VOLUME_NAME, dest_file_name=fake.VOLUME_NAME,
dest_backend_name=fake.DEST_BACKEND_NAME, dest_backend_name=fake.DEST_BACKEND_NAME,
cancel_on_error=True) cancel_on_error=True)
mock_start_file_copy.assert_called_with( mock_start_file_copy.assert_called_with(
fake_vol.name, fake.DEST_POOL_NAME, fake.VOLUME_NAME, fake.DEST_POOL_NAME,
src_ontap_volume=fake.POOL_NAME, src_ontap_volume=fake.POOL_NAME,
dest_file_name=fake.VOLUME_NAME) dest_file_name=fake.VOLUME_NAME)
mock_get_file_copy_status.assert_called_with(fake.JOB_UUID) mock_get_file_copy_status.assert_called_with(fake.JOB_UUID)
mock_cancel_file_copy.assert_called_once_with( mock_cancel_file_copy.assert_called_once_with(
fake.JOB_UUID, fake_vol, fake.DEST_POOL_NAME, fake.JOB_UUID, fake.VOLUME_NAME, fake.DEST_POOL_NAME,
dest_backend_name=fake.DEST_BACKEND_NAME) dest_backend_name=fake.DEST_BACKEND_NAME)
def test_migrate_volume_to_vserver(self): def test_migrate_volume_to_vserver(self):
@ -1984,9 +2012,9 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase):
fake.DEST_VSERVER_NAME, fake.DEST_BACKEND_NAME) fake.DEST_VSERVER_NAME, fake.DEST_BACKEND_NAME)
mock_copy_file.assert_called_once_with( mock_copy_file.assert_called_once_with(
fake_vol, fake.EXPORT_PATH[1:], fake.VSERVER_NAME, fake_vol.name, fake_vol.id, fake.EXPORT_PATH[1:],
fake.DEST_EXPORT_PATH[1:], fake.DEST_VSERVER_NAME, fake.VSERVER_NAME, fake.DEST_EXPORT_PATH[1:],
dest_backend_name=fake.DEST_BACKEND_NAME, fake.DEST_VSERVER_NAME, dest_backend_name=fake.DEST_BACKEND_NAME,
cancel_on_error=True) cancel_on_error=True)
mock_create_vserver_peer.assert_called_once_with( mock_create_vserver_peer.assert_called_once_with(
fake.VSERVER_NAME, fake.BACKEND_NAME, fake.DEST_VSERVER_NAME, fake.VSERVER_NAME, fake.BACKEND_NAME, fake.DEST_VSERVER_NAME,
@ -2051,9 +2079,9 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase):
fake.VSERVER_NAME, fake.BACKEND_NAME, fake.DEST_VSERVER_NAME, fake.VSERVER_NAME, fake.BACKEND_NAME, fake.DEST_VSERVER_NAME,
['file_copy']) ['file_copy'])
mock_copy_file.assert_called_once_with( mock_copy_file.assert_called_once_with(
fake_vol, fake.EXPORT_PATH[1:], fake.VSERVER_NAME, fake_vol.name, fake_vol.id, fake.EXPORT_PATH[1:],
fake.DEST_EXPORT_PATH[1:], fake.DEST_VSERVER_NAME, fake.VSERVER_NAME, fake.DEST_EXPORT_PATH[1:],
dest_backend_name=fake.DEST_BACKEND_NAME, fake.DEST_VSERVER_NAME, dest_backend_name=fake.DEST_BACKEND_NAME,
cancel_on_error=True) cancel_on_error=True)
mock_finish_volume_migration.assert_not_called() mock_finish_volume_migration.assert_not_called()
@ -2084,9 +2112,9 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase):
fake.VSERVER_NAME, fake.BACKEND_NAME, fake.DEST_VSERVER_NAME, fake.VSERVER_NAME, fake.BACKEND_NAME, fake.DEST_VSERVER_NAME,
['file_copy']) ['file_copy'])
mock_copy_file.assert_called_once_with( mock_copy_file.assert_called_once_with(
fake_vol, fake.EXPORT_PATH[1:], fake.VSERVER_NAME, fake_vol.name, fake_vol.id, fake.EXPORT_PATH[1:],
fake.DEST_EXPORT_PATH[1:], fake.DEST_VSERVER_NAME, fake.VSERVER_NAME, fake.DEST_EXPORT_PATH[1:],
dest_backend_name=fake.DEST_BACKEND_NAME, fake.DEST_VSERVER_NAME, dest_backend_name=fake.DEST_BACKEND_NAME,
cancel_on_error=True) cancel_on_error=True)
mock_finish_volume_migration.assert_not_called() mock_finish_volume_migration.assert_not_called()
@ -2105,8 +2133,8 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase):
fake.DEST_BACKEND_NAME) fake.DEST_BACKEND_NAME)
mock_copy_file.assert_called_once_with( mock_copy_file.assert_called_once_with(
fake_vol, fake.EXPORT_PATH[1:], fake.VSERVER_NAME, fake_vol.name, fake_vol.id, fake.EXPORT_PATH[1:],
fake.DEST_EXPORT_PATH[1:], fake.VSERVER_NAME, fake.VSERVER_NAME, fake.DEST_EXPORT_PATH[1:], fake.VSERVER_NAME,
dest_backend_name=fake.DEST_BACKEND_NAME, dest_backend_name=fake.DEST_BACKEND_NAME,
cancel_on_error=True) cancel_on_error=True)
mock_finish_volume_migration.assert_called_once_with( mock_finish_volume_migration.assert_called_once_with(
@ -2133,8 +2161,8 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase):
fake.DEST_BACKEND_NAME) fake.DEST_BACKEND_NAME)
mock_copy_file.assert_called_once_with( mock_copy_file.assert_called_once_with(
fake_vol, fake.EXPORT_PATH[1:], fake.VSERVER_NAME, fake_vol.name, fake_vol.id, fake.EXPORT_PATH[1:],
fake.DEST_EXPORT_PATH[1:], fake.VSERVER_NAME, fake.VSERVER_NAME, fake.DEST_EXPORT_PATH[1:], fake.VSERVER_NAME,
dest_backend_name=fake.DEST_BACKEND_NAME, dest_backend_name=fake.DEST_BACKEND_NAME,
cancel_on_error=True) cancel_on_error=True)
mock_finish_volume_migration.assert_not_called() mock_finish_volume_migration.assert_not_called()
@ -2159,8 +2187,8 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase):
fake.DEST_BACKEND_NAME) fake.DEST_BACKEND_NAME)
mock_copy_file.assert_called_once_with( mock_copy_file.assert_called_once_with(
fake_vol, fake.EXPORT_PATH[1:], fake.VSERVER_NAME, fake_vol.name, fake_vol.id, fake.EXPORT_PATH[1:],
fake.DEST_EXPORT_PATH[1:], fake.VSERVER_NAME, fake.VSERVER_NAME, fake.DEST_EXPORT_PATH[1:], fake.VSERVER_NAME,
dest_backend_name=fake.DEST_BACKEND_NAME, dest_backend_name=fake.DEST_BACKEND_NAME,
cancel_on_error=True) cancel_on_error=True)
mock_finish_volume_migration.assert_not_called() mock_finish_volume_migration.assert_not_called()

View File

@ -648,7 +648,7 @@ class NetAppNfsDriver(driver.ManageableVD,
raise NotImplementedError() raise NotImplementedError()
def _copy_from_img_service(self, context, volume, image_service, def _copy_from_img_service(self, context, volume, image_service,
image_id): image_id, use_copyoffload_tool=False):
raise NotImplementedError() raise NotImplementedError()
def clone_image(self, context, volume, def clone_image(self, context, volume,
@ -683,10 +683,15 @@ class NetAppNfsDriver(driver.ManageableVD,
cloned = self._direct_nfs_clone(volume, image_location, cloned = self._direct_nfs_clone(volume, image_location,
image_id) image_id)
# Try to use the copy offload tool # Try to use the deprecated copy offload tool or file copy.
if not cloned and col_path and major == 1 and minor >= 20: if not cloned:
cloned = self._copy_from_img_service(context, volume, # We will use copy offload tool if the copy offload tool
image_service, image_id) # path exists and the version is greater than or equal to
# 1.20
use_tool = bool(col_path) and (major == 1 and minor >= 20)
cloned = self._copy_from_img_service(
context, volume, image_service, image_id,
use_copyoffload_tool=use_tool)
if cloned: if cloned:
self._do_qos_for_volume(volume, extra_specs) self._do_qos_for_volume(volume, extra_specs)

View File

@ -620,7 +620,7 @@ class NetAppCmodeNfsDriver(nfs_base.NetAppNfsDriver,
if not vserver: if not vserver:
raise exception.NotFound(_("Unable to locate an SVM that is " raise exception.NotFound(_("Unable to locate an SVM that is "
"managing the IP address '%s'") % ip) "managing the IP address '%s'") % ip)
return ip return ip, vserver
def _copy_from_cache(self, volume, image_id, cache_result): def _copy_from_cache(self, volume, image_id, cache_result):
"""Try copying image file_name from cached file_name.""" """Try copying image file_name from cached file_name."""
@ -642,6 +642,11 @@ class NetAppCmodeNfsDriver(nfs_base.NetAppNfsDriver,
LOG.debug("Trying copy from cache using copy offload.") LOG.debug("Trying copy from cache using copy offload.")
self._copy_from_remote_cache(volume, image_id, cache_copy) self._copy_from_remote_cache(volume, image_id, cache_copy)
copied = True copied = True
elif cache_copy:
LOG.debug("Trying copy from cache using file copy.")
self._copy_from_remote_cache(volume, image_id, cache_copy,
use_copyoffload_tool=False)
copied = True
except Exception: except Exception:
LOG.exception('Error in workflow copy from cache.') LOG.exception('Error in workflow copy from cache.')
return copied return copied
@ -670,41 +675,55 @@ class NetAppCmodeNfsDriver(nfs_base.NetAppNfsDriver,
cache_copy = res cache_copy = res
return cache_copy, found_local_copy return cache_copy, found_local_copy
def _copy_from_remote_cache(self, volume, image_id, cache_copy): def _copy_from_remote_cache(self, volume, image_id, cache_copy,
use_copyoffload_tool=True):
"""Copies the remote cached image to the provided volume. """Copies the remote cached image to the provided volume.
Executes the copy offload binary which copies the cached image to Executes either the copy offload binary or the file copy operation,
the destination path of the provided volume. Also registers the new copying the cached image to the destination path of the provided
copy of the image as a cached image. volume. Also registers the new copy of the image as a cached image.
""" """
(nfs_share, file_name) = cache_copy (nfs_share, file_name) = cache_copy
col_path = self.configuration.netapp_copyoffload_tool_path (src_ip, src_vserver, src_share_path, src_path) = (
src_ip, src_path = self._get_source_ip_and_path(nfs_share, file_name) self._get_source_ip_and_path(nfs_share, file_name))
dest_ip, dest_path = self._get_destination_ip_and_path(volume) (dest_ip, dest_vserver, dest_path) = (
self._get_destination_ip_and_path(volume))
# NOTE(felipe_rodrigues): the copy offload tool code will be removed in
# the Antelope release.
col_path = self.configuration.netapp_copyoffload_tool_path
if use_copyoffload_tool and col_path:
# Always run copy offload as regular user, it's sufficient # Always run copy offload as regular user, it's sufficient
# and rootwrap doesn't allow copy offload to run as root anyways. # and rootwrap doesn't allow copy offload to run as root anyways.
self._execute(col_path, src_ip, dest_ip, src_path, dest_path, self._execute(col_path, src_ip, dest_ip, src_path, dest_path,
run_as_root=False, check_exit_code=0) run_as_root=False, check_exit_code=0)
LOG.debug("Copied image from cache to volume %s using "
"copy offload.", volume['id'])
else:
dest_share_path = dest_path.rsplit("/", 1)[0]
self._copy_file(file_name, file_name, src_share_path, src_vserver,
dest_share_path, dest_vserver,
dest_backend_name=self.backend_name,
dest_file_name=volume.name)
LOG.debug("Copied image from cache to volume %s using "
"file copy operation.", volume['id'])
self._register_image_in_cache(volume, image_id) self._register_image_in_cache(volume, image_id)
LOG.debug("Copied image from cache to volume %s using copy offload.",
volume['id'])
def _get_source_ip_and_path(self, nfs_share, file_name): def _get_source_ip_and_path(self, nfs_share, file_name):
host, share_path = na_utils.get_export_host_junction_path(nfs_share) host, share_path = na_utils.get_export_host_junction_path(nfs_share)
src_ip = self._get_ip_verify_on_cluster(host) (src_ip, src_vserver) = self._get_ip_verify_on_cluster(host)
src_path = os.path.join(share_path, file_name) src_path = os.path.join(share_path, file_name)
return src_ip, src_path return src_ip, src_vserver, share_path, src_path
def _get_destination_ip_and_path(self, volume): def _get_destination_ip_and_path(self, volume):
share = volume_utils.extract_host(volume['host'], level='pool') share = volume_utils.extract_host(volume['host'], level='pool')
share_ip, share_path = na_utils.get_export_host_junction_path(share) share_ip, share_path = na_utils.get_export_host_junction_path(share)
dest_ip = self._get_ip_verify_on_cluster(share_ip) (dest_ip, vserver) = self._get_ip_verify_on_cluster(share_ip)
dest_path = os.path.join(share_path, volume['name']) dest_path = os.path.join(share_path, volume['name'])
return dest_ip, dest_path return dest_ip, vserver, dest_path
def _clone_file_dst_exists(self, share, src_name, dst_name, def _clone_file_dst_exists(self, share, src_name, dst_name,
dest_exists=False): dest_exists=False):
@ -714,38 +733,39 @@ class NetAppCmodeNfsDriver(nfs_base.NetAppNfsDriver,
dest_exists=dest_exists) dest_exists=dest_exists)
def _copy_from_img_service(self, context, volume, image_service, def _copy_from_img_service(self, context, volume, image_service,
image_id): image_id, use_copyoffload_tool=True):
"""Copies from the image service using copy offload.""" """Copies from the image service using copy offload or file copy."""
LOG.debug("Trying copy from image service using copy offload.")
image_loc = image_service.get_location(context, image_id) image_loc = image_service.get_location(context, image_id)
locations = self._construct_image_nfs_url(image_loc) locations = self._construct_image_nfs_url(image_loc)
src_ip = None src_ip = None
src_vserver = None
src_volume = None
selected_loc = None selected_loc = None
cloned = False cloned = False
# this will match the first location that has a valid IP on cluster # this will match the first location that has a valid IP on cluster
for location in locations: for location in locations:
conn, dr = self._check_get_nfs_path_segs(location) conn, src_volume = self._check_get_nfs_path_segs(location)
if conn: if conn:
try: try:
src_ip = self._get_ip_verify_on_cluster(conn.split(':')[0]) (src_ip, src_vserver) = (
self._get_ip_verify_on_cluster(conn.split(':')[0]))
selected_loc = location selected_loc = location
break break
except exception.NotFound: except exception.NotFound:
pass pass
if src_ip is None: if src_ip is None or src_vserver is None:
raise exception.NotFound(_("Source host details not found.")) raise exception.NotFound(_("Source host details not found."))
(__, ___, img_file) = selected_loc.rpartition('/') (__, ___, img_file) = selected_loc.rpartition('/')
src_path = os.path.join(dr, img_file)
dst_ip, vol_path = self._get_destination_ip_and_path(volume) (dst_ip, dest_vserver, vol_path) = (
share_path = vol_path.rsplit("/", 1)[0] self._get_destination_ip_and_path(volume))
dst_share = dst_ip + ':' + share_path dest_share_path = vol_path.rsplit("/", 1)[0]
dst_share = dst_ip + ':' + dest_share_path
# tmp file is required to deal with img formats # tmp file is required to deal with img formats
tmp_img_file = six.text_type(uuid.uuid4()) tmp_img_file = six.text_type(uuid.uuid4())
col_path = self.configuration.netapp_copyoffload_tool_path
img_info = image_service.show(context, image_id) img_info = image_service.show(context, image_id)
self._check_share_can_hold_size(dst_share, img_info['size']) self._check_share_can_hold_size(dst_share, img_info['size'])
run_as_root = self._execute_as_root run_as_root = self._execute_as_root
@ -754,14 +774,27 @@ class NetAppCmodeNfsDriver(nfs_base.NetAppNfsDriver,
dst_img_local = os.path.join(dst_dir, tmp_img_file) dst_img_local = os.path.join(dst_dir, tmp_img_file)
try: try:
dst_img_serv_path = os.path.join( # NOTE(felipe_rodrigues): the copy offload tool code will be
share_path, tmp_img_file) # removed in the AA release.
col_path = self.configuration.netapp_copyoffload_tool_path
if col_path and use_copyoffload_tool:
LOG.debug("Trying copy from image service using copy offload.")
dst_img_serv_path = os.path.join(dest_share_path, tmp_img_file)
src_path = os.path.join(src_volume, img_file)
# Always run copy offload as regular user, it's sufficient # Always run copy offload as regular user, it's sufficient
# and rootwrap doesn't allow copy offload to run as root # and rootwrap doesn't allow copy offload to run as root
# anyways. # anyways.
self._execute(col_path, src_ip, dst_ip, src_path, self._execute(col_path, src_ip, dst_ip, src_path,
dst_img_serv_path, run_as_root=False, dst_img_serv_path, run_as_root=False,
check_exit_code=0) check_exit_code=0)
else:
LOG.debug("Trying copy from image service using file copy.")
src_volume = ''.join(src_volume.split("/", 1))
dest_share_path = ''.join(dest_share_path.split("/", 1))
self._copy_file(img_file, img_file, src_volume, src_vserver,
dest_share_path, dest_vserver,
dest_backend_name=self.backend_name,
dest_file_name=tmp_img_file)
self._discover_file_till_timeout(dst_img_local, timeout=120) self._discover_file_till_timeout(dst_img_local, timeout=120)
LOG.debug('Copied image %(img)s to tmp file %(tmp)s.', LOG.debug('Copied image %(img)s to tmp file %(tmp)s.',
@ -1072,7 +1105,7 @@ class NetAppCmodeNfsDriver(nfs_base.NetAppNfsDriver,
"""Check whether storage can perform clone file for FlexGroup""" """Check whether storage can perform clone file for FlexGroup"""
return self.zapi_client.features.FLEXGROUP_CLONE_FILE return self.zapi_client.features.FLEXGROUP_CLONE_FILE
def _cancel_file_copy(self, job_uuid, volume, dest_pool, def _cancel_file_copy(self, job_uuid, file_name, dest_pool,
dest_backend_name=None): dest_backend_name=None):
"""Cancel an on-going file copy operation.""" """Cancel an on-going file copy operation."""
try: try:
@ -1082,7 +1115,7 @@ class NetAppCmodeNfsDriver(nfs_base.NetAppNfsDriver,
self.zapi_client.destroy_file_copy(job_uuid) self.zapi_client.destroy_file_copy(job_uuid)
except na_utils.NetAppDriverException: except na_utils.NetAppDriverException:
dest_client = dot_utils.get_client_for_backend(dest_backend_name) dest_client = dot_utils.get_client_for_backend(dest_backend_name)
file_path = '%s/%s' % (dest_pool, volume.name) file_path = '%s/%s' % (dest_pool, file_name)
try: try:
dest_client.delete_file(file_path) dest_client.delete_file(file_path)
except Exception: except Exception:
@ -1091,17 +1124,17 @@ class NetAppCmodeNfsDriver(nfs_base.NetAppNfsDriver,
'pool %s and delete it manually to avoid unused ' 'pool %s and delete it manually to avoid unused '
'resources.', file_path, dest_pool) 'resources.', file_path, dest_pool)
def _copy_file(self, volume, src_ontap_volume, src_vserver, def _copy_file(self, file_name, volume_id, src_ontap_volume, src_vserver,
dest_ontap_volume, dest_vserver, dest_file_name=None, dest_ontap_volume, dest_vserver, dest_file_name=None,
dest_backend_name=None, cancel_on_error=False): dest_backend_name=None, cancel_on_error=False):
"""Copies file from an ONTAP volume to another.""" """Copies file from an ONTAP volume to another."""
job_uuid = self.zapi_client.start_file_copy( job_uuid = self.zapi_client.start_file_copy(
volume.name, dest_ontap_volume, src_ontap_volume=src_ontap_volume, file_name, dest_ontap_volume, src_ontap_volume=src_ontap_volume,
dest_file_name=dest_file_name) dest_file_name=dest_file_name)
LOG.debug('Start copying file %(vol)s from ' LOG.debug('Start copying file %(file)s from '
'%(src_vserver)s:%(src_ontap_vol)s to ' '%(src_vserver)s:%(src_ontap_vol)s to '
'%(dest_vserver)s:%(dest_ontap_vol)s. Job UUID is %(job)s.', '%(dest_vserver)s:%(dest_ontap_vol)s. Job UUID is %(job)s.',
{'vol': volume.name, 'src_vserver': src_vserver, {'file': file_name, 'src_vserver': src_vserver,
'src_ontap_vol': src_ontap_volume, 'src_ontap_vol': src_ontap_volume,
'dest_vserver': dest_vserver, 'dest_vserver': dest_vserver,
'dest_ontap_vol': dest_ontap_volume, 'dest_ontap_vol': dest_ontap_volume,
@ -1116,11 +1149,11 @@ class NetAppCmodeNfsDriver(nfs_base.NetAppNfsDriver,
"corresponding Job UUID % doesn't " "corresponding Job UUID % doesn't "
"exist.")) "exist."))
raise na_utils.NetAppDriverException( raise na_utils.NetAppDriverException(
status_error_msg % (volume.id, job_uuid)) status_error_msg % (file_name, job_uuid))
elif copy_status['job-status'] == 'destroyed': elif copy_status['job-status'] == 'destroyed':
status_error_msg = (_('Error copying file %s. %s.')) status_error_msg = (_('Error copying file %s. %s.'))
raise na_utils.NetAppDriverException( raise na_utils.NetAppDriverException(
status_error_msg % (volume.id, status_error_msg % (file_name,
copy_status['last-failure-reason'])) copy_status['last-failure-reason']))
elif copy_status['job-status'] == 'complete': elif copy_status['job-status'] == 'complete':
raise loopingcall.LoopingCallDone() raise loopingcall.LoopingCallDone()
@ -1137,7 +1170,7 @@ class NetAppCmodeNfsDriver(nfs_base.NetAppNfsDriver,
if cancel_on_error: if cancel_on_error:
try: try:
self._cancel_file_copy( self._cancel_file_copy(
job_uuid, volume, dest_ontap_volume, job_uuid, file_name, dest_ontap_volume,
dest_backend_name=dest_backend_name) dest_backend_name=dest_backend_name)
except na_utils.NetAppDriverException as ex: except na_utils.NetAppDriverException as ex:
LOG.error("Failed to cancel file copy operation. %s", LOG.error("Failed to cancel file copy operation. %s",
@ -1146,7 +1179,7 @@ class NetAppCmodeNfsDriver(nfs_base.NetAppNfsDriver,
ctxt.reraise = False ctxt.reraise = False
msg = (_('Timeout waiting volume %s to complete ' msg = (_('Timeout waiting volume %s to complete '
'migration.')) 'migration.'))
raise na_utils.NetAppDriverTimeout(msg % volume.id) raise na_utils.NetAppDriverTimeout(msg % volume_id)
def _finish_volume_migration(self, src_volume, dest_pool): def _finish_volume_migration(self, src_volume, dest_pool):
"""Finish volume migration to another ONTAP volume.""" """Finish volume migration to another ONTAP volume."""
@ -1171,8 +1204,8 @@ class NetAppCmodeNfsDriver(nfs_base.NetAppNfsDriver,
[vserver_peer_application]) [vserver_peer_application])
src_ontap_volume_name = src_pool.split(':/')[1] src_ontap_volume_name = src_pool.split(':/')[1]
dest_ontap_volume_name = dest_pool.split(':/')[1] dest_ontap_volume_name = dest_pool.split(':/')[1]
self._copy_file(volume, src_ontap_volume_name, src_vserver, self._copy_file(volume.name, volume.id, src_ontap_volume_name,
dest_ontap_volume_name, dest_vserver, src_vserver, dest_ontap_volume_name, dest_vserver,
dest_backend_name=dest_backend_name, dest_backend_name=dest_backend_name,
cancel_on_error=True) cancel_on_error=True)
updates = self._finish_volume_migration(volume, dest_pool) updates = self._finish_volume_migration(volume, dest_pool)
@ -1193,7 +1226,7 @@ class NetAppCmodeNfsDriver(nfs_base.NetAppNfsDriver,
'vserver': vserver}) 'vserver': vserver})
src_ontap_volume_name = src_pool.split(':/')[1] src_ontap_volume_name = src_pool.split(':/')[1]
dest_ontap_volume_name = dest_pool.split(':/')[1] dest_ontap_volume_name = dest_pool.split(':/')[1]
self._copy_file(volume, src_ontap_volume_name, vserver, self._copy_file(volume.name, volume.id, src_ontap_volume_name, vserver,
dest_ontap_volume_name, vserver, dest_ontap_volume_name, vserver,
dest_backend_name=dest_backend_name, dest_backend_name=dest_backend_name,
cancel_on_error=True) cancel_on_error=True)

View File

@ -0,0 +1,9 @@
---
features:
- |
NetApp NFS driver: add an alternative approach to perform the efficient clone
image when the Glance source store and Cinder destination pool are not in the
same FlexVol, but they are in the same Cluster. Previously, the driver required
the copy offload tool for doing it efficiently, which is no longer available.
Now, the operators can maintain their efficient clone image by relying on the
storage file copy operation.