openstack/manila
Code Issues Proposed changes

NetApp cDOT: Add support for QoS/throughput ceilings

Browse Source 
ONTAP supports assigning QoS policy groups to storage
objects and workloads. [1]

Expose this functionality through the ONTAP manila
drivers (DHSS=True/False, NFS, CIFS).

The drivers will set the capability "qos" to True if the
configured credentials have access to create qos policy
groups on the configured ONTAP backend. When 'qos'
extra-spec is set in share types, scoped extra-specs can
be used to specify QoS ceiling values in iops or bps.

The drivers support the following QoS specs:
'netapp:maxiops', 'netapp:maxiopspergib', 'netapp:maxbps',
'netapp:maxbpspergib'. Policies are created on-demand
and manipulated as and when shares are manipulated
through manila.

[1] http://docs.netapp.com/ontap-9/index.jsp?topic=%2Fcom.netapp.doc.pow-perf-mon%2FGUID-38357C43-FB36-419D-B31F-6FD75B47254D.html

Implements: blueprint netapp-cdot-qos
Change-Id: I6f82c012ea60cfb1e9f82a696e2346ee95c60df3
This commit is contained in:
Goutham Pacha Ravi 2017-06-24 16:56:03 -04:00
parent 96a037fb67
commit 10395c9aea
14 changed files with 1371 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
doc/source/devref/share_back_ends_feature_support_mapping.rst
View File

@ -220,7 +220,7 @@ More information: :ref:`capabilities_and_extra_specs`
+----------------------------------------+-----------+------------+--------+-------------+-------------------+--------------------+-----+----------------------------+--------------------+--------------------+
| Generic (Cinder as back-end) | J | K | \- | \- | \- | L | \- | J | \- | \- |
+----------------------------------------+-----------+------------+--------+-------------+-------------------+--------------------+-----+----------------------------+--------------------+--------------------+
| NetApp Clustered Data ONTAP | J | K | M | M | M | L | \- | J | O | \- |
| NetApp Clustered Data ONTAP | J | K | M | M | M | L | P | J | O | \- |
+----------------------------------------+-----------+------------+--------+-------------+-------------------+--------------------+-----+----------------------------+--------------------+--------------------+
| EMC VMAX | O | \- | \- | \- | \- | O | \- | O | \- | \- |
+----------------------------------------+-----------+------------+--------+-------------+-------------------+--------------------+-----+----------------------------+--------------------+--------------------+

195
manila/share/drivers/netapp/dataontap/client/client_cmode.py
View File

@ -1422,7 +1422,8 @@ class NetAppCmodeClient(client_base.NetAppBaseClient):
thin_provisioned=False, snapshot_policy=None,
language=None, dedup_enabled=False,
compression_enabled=False, max_files=None,
snapshot_reserve=None, volume_type='rw', **options):
snapshot_reserve=None, volume_type='rw',
qos_policy_group=None, **options):
"""Creates a volume."""
api_args = {
@ -1442,6 +1443,8 @@ class NetAppCmodeClient(client_base.NetAppBaseClient):
if snapshot_reserve is not None:
api_args['percentage-snapshot-reserve'] = six.text_type(
snapshot_reserve)
if qos_policy_group is not None:
api_args['qos-policy-group-name'] = qos_policy_group
self.send_request('volume-create', api_args)
# cDOT compression requires that deduplication be enabled.
@ -1573,11 +1576,12 @@ class NetAppCmodeClient(client_base.NetAppBaseClient):
self.send_request('volume-rename', api_args)
@na_utils.trace
def manage_volume(self, aggregate_name, volume_name,
def modify_volume(self, aggregate_name, volume_name,
thin_provisioned=False, snapshot_policy=None,
language=None, dedup_enabled=False,
compression_enabled=False, max_files=None, **options):
"""Update volume as needed to bring under management as a share."""
compression_enabled=False, max_files=None,
qos_policy_group=None, **options):
"""Update backend volume for a share as necessary."""
api_args = {
'query': {
'volume-attributes': {
@ -1594,7 +1598,7 @@ class NetAppCmodeClient(client_base.NetAppBaseClient):
'volume-snapshot-attributes': {},
'volume-space-attributes': {
'space-guarantee': ('none' if thin_provisioned else
'volume')
'volume'),
},
},
},
@ -1609,6 +1613,11 @@ class NetAppCmodeClient(client_base.NetAppBaseClient):
api_args['attributes']['volume-attributes'][
'volume-snapshot-attributes'][
'snapshot-policy'] = snapshot_policy
if qos_policy_group:
api_args['attributes']['volume-attributes'][
'volume-qos-attributes'] = {
'policy-group-name': qos_policy_group,
}
self.send_request('volume-modify-iter', api_args)
@ -1766,9 +1775,12 @@ class NetAppCmodeClient(client_base.NetAppBaseClient):
'type': None,
'style': None,
},
'volume-qos-attributes': {
'policy-group-name': None,
},
'volume-space-attributes': {
'size': None,
}
},
},
},
}
@ -1789,6 +1801,8 @@ class NetAppCmodeClient(client_base.NetAppBaseClient):
volume_id_attributes = volume_attributes.get_child_by_name(
'volume-id-attributes') or netapp_api.NaElement('none')
volume_qos_attributes = volume_attributes.get_child_by_name(
'volume-qos-attributes') or netapp_api.NaElement('none')
volume_space_attributes = volume_attributes.get_child_by_name(
'volume-space-attributes') or netapp_api.NaElement('none')
@ -1803,6 +1817,8 @@ class NetAppCmodeClient(client_base.NetAppBaseClient):
'type': volume_id_attributes.get_child_content('type'),
'style': volume_id_attributes.get_child_content('style'),
'size': volume_space_attributes.get_child_content('size'),
'qos-policy-group-name': volume_qos_attributes.get_child_content(
'policy-group-name')
}
return volume
@ -1883,9 +1899,12 @@ class NetAppCmodeClient(client_base.NetAppBaseClient):
'style': None,
'owning-vserver-name': None,
},
'volume-qos-attributes': {
'policy-group-name': None,
},
'volume-space-attributes': {
'size': None,
}
},
},
},
}
@ -1899,6 +1918,8 @@ class NetAppCmodeClient(client_base.NetAppBaseClient):
'volume-attributes') or netapp_api.NaElement('none')
volume_id_attributes = volume_attributes.get_child_by_name(
'volume-id-attributes') or netapp_api.NaElement('none')
volume_qos_attributes = volume_attributes.get_child_by_name(
'volume-qos-attributes') or netapp_api.NaElement('none')
volume_space_attributes = volume_attributes.get_child_by_name(
'volume-space-attributes') or netapp_api.NaElement('none')
@ -1913,12 +1934,16 @@ class NetAppCmodeClient(client_base.NetAppBaseClient):
'owning-vserver-name': volume_id_attributes.get_child_content(
'owning-vserver-name'),
'size': volume_space_attributes.get_child_content('size'),
'qos-policy-group-name': volume_qos_attributes.get_child_content(
'policy-group-name')
}
return volume
@na_utils.trace
def create_volume_clone(self, volume_name, parent_volume_name,
parent_snapshot_name=None, split=False, **options):
parent_snapshot_name=None, split=False,
qos_policy_group=None, **options):
"""Clones a volume."""
api_args = {
'volume': volume_name,
@ -1926,6 +1951,10 @@ class NetAppCmodeClient(client_base.NetAppBaseClient):
'parent-snapshot': parent_snapshot_name,
'junction-path': '/%s' % volume_name,
}
if qos_policy_group is not None:
api_args['qos-policy-group-name'] = qos_policy_group
self.send_request('volume-clone-create', api_args)
if split:
@ -2515,6 +2544,27 @@ class NetAppCmodeClient(client_base.NetAppBaseClient):
}
self.send_request('volume-modify-iter', api_args)
@na_utils.trace
def set_qos_policy_group_for_volume(self, volume_name,
qos_policy_group_name):
api_args = {
'query': {
'volume-attributes': {
'volume-id-attributes': {
'name': volume_name,
},
},
},
'attributes': {
'volume-attributes': {
'volume-qos-attributes': {
'policy-group-name': qos_policy_group_name,
},
},
},
}
self.send_request('volume-modify-iter', api_args)
@na_utils.trace
def get_nfs_export_policy_for_volume(self, volume_name):
"""Get the name of the export policy for a volume."""
@ -3443,3 +3493,132 @@ class NetAppCmodeClient(client_base.NetAppBaseClient):
'phase': volume_move_info.get_child_content('phase'),
}
return status_info
@na_utils.trace
def qos_policy_group_exists(self, qos_policy_group_name):
"""Checks if a QoS policy group exists."""
try:
self.qos_policy_group_get(qos_policy_group_name)
except exception.NetAppException:
return False
return True
@na_utils.trace
def qos_policy_group_get(self, qos_policy_group_name):
"""Checks if a QoS policy group exists."""
api_args = {
'query': {
'qos-policy-group-info': {
'policy-group': qos_policy_group_name,
},
},
'desired-attributes': {
'qos-policy-group-info': {
'policy-group': None,
'vserver': None,
'max-throughput': None,
'num-workloads': None
},
},
}
result = self.send_request('qos-policy-group-get-iter', api_args,
False)
if not self._has_records(result):
msg = _("No QoS policy group found with name %s.")
raise exception.NetAppException(msg % qos_policy_group_name)
attributes_list = result.get_child_by_name(
'attributes-list') or netapp_api.NaElement('none')
qos_policy_group_info = attributes_list.get_child_by_name(
'qos-policy-group-info') or netapp_api.NaElement('none')
policy_info = {
'policy-group': qos_policy_group_info.get_child_content(
'policy-group'),
'vserver': qos_policy_group_info.get_child_content('vserver'),
'max-throughput': qos_policy_group_info.get_child_content(
'max-throughput'),
'num-workloads': int(qos_policy_group_info.get_child_content(
'num-workloads')),
}
return policy_info
@na_utils.trace
def qos_policy_group_create(self, qos_policy_group_name, vserver,
max_throughput=None):
"""Creates a QoS policy group."""
api_args = {
'policy-group': qos_policy_group_name,
'vserver': vserver,
}
if max_throughput:
api_args['max-throughput'] = max_throughput
return self.send_request('qos-policy-group-create', api_args, False)
@na_utils.trace
def qos_policy_group_modify(self, qos_policy_group_name, max_throughput):
"""Modifies a QoS policy group."""
api_args = {
'policy-group': qos_policy_group_name,
'max-throughput': max_throughput,
}
return self.send_request('qos-policy-group-modify', api_args, False)
@na_utils.trace
def qos_policy_group_delete(self, qos_policy_group_name):
"""Attempts to delete a QoS policy group."""
api_args = {'policy-group': qos_policy_group_name}
return self.send_request('qos-policy-group-delete', api_args, False)
@na_utils.trace
def qos_policy_group_rename(self, qos_policy_group_name, new_name):
"""Renames a QoS policy group."""
api_args = {
'policy-group-name': qos_policy_group_name,
'new-name': new_name,
}
return self.send_request('qos-policy-group-rename', api_args, False)
@na_utils.trace
def mark_qos_policy_group_for_deletion(self, qos_policy_group_name):
"""Soft delete backing QoS policy group for a manila share."""
# NOTE(gouthamr): ONTAP deletes storage objects asynchronously. As
# long as garbage collection hasn't occurred, assigned QoS policy may
# still be tagged "in use". So, we rename the QoS policy group using a
# specific pattern and later attempt on a best effort basis to
# delete any QoS policy groups matching that pattern.
if self.qos_policy_group_exists(qos_policy_group_name):
new_name = DELETED_PREFIX + qos_policy_group_name
try:
self.qos_policy_group_rename(qos_policy_group_name, new_name)
except netapp_api.NaApiError as ex:
msg = ('Rename failure in cleanup of cDOT QoS policy '
'group %(name)s: %(ex)s')
msg_args = {'name': qos_policy_group_name, 'ex': ex}
LOG.warning(msg, msg_args)
# Attempt to delete any QoS policies named "deleted_manila-*".
self.remove_unused_qos_policy_groups()
@na_utils.trace
def remove_unused_qos_policy_groups(self):
"""Deletes all QoS policy groups that are marked for deletion."""
api_args = {
'query': {
'qos-policy-group-info': {
'policy-group': '%s*' % DELETED_PREFIX,
}
},
'max-records': 3500,
'continue-on-failure': 'true',
'return-success-list': 'false',
'return-failure-list': 'false',
}
try:
self.send_request('qos-policy-group-delete-iter', api_args, False)
except netapp_api.NaApiError as ex:
msg = 'Could not delete QoS policy groups. Details: %(ex)s'
msg_args = {'ex': ex}
LOG.debug(msg, msg_args)

33
manila/share/drivers/netapp/dataontap/cluster_mode/data_motion.py
View File

@ -92,6 +92,12 @@ class DataMotionSession(object):
'share_id': share_obj['id'].replace('-', '_')}
return volume_name
def _get_backend_qos_policy_group_name(self, share):
"""Get QoS policy name according to QoS policy group name template."""
__, config = self._get_backend_config_obj(share)
return config.netapp_qos_policy_group_name_template % {
'share_id': share['id'].replace('-', '_')}
def get_vserver_from_share(self, share_obj):
share_server = share_obj.get('share_server')
if share_server:
@ -99,11 +105,14 @@ class DataMotionSession(object):
if backend_details:
return backend_details.get('vserver_name')
def get_backend_info_for_share(self, share_obj):
def _get_backend_config_obj(self, share_obj):
backend_name = share_utils.extract_host(
share_obj['host'], level='backend_name')
config = get_backend_configuration(backend_name)
return backend_name, config
def get_backend_info_for_share(self, share_obj):
backend_name, config = self._get_backend_config_obj(share_obj)
vserver = (self.get_vserver_from_share(share_obj) or
config.netapp_vserver)
volume_name = self._get_backend_volume_name(
@ -379,3 +388,23 @@ class DataMotionSession(object):
new_src_volume_name,
replica_vserver,
replica_volume_name)
@na_utils.trace
def remove_qos_on_old_active_replica(self, orig_active_replica):
old_active_replica_qos_policy = (
self._get_backend_qos_policy_group_name(orig_active_replica)
)
replica_volume_name, replica_vserver, replica_backend = (
self.get_backend_info_for_share(orig_active_replica))
replica_client = get_client_for_backend(
replica_backend, vserver_name=replica_vserver)
try:
replica_client.set_qos_policy_group_for_volume(
replica_volume_name, 'none')
replica_client.mark_qos_policy_group_for_deletion(
old_active_replica_qos_policy)
except exception.StorageCommunicationException:
LOG.exception("Could not communicate with the backend "
"for replica %s to unset QoS policy and mark "
"the QoS policy group for deletion.",
orig_active_replica['id'])

264
manila/share/drivers/netapp/dataontap/cluster_mode/lib_base.py
View File

@ -82,6 +82,15 @@ class NetAppCmodeFileStorageLibrary(object):
'compression': 'netapp:compression',
}
QOS_SPECS = {
'netapp:maxiops': 'maxiops',
'netapp:maxiopspergib': 'maxiopspergib',
'netapp:maxbps': 'maxbps',
'netapp:maxbpspergib': 'maxbpspergib',
}
SIZE_DEPENDENT_QOS_SPECS = {'maxiopspergib', 'maxbpspergib'}
def __init__(self, driver_name, **kwargs):
na_utils.validate_driver_instantiation(**kwargs)
@ -207,6 +216,11 @@ class NetAppCmodeFileStorageLibrary(object):
"""Get snapshot name according to snapshot name template."""
return 'share_cg_snapshot_' + snapshot_id.replace('-', '_')
def _get_backend_qos_policy_group_name(self, share_id):
"""Get QoS policy name according to QoS policy group name template."""
return self.configuration.netapp_qos_policy_group_name_template % {
'share_id': share_id.replace('-', '_')}
@na_utils.trace
def _get_aggregate_space(self):
aggregates = self._find_matching_aggregates()
@ -273,9 +287,12 @@ class NetAppCmodeFileStorageLibrary(object):
aggr_space = self._get_aggregate_space()
aggregates = aggr_space.keys()
# Get up-to-date node utilization metrics just once.
if self._have_cluster_creds:
# Get up-to-date node utilization metrics just once.
self._perf_library.update_performance_cache({}, self._ssc_stats)
qos_support = True
else:
qos_support = False
for aggr_name in sorted(aggregates):
@ -298,7 +315,7 @@ class NetAppCmodeFileStorageLibrary(object):
'total_capacity_gb': total_capacity_gb,
'free_capacity_gb': free_capacity_gb,
'allocated_capacity_gb': allocated_capacity_gb,
'qos': 'False',
'qos': qos_support,
'reserved_percentage': reserved_percentage,
'dedupe': [True, False],
'compression': [True, False],
@ -423,7 +440,7 @@ class NetAppCmodeFileStorageLibrary(object):
def create_share(self, context, share, share_server):
"""Creates new share."""
vserver, vserver_client = self._get_vserver(share_server=share_server)
self._allocate_container(share, vserver_client)
self._allocate_container(share, vserver, vserver_client)
return self._create_export(share, share_server, vserver,
vserver_client)
@ -432,12 +449,14 @@ class NetAppCmodeFileStorageLibrary(object):
share_server=None):
"""Creates new share from snapshot."""
vserver, vserver_client = self._get_vserver(share_server=share_server)
self._allocate_container_from_snapshot(share, snapshot, vserver_client)
self._allocate_container_from_snapshot(
share, snapshot, vserver, vserver_client)
return self._create_export(share, share_server, vserver,
vserver_client)
@na_utils.trace
def _allocate_container(self, share, vserver_client, replica=False):
def _allocate_container(self, share, vserver, vserver_client,
replica=False):
"""Create new share on aggregate."""
share_name = self._get_backend_share_name(share['id'])
@ -447,7 +466,8 @@ class NetAppCmodeFileStorageLibrary(object):
msg = _("Pool is not available in the share host field.")
raise exception.InvalidHost(reason=msg)
provisioning_options = self._get_provisioning_options_for_share(share)
provisioning_options = self._get_provisioning_options_for_share(
share, vserver, replica=replica)
if replica:
# If this volume is intended to be a replication destination,
@ -582,8 +602,55 @@ class NetAppCmodeFileStorageLibrary(object):
# provisioning methods from the client API library.
return dict(zip(provisioning_args, provisioning_values))
def _get_normalized_qos_specs(self, extra_specs):
if not extra_specs.get('qos'):
return {}
normalized_qos_specs = {
self.QOS_SPECS[key.lower()]: value
for key, value in extra_specs.items()
if self.QOS_SPECS.get(key.lower())
}
if not normalized_qos_specs:
msg = _("The extra-spec 'qos' is set to True, but no netapp "
"supported qos-specs have been specified in the share "
"type. Cannot provision a QoS policy. Specify any of the "
"following extra-specs and try again: %s")
raise exception.NetAppException(msg % list(self.QOS_SPECS))
# TODO(gouthamr): Modify check when throughput floors are allowed
if len(normalized_qos_specs) > 1:
msg = _('Only one NetApp QoS spec can be set at a time. '
'Specified QoS limits: %s')
raise exception.NetAppException(msg % normalized_qos_specs)
return normalized_qos_specs
def _get_max_throughput(self, share_size, qos_specs):
# QoS limits are exclusive of one another.
if 'maxiops' in qos_specs:
return '%siops' % qos_specs['maxiops']
elif 'maxiopspergib' in qos_specs:
return '%siops' % six.text_type(
int(qos_specs['maxiopspergib']) * int(share_size))
elif 'maxbps' in qos_specs:
return '%sB/s' % qos_specs['maxbps']
elif 'maxbpspergib' in qos_specs:
return '%sB/s' % six.text_type(
int(qos_specs['maxbpspergib']) * int(share_size))
@na_utils.trace
def _get_provisioning_options_for_share(self, share):
def _create_qos_policy_group(self, share, vserver, qos_specs):
max_throughput = self._get_max_throughput(share['size'], qos_specs)
qos_policy_group_name = self._get_backend_qos_policy_group_name(
share['id'])
self._client.qos_policy_group_create(qos_policy_group_name, vserver,
max_throughput=max_throughput)
return qos_policy_group_name
@na_utils.trace
def _get_provisioning_options_for_share(self, share, vserver,
replica=False):
"""Return provisioning options from a share.
Starting with a share, this method gets the extra specs, rationalizes
@ -594,7 +661,13 @@ class NetAppCmodeFileStorageLibrary(object):
extra_specs = share_types.get_extra_specs_from_share(share)
extra_specs = self._remap_standard_boolean_extra_specs(extra_specs)
self._check_extra_specs_validity(share, extra_specs)
return self._get_provisioning_options(extra_specs)
provisioning_options = self._get_provisioning_options(extra_specs)
qos_specs = self._get_normalized_qos_specs(extra_specs)
if qos_specs and not replica:
qos_policy_group = self._create_qos_policy_group(
share, vserver, qos_specs)
provisioning_options['qos_policy_group'] = qos_policy_group
return provisioning_options
@na_utils.trace
def _get_provisioning_options(self, specs):
@ -627,7 +700,7 @@ class NetAppCmodeFileStorageLibrary(object):
@na_utils.trace
def _allocate_container_from_snapshot(
self, share, snapshot, vserver_client,
self, share, snapshot, vserver, vserver_client,
snapshot_name_func=_get_backend_snapshot_name):
"""Clones existing share."""
share_name = self._get_backend_share_name(share['id'])
@ -637,7 +710,8 @@ class NetAppCmodeFileStorageLibrary(object):
else:
parent_snapshot_name = snapshot['provider_location']
provisioning_options = self._get_provisioning_options_for_share(share)
provisioning_options = self._get_provisioning_options_for_share(
share, vserver)
LOG.debug('Creating share from snapshot %s', snapshot['id'])
vserver_client.create_volume_clone(share_name, parent_share_name,
@ -667,6 +741,10 @@ class NetAppCmodeFileStorageLibrary(object):
if self._share_exists(share_name, vserver_client):
self._remove_export(share, vserver_client)
self._deallocate_container(share_name, vserver_client)
qos_policy_for_share = self._get_backend_qos_policy_group_name(
share['id'])
self._client.mark_qos_policy_group_for_deletion(
qos_policy_for_share)
else:
LOG.info("Share %s does not exist.", share['id'])
@ -856,7 +934,7 @@ class NetAppCmodeFileStorageLibrary(object):
@na_utils.trace
def manage_existing(self, share, driver_options):
vserver, vserver_client = self._get_vserver(share_server=None)
share_size = self._manage_container(share, vserver_client)
share_size = self._manage_container(share, vserver, vserver_client)
export_locations = self._create_export(share, None, vserver,
vserver_client)
return {'size': share_size, 'export_locations': export_locations}
@ -866,7 +944,7 @@ class NetAppCmodeFileStorageLibrary(object):
pass
@na_utils.trace
def _manage_container(self, share, vserver_client):
def _manage_container(self, share, vserver, vserver_client):
"""Bring existing volume under management as a share."""
protocol_helper = self._get_helper(share)
@ -891,6 +969,9 @@ class NetAppCmodeFileStorageLibrary(object):
msg_args = {'volume': volume_name, 'aggr': aggregate_name}
raise exception.ManageInvalidShare(reason=msg % msg_args)
# When calculating the size, round up to the next GB.
volume_size = int(math.ceil(float(volume['size']) / units.Gi))
# Ensure volume is manageable
self._validate_volume_for_manage(volume, vserver_client)
@ -918,8 +999,13 @@ class NetAppCmodeFileStorageLibrary(object):
vserver_client.set_volume_name(volume_name, share_name)
vserver_client.mount_volume(share_name)
qos_policy_group_name = self._modify_or_create_qos_for_existing_share(
share, extra_specs, vserver, vserver_client)
if qos_policy_group_name:
provisioning_options['qos_policy_group'] = qos_policy_group_name
# Modify volume to match extra specs
vserver_client.manage_volume(aggregate_name, share_name,
vserver_client.modify_volume(aggregate_name, share_name,
**provisioning_options)
# Save original volume info to private storage
@ -929,8 +1015,7 @@ class NetAppCmodeFileStorageLibrary(object):
}
self.private_storage.update(share['id'], original_data)
# When calculating the size, round up to the next GB.
return int(math.ceil(float(volume['size']) / units.Gi))
return volume_size
@na_utils.trace
def _validate_volume_for_manage(self, volume, vserver_client):
@ -1039,7 +1124,7 @@ class NetAppCmodeFileStorageLibrary(object):
for clone in clone_list:
self._allocate_container_from_snapshot(
clone['share'], clone['snapshot'], vserver_client,
clone['share'], clone['snapshot'], vserver, vserver_client,
NetAppCmodeFileStorageLibrary._get_backend_cg_snapshot_name)
export_locations = self._create_export(clone['share'],
@ -1151,6 +1236,27 @@ class NetAppCmodeFileStorageLibrary(object):
return None, None
@na_utils.trace
def _adjust_qos_policy_with_volume_resize(self, share, new_size,
vserver_client):
# Adjust QoS policy on a share if any
if self._have_cluster_creds:
share_name = self._get_backend_share_name(share['id'])
share_on_the_backend = vserver_client.get_volume(share_name)
qos_policy_on_share = share_on_the_backend['qos-policy-group-name']
if qos_policy_on_share is None:
return
extra_specs = share_types.get_extra_specs_from_share(share)
qos_specs = self._get_normalized_qos_specs(extra_specs)
size_dependent_specs = {k: v for k, v in qos_specs.items() if k in
self.SIZE_DEPENDENT_QOS_SPECS}
if size_dependent_specs:
max_throughput = self._get_max_throughput(
new_size, size_dependent_specs)
self._client.qos_policy_group_modify(
qos_policy_on_share, max_throughput)
@na_utils.trace
def extend_share(self, share, new_size, share_server=None):
"""Extends size of existing share."""
@ -1159,6 +1265,8 @@ class NetAppCmodeFileStorageLibrary(object):
LOG.debug('Extending share %(name)s to %(size)s GB.',
{'name': share_name, 'size': new_size})
vserver_client.set_volume_size(share_name, new_size)
self._adjust_qos_policy_with_volume_resize(share, new_size,
vserver_client)
@na_utils.trace
def shrink_share(self, share, new_size, share_server=None):
@ -1168,6 +1276,8 @@ class NetAppCmodeFileStorageLibrary(object):
LOG.debug('Shrinking share %(name)s to %(size)s GB.',
{'name': share_name, 'size': new_size})
vserver_client.set_volume_size(share_name, new_size)
self._adjust_qos_policy_with_volume_resize(share, new_size,
vserver_client)
@na_utils.trace
def update_access(self, context, share, access_rules, add_rules,
@ -1288,7 +1398,8 @@ class NetAppCmodeFileStorageLibrary(object):
vserver_client = data_motion.get_client_for_backend(
dest_backend, vserver_name=vserver)
self._allocate_container(new_replica, vserver_client, replica=True)
self._allocate_container(new_replica, vserver, vserver_client,
replica=True)
# 2. Setup SnapMirror
dm_session.create_snapmirror(active_replica, new_replica)
@ -1454,8 +1565,52 @@ class NetAppCmodeFileStorageLibrary(object):
replica_list)
new_replica_list.append(r)
self._handle_qos_on_replication_change(dm_session,
new_active_replica,
orig_active_replica,
share_server=share_server)
return new_replica_list
def _handle_qos_on_replication_change(self, dm_session, new_active_replica,
orig_active_replica,
share_server=None):
# QoS operations: Remove and purge QoS policy on old active replica
# if any and create a new policy on the destination if necessary.
extra_specs = share_types.get_extra_specs_from_share(
orig_active_replica)
qos_specs = self._get_normalized_qos_specs(extra_specs)
if qos_specs and self._have_cluster_creds:
dm_session.remove_qos_on_old_active_replica(orig_active_replica)
# Check if a QoS policy already exists for the promoted replica,
# if it does, modify it as necessary, else create it:
try:
new_active_replica_qos_policy = (
self._get_backend_qos_policy_group_name(
new_active_replica['id']))
vserver, vserver_client = self._get_vserver(
share_server=share_server)
volume_name_on_backend = self._get_backend_share_name(
new_active_replica['id'])
if not self._client.qos_policy_group_exists(
new_active_replica_qos_policy):
self._create_qos_policy_group(
new_active_replica, vserver, qos_specs)
else:
max_throughput = self._get_max_throughput(
new_active_replica['size'], qos_specs)
self._client.qos_policy_group_modify(
new_active_replica_qos_policy, max_throughput)
vserver_client.set_qos_policy_group_for_volume(
volume_name_on_backend, new_active_replica_qos_policy)
LOG.info("QoS policy applied successfully for promoted "
"replica: %s", new_active_replica['id'])
except Exception:
LOG.exception("Could not apply QoS to the promoted replica.")
def _convert_destination_replica_to_independent(
self, context, dm_session, orig_active_replica, replica,
access_rules, share_server=None):
@ -1721,12 +1876,13 @@ class NetAppCmodeFileStorageLibrary(object):
destination_host, level='backend_name')
destination_aggregate = share_utils.extract_host(
destination_host, level='pool')
# Validate new share type extra-specs are valid on the
# destination
# Validate new extra-specs are valid on the destination
extra_specs = share_types.get_extra_specs_from_share(
destination_share)
self._check_extra_specs_validity(
destination_share, extra_specs)
# TODO(gouthamr): Check whether QoS min-throughputs can be
# honored on the destination aggregate when supported.
self._check_aggregate_extra_specs_validity(
destination_aggregate, extra_specs)
@ -1916,11 +2072,15 @@ class NetAppCmodeFileStorageLibrary(object):
extra_specs = share_types.get_extra_specs_from_share(
destination_share)
provisioning_options = self._get_provisioning_options(extra_specs)
qos_policy_group_name = self._modify_or_create_qos_for_existing_share(
destination_share, extra_specs, vserver, vserver_client)
if qos_policy_group_name:
provisioning_options['qos_policy_group'] = qos_policy_group_name
destination_aggregate = share_utils.extract_host(
destination_share['host'], level='pool')
# Modify volume to match extra specs
vserver_client.manage_volume(destination_aggregate,
vserver_client.modify_volume(destination_aggregate,
new_share_volume_name,
**provisioning_options)
@ -1954,6 +2114,70 @@ class NetAppCmodeFileStorageLibrary(object):
'snapshot_updates': snapshot_updates,
}
@na_utils.trace
def _modify_or_create_qos_for_existing_share(self, share, extra_specs,
vserver, vserver_client):
"""Gets/Creates QoS policy for an existing FlexVol.
The share's assigned QoS policy is renamed and adjusted if the policy
is exclusive to the FlexVol. If the policy includes other workloads
besides the FlexVol, a new policy is created with the specs necessary.
"""
qos_specs = self._get_normalized_qos_specs(extra_specs)
if not qos_specs:
return
backend_share_name = self._get_backend_share_name(share['id'])
qos_policy_group_name = self._get_backend_qos_policy_group_name(
share['id'])
create_new_qos_policy_group = True
backend_volume = vserver_client.get_volume(
backend_share_name)
backend_volume_size = int(
math.ceil(float(backend_volume['size']) / units.Gi))
LOG.debug("Checking for a pre-existing QoS policy group that "
"is exclusive to the volume %s." % backend_share_name)
# Does the volume have an exclusive QoS policy that we can rename?
if backend_volume['qos-policy-group-name'] is not None:
existing_qos_policy_group = self._client.qos_policy_group_get(
backend_volume['qos-policy-group-name'])
if existing_qos_policy_group['num-workloads'] == 1:
# Yay, can set max-throughput and rename
msg = ("Found pre-existing QoS policy %(policy)s and it is "
"exclusive to the volume %(volume)s. Modifying and "
"renaming this policy to %(new_policy)s.")
msg_args = {
'policy': backend_volume['qos-policy-group-name'],
'volume': backend_share_name,
'new_policy': qos_policy_group_name,
}
LOG.debug(msg, msg_args)
max_throughput = self._get_max_throughput(
backend_volume_size, qos_specs)
self._client.qos_policy_group_modify(
backend_volume['qos-policy-group-name'], max_throughput)
self._client.qos_policy_group_rename(
backend_volume['qos-policy-group-name'],
qos_policy_group_name)
create_new_qos_policy_group = False
if create_new_qos_policy_group:
share_obj = {
'size': backend_volume_size,
'id': share['id'],
}
LOG.debug("No existing QoS policy group found for "
"volume. Creating a new one with name %s.",
qos_policy_group_name)
self._create_qos_policy_group(share_obj, vserver, qos_specs)
return qos_policy_group_name
def _wait_for_cutover_completion(self, source_share, share_server):
retries = (self.configuration.netapp_volume_move_cutover_timeout / 5

1
manila/share/drivers/netapp/dataontap/cluster_mode/lib_multi_svm.py
View File

@ -101,6 +101,7 @@ class NetAppCmodeMultiSVMFileStorageLibrary(
"""Handle various cleanup activities."""
self._client.prune_deleted_nfs_export_policies()
self._client.prune_deleted_snapshots()
self._client.remove_unused_qos_policy_groups()
(super(NetAppCmodeMultiSVMFileStorageLibrary, self).
_handle_housekeeping_tasks())

4
manila/share/drivers/netapp/dataontap/cluster_mode/lib_single_svm.py
View File

@ -115,6 +115,10 @@ class NetAppCmodeSingleSVMFileStorageLibrary(
vserver_client.prune_deleted_nfs_export_policies()
vserver_client.prune_deleted_snapshots()
if self._have_cluster_creds:
# Harvest soft-deleted QoS policy groups
vserver_client.remove_unused_qos_policy_groups()
(super(NetAppCmodeSingleSVMFileStorageLibrary, self).
_handle_housekeeping_tasks())

3
manila/share/drivers/netapp/options.py
View File

@ -72,6 +72,9 @@ netapp_provisioning_opts = [
cfg.StrOpt('netapp_vserver_name_template',
default='os_%s',
help='Name template to use for new Vserver.'),
cfg.StrOpt('netapp_qos_policy_group_name_template',
help='NetApp QoS policy group name template.',
default='qos_share_%(share_id)s'),
cfg.StrOpt('netapp_port_name_search_pattern',
default='(.*)',
help='Pattern for overriding the selection of network ports '

63
manila/tests/share/drivers/netapp/dataontap/client/fakes.py
View File

@ -67,6 +67,8 @@ DELETED_EXPORT_POLICIES = {
'deleted_manila_fake_policy_3',
],
}
QOS_POLICY_GROUP_NAME = 'fake_qos_policy_group_name'
QOS_MAX_THROUGHPUT = '5000B/s'
USER_NAME = 'fake_user'
@ -142,6 +144,13 @@ EMS_MESSAGE = {
'auto-support': 'false',
}
QOS_POLICY_GROUP = {
'policy-group': QOS_POLICY_GROUP_NAME,
'vserver': VSERVER_NAME,
'max-throughput': QOS_MAX_THROUGHPUT,
'num-workloads': 1,
}
NO_RECORDS_RESPONSE = etree.XML("""
<results status="passed">
<num-records>0</num-records>
@ -152,6 +161,13 @@ PASSED_RESPONSE = etree.XML("""
<results status="passed" />
""")
PASSED_FAILED_ITER_RESPONSE = etree.XML("""
<results status="passed">
<num-failed>0</num-failed>
<num-succeeded>1</num-succeeded>
</results>
""")
INVALID_GET_ITER_RESPONSE_NO_ATTRIBUTES = etree.XML("""
<results status="passed">
<num-records>1</num-records>
@ -1983,6 +1999,36 @@ VOLUME_GET_ITER_JUNCTIONED_VOLUMES_RESPONSE = etree.XML("""
""")
VOLUME_GET_ITER_VOLUME_TO_MANAGE_RESPONSE = etree.XML("""
<results status="passed">
<attributes-list>
<volume-attributes>
<volume-id-attributes>
<containing-aggregate-name>%(aggr)s</containing-aggregate-name>
<junction-path>/%(volume)s</junction-path>
<name>%(volume)s</name>
<owning-vserver-name>%(vserver)s</owning-vserver-name>
<style>flex</style>
<type>rw</type>
</volume-id-attributes>
<volume-space-attributes>
<size>%(size)s</size>
</volume-space-attributes>
<volume-qos-attributes>
<policy-group-name>%(qos-policy-group-name)s</policy-group-name>
</volume-qos-attributes>
</volume-attributes>
</attributes-list>
<num-records>1</num-records>
</results>
""" % {
'aggr': SHARE_AGGREGATE_NAME,
'vserver': VSERVER_NAME,
'volume': SHARE_NAME,
'size': SHARE_SIZE,
'qos-policy-group-name': QOS_POLICY_GROUP_NAME,
})
VOLUME_GET_ITER_NO_QOS_RESPONSE = etree.XML("""
<results status="passed">
<attributes-list>
<volume-attributes>
@ -2332,6 +2378,23 @@ NET_ROUTES_CREATE_RESPONSE = etree.XML("""
'subnet': SUBNET,
})
QOS_POLICY_GROUP_GET_ITER_RESPONSE = etree.XML("""
<results status="passed">
<attributes-list>
<qos-policy-group-info>
<max-throughput>%(max_througput)s</max-throughput>
<num-workloads>1</num-workloads>
<policy-group>%(qos_policy_group_name)s</policy-group>
<vserver>%(vserver)s</vserver>
</qos-policy-group-info>
</attributes-list>
<num-records>1</num-records>
</results>""" % {
'qos_policy_group_name': QOS_POLICY_GROUP_NAME,
'vserver': VSERVER_NAME,
'max_througput': QOS_MAX_THROUGHPUT,
})
FAKE_VOL_XML = """<volume-info xmlns='http://www.netapp.com/filer/admin'>
<name>open123</name>
<state>online</state>

337
manila/tests/share/drivers/netapp/dataontap/client/test_client_cmode.py
View File

@ -2483,7 +2483,8 @@ class NetAppClientCmodeTestCase(test.TestCase):
self.client.send_request.assert_called_once_with('volume-create',
volume_create_args)
def test_create_volume_with_extra_specs(self):
@ddt.data(None, fake.QOS_POLICY_GROUP_NAME)
def test_create_volume_with_extra_specs(self, qos_policy_group_name):
self.mock_object(self.client, 'set_volume_max_files')
self.mock_object(self.client, 'enable_dedup')
@ -2494,7 +2495,8 @@ class NetAppClientCmodeTestCase(test.TestCase):
fake.SHARE_AGGREGATE_NAME, fake.SHARE_NAME, 100,
thin_provisioned=True, language='en-US',
snapshot_policy='default', dedup_enabled=True,
compression_enabled=True, max_files=5000, snapshot_reserve=15)
compression_enabled=True, max_files=5000, snapshot_reserve=15,
qos_policy_group=qos_policy_group_name)
volume_create_args = {
'containing-aggr-name': fake.SHARE_AGGREGATE_NAME,
@ -2508,6 +2510,10 @@ class NetAppClientCmodeTestCase(test.TestCase):
'percentage-snapshot-reserve': '15',
}
if qos_policy_group_name:
volume_create_args.update(
{'qos-policy-group-name': qos_policy_group_name})
self.client.send_request.assert_called_with('volume-create',
volume_create_args)
self.client.set_volume_max_files.assert_called_once_with(
@ -2645,13 +2651,13 @@ class NetAppClientCmodeTestCase(test.TestCase):
self.client.send_request.assert_called_once_with(
'volume-rename', volume_rename_api_args)
def test_manage_volume_no_optional_args(self):
def test_modify_volume_no_optional_args(self):
self.mock_object(self.client, 'send_request')
mock_update_volume_efficiency_attributes = self.mock_object(
self.client, 'update_volume_efficiency_attributes')
self.client.manage_volume(fake.SHARE_AGGREGATE_NAME, fake.SHARE_NAME)
self.client.modify_volume(fake.SHARE_AGGREGATE_NAME, fake.SHARE_NAME)
volume_modify_iter_api_args = {
'query': {
@ -2679,20 +2685,21 @@ class NetAppClientCmodeTestCase(test.TestCase):
mock_update_volume_efficiency_attributes.assert_called_once_with(
fake.SHARE_NAME, False, False)
def test_manage_volume_all_optional_args(self):
def test_modify_volume_all_optional_args(self):
self.mock_object(self.client, 'send_request')
mock_update_volume_efficiency_attributes = self.mock_object(
self.client, 'update_volume_efficiency_attributes')
self.client.manage_volume(fake.SHARE_AGGREGATE_NAME,
self.client.modify_volume(fake.SHARE_AGGREGATE_NAME,
fake.SHARE_NAME,
thin_provisioned=True,
snapshot_policy=fake.SNAPSHOT_POLICY_NAME,
language=fake.LANGUAGE,
dedup_enabled=True,
compression_enabled=False,
max_files=fake.MAX_FILES)
max_files=fake.MAX_FILES,
qos_policy_group=fake.QOS_POLICY_GROUP_NAME)
volume_modify_iter_api_args = {
'query': {
@ -2717,6 +2724,9 @@ class NetAppClientCmodeTestCase(test.TestCase):
'volume-space-attributes': {
'space-guarantee': 'none',
},
'volume-qos-attributes': {
'policy-group-name': fake.QOS_POLICY_GROUP_NAME,
},
},
},
}
@ -3093,7 +3103,10 @@ class NetAppClientCmodeTestCase(test.TestCase):
},
'volume-space-attributes': {
'size': None,
}
},
'volume-qos-attributes': {
'policy-group-name': None,
},
},
},
}
@ -3106,6 +3119,58 @@ class NetAppClientCmodeTestCase(test.TestCase):
'style': 'flex',
'size': fake.SHARE_SIZE,
'owning-vserver-name': fake.VSERVER_NAME,
'qos-policy-group-name': fake.QOS_POLICY_GROUP_NAME,
}
self.client.send_request.assert_has_calls([
mock.call('volume-get-iter', volume_get_iter_args)])
self.assertDictEqual(expected, result)
def test_get_volume_no_qos(self):
api_response = netapp_api.NaElement(
fake.VOLUME_GET_ITER_NO_QOS_RESPONSE)
self.mock_object(self.client,
'send_request',
mock.Mock(return_value=api_response))
result = self.client.get_volume(fake.SHARE_NAME)
volume_get_iter_args = {
'query': {
'volume-attributes': {
'volume-id-attributes': {
'name': fake.SHARE_NAME,
},
},
},
'desired-attributes': {
'volume-attributes': {
'volume-id-attributes': {
'containing-aggregate-name': None,
'junction-path': None,
'name': None,
'owning-vserver-name': None,
'type': None,
'style': None,
},
'volume-space-attributes': {
'size': None,
},
'volume-qos-attributes': {
'policy-group-name': None,
},
},
},
}
expected = {
'aggregate': fake.SHARE_AGGREGATE_NAME,
'junction-path': '/%s' % fake.SHARE_NAME,
'name': fake.SHARE_NAME,
'type': 'rw',
'style': 'flex',
'size': fake.SHARE_SIZE,
'owning-vserver-name': fake.VSERVER_NAME,
'qos-policy-group-name': None,
}
self.client.send_request.assert_has_calls([
mock.call('volume-get-iter', volume_get_iter_args)])
@ -3230,7 +3295,10 @@ class NetAppClientCmodeTestCase(test.TestCase):
},
'volume-space-attributes': {
'size': None,
}
},
'volume-qos-attributes': {
'policy-group-name': None,
},
},
},
}
@ -3241,7 +3309,8 @@ class NetAppClientCmodeTestCase(test.TestCase):
'type': 'rw',
'style': 'flex',
'size': fake.SHARE_SIZE,
'owning-vserver-name': fake.VSERVER_NAME
'owning-vserver-name': fake.VSERVER_NAME,
'qos-policy-group-name': fake.QOS_POLICY_GROUP_NAME,
}
self.client.send_iter_request.assert_has_calls([
mock.call('volume-get-iter', volume_get_iter_args)])
@ -3259,14 +3328,16 @@ class NetAppClientCmodeTestCase(test.TestCase):
self.assertIsNone(result)
def test_create_volume_clone(self):
@ddt.data(None, fake.QOS_POLICY_GROUP_NAME)
def test_create_volume_clone(self, qos_policy_group_name):
self.mock_object(self.client, 'send_request')
self.mock_object(self.client, 'split_volume_clone')
self.client.create_volume_clone(fake.SHARE_NAME,
fake.PARENT_SHARE_NAME,
fake.PARENT_SNAPSHOT_NAME)
fake.PARENT_SNAPSHOT_NAME,
qos_policy_group=qos_policy_group_name)
volume_clone_create_args = {
'volume': fake.SHARE_NAME,
@ -3275,6 +3346,10 @@ class NetAppClientCmodeTestCase(test.TestCase):
'junction-path': '/%s' % fake.SHARE_NAME
}
if qos_policy_group_name:
volume_clone_create_args.update(
{'qos-policy-group-name': fake.QOS_POLICY_GROUP_NAME})
self.client.send_request.assert_has_calls([
mock.call('volume-clone-create', volume_clone_create_args)])
self.assertFalse(self.client.split_volume_clone.called)
@ -4269,6 +4344,32 @@ class NetAppClientCmodeTestCase(test.TestCase):
self.client.send_request.assert_has_calls([
mock.call('volume-modify-iter', volume_modify_iter_args)])
def test_set_qos_policy_group_for_volume(self):
self.mock_object(self.client, 'send_request')
self.client.set_qos_policy_group_for_volume(fake.SHARE_NAME,
fake.QOS_POLICY_GROUP_NAME)
volume_modify_iter_args = {
'query': {
'volume-attributes': {
'volume-id-attributes': {
'name': fake.SHARE_NAME,
},
},
},
'attributes': {
'volume-attributes': {
'volume-qos-attributes': {
'policy-group-name': fake.QOS_POLICY_GROUP_NAME,
},
},
},
}
self.client.send_request.assert_called_once_with(
'volume-modify-iter', volume_modify_iter_args)
def test_get_nfs_export_policy_for_volume(self):
api_response = netapp_api.NaElement(
@ -5759,3 +5860,215 @@ class NetAppClientCmodeTestCase(test.TestCase):
self.assertDictMatch(expected_status_info, actual_status_info)
self.client.send_iter_request.assert_called_once_with(
'volume-move-get-iter', expected_api_args)
def test_qos_policy_group_exists_no_records(self):
self.mock_object(self.client, 'qos_policy_group_get', mock.Mock(
side_effect=exception.NetAppException))
policy_exists = self.client.qos_policy_group_exists(
'i-dont-exist-but-i-am')
self.assertIs(False, policy_exists)
def test_qos_policy_group_exists(self):
self.mock_object(self.client, 'qos_policy_group_get',
mock.Mock(return_value=fake.QOS_POLICY_GROUP))
policy_exists = self.client.qos_policy_group_exists(
fake.QOS_POLICY_GROUP_NAME)
self.assertIs(True, policy_exists)
def test_qos_policy_group_get_none_found(self):
no_records_response = netapp_api.NaElement(fake.NO_RECORDS_RESPONSE)
self.mock_object(self.client, 'send_request',
mock.Mock(return_value=no_records_response))
self.assertRaises(exception.NetAppException,
self.client.qos_policy_group_get,
'non-existent-qos-policy')
qos_policy_group_get_iter_args = {
'query': {
'qos-policy-group-info': {
'policy-group': 'non-existent-qos-policy',
},
},
'desired-attributes': {
'qos-policy-group-info': {
'policy-group': None,
'vserver': None,
'max-throughput': None,
'num-workloads': None
},
},
}
self.client.send_request.assert_called_once_with(
'qos-policy-group-get-iter', qos_policy_group_get_iter_args, False)
def test_qos_policy_group_get(self):
api_response = netapp_api.NaElement(
fake.QOS_POLICY_GROUP_GET_ITER_RESPONSE)
self.mock_object(self.client, 'send_request',
mock.Mock(return_value=api_response))
qos_info = self.client.qos_policy_group_get(fake.QOS_POLICY_GROUP_NAME)
qos_policy_group_get_iter_args = {
'query': {
'qos-policy-group-info': {
'policy-group': fake.QOS_POLICY_GROUP_NAME,
},
},
'desired-attributes': {
'qos-policy-group-info': {
'policy-group': None,
'vserver': None,
'max-throughput': None,
'num-workloads': None
},
},
}
self.client.send_request.assert_called_once_with(
'qos-policy-group-get-iter', qos_policy_group_get_iter_args, False)
self.assertDictMatch(fake.QOS_POLICY_GROUP, qos_info)
@ddt.data(None, fake.QOS_MAX_THROUGHPUT)
def test_qos_policy_group_create(self, max_throughput):
self.mock_object(self.client, 'send_request',
mock.Mock(return_value=fake.PASSED_RESPONSE))
self.client.qos_policy_group_create(
fake.QOS_POLICY_GROUP_NAME, fake.VSERVER_NAME,
max_throughput=max_throughput)
qos_policy_group_create_args = {
'policy-group': fake.QOS_POLICY_GROUP_NAME,
'vserver': fake.VSERVER_NAME,
}
if max_throughput:
qos_policy_group_create_args.update(
{'max-throughput': max_throughput})
self.client.send_request.assert_called_once_with(
'qos-policy-group-create', qos_policy_group_create_args, False)
def test_qos_policy_group_modify(self):
self.mock_object(self.client, 'send_request',
mock.Mock(return_value=fake.PASSED_RESPONSE))
self.client.qos_policy_group_modify(fake.QOS_POLICY_GROUP_NAME,
'3000iops')
qos_policy_group_modify_args = {
'policy-group': fake.QOS_POLICY_GROUP_NAME,
'max-throughput': '3000iops',
}
self.client.send_request.assert_called_once_with(
'qos-policy-group-modify', qos_policy_group_modify_args, False)
def test_qos_policy_group_delete(self):
self.mock_object(self.client, 'send_request',
mock.Mock(return_value=fake.PASSED_RESPONSE))
self.client.qos_policy_group_delete(fake.QOS_POLICY_GROUP_NAME)
qos_policy_group_delete_args = {
'policy-group': fake.QOS_POLICY_GROUP_NAME,
}
self.client.send_request.assert_called_once_with(
'qos-policy-group-delete', qos_policy_group_delete_args, False)
def test_qos_policy_group_rename(self):
self.mock_object(self.client, 'send_request',
mock.Mock(return_value=fake.PASSED_RESPONSE))
self.client.qos_policy_group_rename(
fake.QOS_POLICY_GROUP_NAME, 'new_' + fake.QOS_POLICY_GROUP_NAME)
qos_policy_group_rename_args = {
'policy-group-name': fake.QOS_POLICY_GROUP_NAME,
'new-name': 'new_' + fake.QOS_POLICY_GROUP_NAME,
}
self.client.send_request.assert_called_once_with(
'qos-policy-group-rename', qos_policy_group_rename_args, False)
def test_mark_qos_policy_group_for_deletion_rename_failure(self):
self.mock_object(self.client, 'qos_policy_group_exists',
mock.Mock(return_value=True))
self.mock_object(self.client, 'qos_policy_group_rename',
mock.Mock(side_effect=netapp_api.NaApiError))
self.mock_object(client_cmode.LOG, 'warning')
self.mock_object(self.client, 'remove_unused_qos_policy_groups')
retval = self.client.mark_qos_policy_group_for_deletion(
fake.QOS_POLICY_GROUP_NAME)
self.assertIsNone(retval)
client_cmode.LOG.warning.assert_called_once()
self.client.qos_policy_group_exists.assert_called_once_with(
fake.QOS_POLICY_GROUP_NAME)
self.client.qos_policy_group_rename.assert_called_once_with(
fake.QOS_POLICY_GROUP_NAME,
client_cmode.DELETED_PREFIX + fake.QOS_POLICY_GROUP_NAME)
self.client.remove_unused_qos_policy_groups.assert_called_once_with()
@ddt.data(True, False)
def test_mark_qos_policy_group_for_deletion_policy_exists(self, exists):
self.mock_object(self.client, 'qos_policy_group_exists',
mock.Mock(return_value=exists))
self.mock_object(self.client, 'qos_policy_group_rename')
mock_remove_unused_policies = self.mock_object(
self.client, 'remove_unused_qos_policy_groups')
self.mock_object(client_cmode.LOG, 'warning')
retval = self.client.mark_qos_policy_group_for_deletion(
fake.QOS_POLICY_GROUP_NAME)
self.assertIsNone(retval)
if exists:
self.client.qos_policy_group_rename.assert_called_once_with(
fake.QOS_POLICY_GROUP_NAME,
client_cmode.DELETED_PREFIX + fake.QOS_POLICY_GROUP_NAME)
mock_remove_unused_policies.assert_called_once_with()
else:
self.assertFalse(self.client.qos_policy_group_rename.called)
self.assertFalse(
self.client.remove_unused_qos_policy_groups.called)
self.assertFalse(client_cmode.LOG.warning.called)
@ddt.data(True, False)
def test_remove_unused_qos_policy_groups_with_failure(self, failed):
if failed:
args = mock.Mock(side_effect=netapp_api.NaApiError)
else:
args = mock.Mock(return_value=fake.PASSED_FAILED_ITER_RESPONSE)
self.mock_object(self.client, 'send_request', args)
self.mock_object(client_cmode.LOG, 'debug')
retval = self.client.remove_unused_qos_policy_groups()
qos_policy_group_delete_iter_args = {
'query': {
'qos-policy-group-info': {
'policy-group': '%s*' % client_cmode.DELETED_PREFIX,
}
},
'max-records': 3500,
'continue-on-failure': 'true',
'return-success-list': 'false',
'return-failure-list': 'false',
}
self.assertIsNone(retval)
self.client.send_request.assert_called_once_with(
'qos-policy-group-delete-iter',
qos_policy_group_delete_iter_args, False)
self.assertIs(failed, client_cmode.LOG.debug.called)

40
manila/tests/share/drivers/netapp/dataontap/cluster_mode/test_data_motion.py
View File

@ -123,6 +123,7 @@ class NetAppCDOTDataMotionTestCase(test.TestCase):
self.backend)
@ddt.ddt
class NetAppCDOTDataMotionSessionTestCase(test.TestCase):
def setUp(self):
@ -511,3 +512,42 @@ class NetAppCDOTDataMotionSessionTestCase(test.TestCase):
self.mock_dest_client.resume_snapmirror.assert_called_once_with(
self.source_vserver, self.fake_src_vol_name,
self.dest_vserver, self.fake_dest_vol_name)
@ddt.data((None, exception.StorageCommunicationException),
(exception.StorageCommunicationException, None))
@ddt.unpack
def test_remove_qos_on_old_active_replica_unreachable_backend(self,
side_eff_1,
side_eff_2):
mock_source_client = mock.Mock()
self.mock_object(data_motion, 'get_client_for_backend',
mock.Mock(return_value=mock_source_client))
self.mock_object(
mock_source_client, 'set_qos_policy_group_for_volume',
mock.Mock(side_effect=side_eff_1))
self.mock_object(
mock_source_client, 'mark_qos_policy_group_for_deletion',
mock.Mock(side_effect=side_eff_2))
self.mock_object(data_motion.LOG, 'exception')
retval = self.dm_session.remove_qos_on_old_active_replica(
self.fake_src_share)
self.assertIsNone(retval)
(mock_source_client.set_qos_policy_group_for_volume
.assert_called_once_with(self.fake_src_vol_name, 'none'))
data_motion.LOG.exception.assert_called_once()
def test_remove_qos_on_old_active_replica(self):
mock_source_client = mock.Mock()
self.mock_object(data_motion, 'get_client_for_backend',
mock.Mock(return_value=mock_source_client))
self.mock_object(data_motion.LOG, 'exception')
retval = self.dm_session.remove_qos_on_old_active_replica(
self.fake_src_share)
self.assertIsNone(retval)
(mock_source_client.set_qos_policy_group_for_volume
.assert_called_once_with(self.fake_src_vol_name, 'none'))
data_motion.LOG.exception.assert_not_called()

442
manila/tests/share/drivers/netapp/dataontap/cluster_mode/test_lib_base.py
View File

@ -610,6 +610,7 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
share_server=fake.SHARE_SERVER)
mock_allocate_container.assert_called_once_with(fake.SHARE,
fake.VSERVER1,
vserver_client)
mock_create_export.assert_called_once_with(fake.SHARE,
fake.SHARE_SERVER,
@ -641,6 +642,7 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
mock_allocate_container_from_snapshot.assert_called_once_with(
fake.SHARE,
fake.SNAPSHOT,
fake.VSERVER1,
vserver_client)
mock_create_export.assert_called_once_with(fake.SHARE,
fake.SHARE_SERVER,
@ -653,14 +655,18 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
return_value=fake.SHARE_NAME))
self.mock_object(share_utils, 'extract_host', mock.Mock(
return_value=fake.POOL_NAME))
self.mock_object(
mock_get_provisioning_opts = self.mock_object(
self.library, '_get_provisioning_options_for_share',
mock.Mock(return_value=copy.deepcopy(fake.PROVISIONING_OPTIONS)))
vserver_client = mock.Mock()
self.library._allocate_container(fake.EXTRA_SPEC_SHARE,
fake.VSERVER1,
vserver_client)
mock_get_provisioning_opts.assert_called_once_with(
fake.EXTRA_SPEC_SHARE, fake.VSERVER1, replica=False)
vserver_client.create_volume.assert_called_once_with(
fake.POOL_NAME, fake.SHARE_NAME, fake.SHARE['size'],
thin_provisioned=True, snapshot_policy='default',
@ -680,14 +686,17 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
return_value=fake.SHARE_NAME))
self.mock_object(share_utils, 'extract_host', mock.Mock(
return_value=fake.POOL_NAME))
self.mock_object(
mock_get_provisioning_opts = self.mock_object(
self.library, '_get_provisioning_options_for_share',
mock.Mock(return_value=copy.deepcopy(fake.PROVISIONING_OPTIONS)))
vserver_client = mock.Mock()
self.library._allocate_container(fake.EXTRA_SPEC_SHARE,
self.library._allocate_container(fake.EXTRA_SPEC_SHARE, fake.VSERVER1,
vserver_client, replica=True)
mock_get_provisioning_opts.assert_called_once_with(
fake.EXTRA_SPEC_SHARE, fake.VSERVER1, replica=True)
vserver_client.create_volume.assert_called_once_with(
fake.POOL_NAME, fake.SHARE_NAME, fake.SHARE['size'],
thin_provisioned=True, snapshot_policy='default',
@ -706,7 +715,7 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
self.assertRaises(exception.InvalidHost,
self.library._allocate_container, fake.SHARE,
vserver_client)
fake.VSERVER1, vserver_client)
self.library._get_backend_share_name.assert_called_once_with(
fake.SHARE['id'])
@ -775,31 +784,53 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
fake.EXTRA_SPEC_SHARE, fake.INVALID_EXTRA_SPEC_COMBO,
list(self.library.BOOLEAN_QUALIFIED_EXTRA_SPECS_MAP))
def test_get_provisioning_options_for_share(self):
@ddt.data({'extra_specs': fake.EXTRA_SPEC, 'is_replica': False},
{'extra_specs': fake.EXTRA_SPEC_WITH_QOS, 'is_replica': True},
{'extra_specs': fake.EXTRA_SPEC, 'is_replica': False},
{'extra_specs': fake.EXTRA_SPEC_WITH_QOS, 'is_replica': True})
@ddt.unpack
def test_get_provisioning_options_for_share(self, extra_specs, is_replica):
qos = True if fake.QOS_EXTRA_SPEC in extra_specs else False
mock_get_extra_specs_from_share = self.mock_object(
share_types, 'get_extra_specs_from_share',
mock.Mock(return_value=fake.EXTRA_SPEC))
mock.Mock(return_value=extra_specs))
mock_remap_standard_boolean_extra_specs = self.mock_object(
self.library, '_remap_standard_boolean_extra_specs',
mock.Mock(return_value=fake.EXTRA_SPEC))
mock.Mock(return_value=extra_specs))
mock_check_extra_specs_validity = self.mock_object(
self.library, '_check_extra_specs_validity')
mock_get_provisioning_options = self.mock_object(
self.library, '_get_provisioning_options',
mock.Mock(return_value=fake.PROVISIONING_OPTIONS))
mock_get_normalized_qos_specs = self.mock_object(
self.library, '_get_normalized_qos_specs',
mock.Mock(return_value={fake.QOS_NORMALIZED_SPEC: 3000}))
mock_create_qos_policy_group = self.mock_object(
self.library, '_create_qos_policy_group', mock.Mock(
return_value=fake.QOS_POLICY_GROUP_NAME))
result = self.library._get_provisioning_options_for_share(
fake.EXTRA_SPEC_SHARE)
fake.EXTRA_SPEC_SHARE, fake.VSERVER1, replica=is_replica)
self.assertEqual(fake.PROVISIONING_OPTIONS, result)
if qos and is_replica:
expected_provisioning_opts = fake.PROVISIONING_OPTIONS
self.assertFalse(mock_create_qos_policy_group.called)
else:
expected_provisioning_opts = fake.PROVISIONING_OPTIONS_WITH_QOS
mock_create_qos_policy_group.assert_called_once_with(
fake.EXTRA_SPEC_SHARE, fake.VSERVER1,
{fake.QOS_NORMALIZED_SPEC: 3000})
self.assertEqual(expected_provisioning_opts, result)
mock_get_extra_specs_from_share.assert_called_once_with(
fake.EXTRA_SPEC_SHARE)
mock_remap_standard_boolean_extra_specs.assert_called_once_with(
fake.EXTRA_SPEC)
extra_specs)
mock_check_extra_specs_validity.assert_called_once_with(
fake.EXTRA_SPEC_SHARE, fake.EXTRA_SPEC)
mock_get_provisioning_options.assert_called_once_with(fake.EXTRA_SPEC)
fake.EXTRA_SPEC_SHARE, extra_specs)
mock_get_provisioning_options.assert_called_once_with(extra_specs)
mock_get_normalized_qos_specs.assert_called_once_with(extra_specs)
def test_get_provisioning_options(self):
result = self.library._get_provisioning_options(fake.EXTRA_SPEC)
@ -879,6 +910,66 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
self.assertEqual(fake.PROVISIONING_OPTIONS_STRING_DEFAULT, result)
@ddt.data({}, {'foo': 'bar'}, {'netapp:maxiops': '3000'},
{'qos': True, 'netapp:absiops': '3000'},
{'qos': True, 'netapp:maxiops:': '3000'})
def test_get_normalized_qos_specs_no_qos_specs(self, extra_specs):
if 'qos' in extra_specs:
self.assertRaises(exception.NetAppException,
self.library._get_normalized_qos_specs,
extra_specs)
else:
self.assertDictMatch(
{}, self.library._get_normalized_qos_specs(extra_specs))
@ddt.data({'qos': True, 'netapp:maxiops': '3000', 'netapp:maxbps': '9000'},
{'qos': True, 'netapp:maxiopspergib': '1000',
'netapp:maxiops': '1000'})
def test_get_normalized_qos_specs_multiple_qos_specs(self, extra_specs):
self.assertRaises(exception.NetAppException,
self.library._get_normalized_qos_specs,
extra_specs)
@ddt.data({'qos': True, 'netapp:maxIOPS': '3000'},
{'qos': True, 'netapp:MAxBPs': '3000', 'clem': 'son'},
{'qos': True, 'netapp:maxbps': '3000', 'tig': 'ers'},
{'qos': True, 'netapp:MAXiopSPerGib': '3000', 'kin': 'gsof'},
{'qos': True, 'netapp:maxiopspergib': '3000', 'coll': 'ege'},
{'qos': True, 'netapp:maxBPSperGiB': '3000', 'foot': 'ball'})
def test_get_normalized_qos_specs(self, extra_specs):
expected_normalized_spec = {
key.lower().split('netapp:')[1]: value
for key, value in extra_specs.items() if 'netapp:' in key
}
qos_specs = self.library._get_normalized_qos_specs(extra_specs)
self.assertDictMatch(expected_normalized_spec, qos_specs)
self.assertEqual(1, len(qos_specs))
@ddt.data({'qos': {'maxiops': '3000'}, 'expected': '3000iops'},
{'qos': {'maxbps': '3000'}, 'expected': '3000B/s'},
{'qos': {'maxbpspergib': '3000'}, 'expected': '12000B/s'},
{'qos': {'maxiopspergib': '3000'}, 'expected': '12000iops'})
@ddt.unpack
def test_get_max_throughput(self, qos, expected):
throughput = self.library._get_max_throughput(4, qos)
self.assertEqual(expected, throughput)
def test_create_qos_policy_group(self):
mock_qos_policy_create = self.mock_object(
self.library._client, 'qos_policy_group_create')
self.library._create_qos_policy_group(
fake.SHARE, fake.VSERVER1, {'maxiops': '3000'})
expected_policy_name = 'qos_share_' + fake.SHARE['id'].replace(
'-', '_')
mock_qos_policy_create.assert_called_once_with(
expected_policy_name, fake.VSERVER1, max_throughput='3000iops')
def test_check_if_max_files_is_valid_with_negative_integer(self):
self.assertRaises(exception.NetAppException,
self.library._check_if_max_files_is_valid,
@ -898,6 +989,7 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
self.assertRaises(exception.InvalidHost,
self.library._allocate_container,
fake_share,
fake.VSERVER1,
vserver_client)
def test_check_aggregate_extra_specs_validity(self):
@ -923,9 +1015,10 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
@ddt.data(None, 'fake_location')
def test_allocate_container_from_snapshot(self, provider_location):
self.mock_object(
mock_get_provisioning_opts = self.mock_object(
self.library, '_get_provisioning_options_for_share',
mock.Mock(return_value=copy.deepcopy(fake.PROVISIONING_OPTIONS)))
vserver = fake.VSERVER1
vserver_client = mock.Mock()
fake_snapshot = copy.deepcopy(fake.SNAPSHOT)
@ -933,6 +1026,7 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
self.library._allocate_container_from_snapshot(fake.SHARE,
fake_snapshot,
vserver,
vserver_client)
share_name = self.library._get_backend_share_name(fake.SHARE['id'])
@ -940,6 +1034,8 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
fake.SNAPSHOT['share_id'])
parent_snapshot_name = self.library._get_backend_snapshot_name(
fake.SNAPSHOT['id']) if not provider_location else 'fake_location'
mock_get_provisioning_opts.assert_called_once_with(
fake.SHARE, fake.VSERVER1)
vserver_client.create_volume_clone.assert_called_once_with(
share_name, parent_share_name, parent_snapshot_name,
thin_provisioned=True, snapshot_policy='default',
@ -983,10 +1079,14 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
share_server=fake.SHARE_SERVER)
share_name = self.library._get_backend_share_name(fake.SHARE['id'])
qos_policy_name = self.library._get_backend_qos_policy_group_name(
fake.SHARE['id'])
mock_share_exists.assert_called_once_with(share_name, vserver_client)
mock_remove_export.assert_called_once_with(fake.SHARE, vserver_client)
mock_deallocate_container.assert_called_once_with(share_name,
vserver_client)
(self.library._client.mark_qos_policy_group_for_deletion
.assert_called_once_with(qos_policy_name))
self.assertEqual(0, lib_base.LOG.info.call_count)
@ddt.data(exception.InvalidInput(reason='fake_reason'),
@ -1011,6 +1111,8 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
self.assertFalse(mock_share_exists.called)
self.assertFalse(mock_remove_export.called)
self.assertFalse(mock_deallocate_container.called)
self.assertFalse(
self.library._client.mark_qos_policy_group_for_deletion.called)
self.assertEqual(1, lib_base.LOG.warning.call_count)
def test_delete_share_not_found(self):
@ -1035,6 +1137,8 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
mock_share_exists.assert_called_once_with(share_name, vserver_client)
self.assertFalse(mock_remove_export.called)
self.assertFalse(mock_deallocate_container.called)
self.assertFalse(
self.library._client.mark_qos_policy_group_for_deletion.called)
self.assertEqual(1, lib_base.LOG.info.call_count)
def test_deallocate_container(self):
@ -1403,6 +1507,7 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
'export_locations': fake.NFS_EXPORTS
}
mock_manage_container.assert_called_once_with(fake.SHARE,
fake.VSERVER1,
vserver_client)
mock_create_export.assert_called_once_with(fake.SHARE,
None,
@ -1416,9 +1521,15 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
self.assertIsNone(result)
def test_manage_container(self):
@ddt.data(True, False)
def test_manage_container_with_qos(self, qos):
vserver_client = mock.Mock()
qos_policy_group_name = fake.QOS_POLICY_GROUP_NAME if qos else None
extra_specs = fake.EXTRA_SPEC_WITH_QOS if qos else fake.EXTRA_SPEC
provisioning_opts = self.library._get_provisioning_options(extra_specs)
if qos:
provisioning_opts['qos_policy_group'] = fake.QOS_POLICY_GROUP_NAME
share_to_manage = copy.deepcopy(fake.SHARE)
share_to_manage['export_location'] = fake.EXPORT_LOCATION
@ -1438,15 +1549,19 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
'_validate_volume_for_manage')
self.mock_object(share_types,
'get_extra_specs_from_share',
mock.Mock(return_value=fake.EXTRA_SPEC))
mock.Mock(return_value=extra_specs))
mock_check_extra_specs_validity = self.mock_object(
self.library,
'_check_extra_specs_validity')
mock_check_aggregate_extra_specs_validity = self.mock_object(
self.library,
'_check_aggregate_extra_specs_validity')
mock_modify_or_create_qos_policy = self.mock_object(
self.library, '_modify_or_create_qos_for_existing_share',
mock.Mock(return_value=qos_policy_group_name))
result = self.library._manage_container(share_to_manage,
fake.VSERVER1,
vserver_client)
mock_get_volume_to_manage.assert_called_once_with(
@ -1454,18 +1569,19 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
mock_validate_volume_for_manage.assert_called_once_with(
fake.FLEXVOL_TO_MANAGE, vserver_client)
mock_check_extra_specs_validity.assert_called_once_with(
share_to_manage, fake.EXTRA_SPEC)
share_to_manage, extra_specs)
mock_check_aggregate_extra_specs_validity.assert_called_once_with(
fake.POOL_NAME, fake.EXTRA_SPEC)
fake.POOL_NAME, extra_specs)
vserver_client.unmount_volume.assert_called_once_with(
fake.FLEXVOL_NAME)
vserver_client.set_volume_name.assert_called_once_with(
fake.FLEXVOL_NAME, fake.SHARE_NAME)
vserver_client.mount_volume.assert_called_once_with(
fake.SHARE_NAME)
vserver_client.manage_volume.assert_called_once_with(
fake.POOL_NAME, fake.SHARE_NAME,
**self.library._get_provisioning_options(fake.EXTRA_SPEC))
vserver_client.modify_volume.assert_called_once_with(
fake.POOL_NAME, fake.SHARE_NAME, **provisioning_opts)
mock_modify_or_create_qos_policy.assert_called_once_with(
share_to_manage, extra_specs, fake.VSERVER1, vserver_client)
original_data = {
'original_name': fake.FLEXVOL_TO_MANAGE['name'],
@ -1494,6 +1610,7 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
self.assertRaises(exception.ManageInvalidShare,
self.library._manage_container,
share_to_manage,
fake.VSERVER1,
vserver_client)
def test_manage_container_not_found(self):
@ -1516,6 +1633,7 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
self.assertRaises(exception.ManageInvalidShare,
self.library._manage_container,
share_to_manage,
fake.VSERVER1,
vserver_client)
def test_manage_container_invalid_extra_specs(self):
@ -1545,6 +1663,7 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
self.assertRaises(exception.ManageExistingShareTypeMismatch,
self.library._manage_container,
share_to_manage,
fake.VSERVER1,
vserver_client)
def test_validate_volume_for_manage(self):
@ -1782,10 +1901,12 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
mock_allocate_container_from_snapshot.assert_has_calls([
mock.call(fake.COLLATED_CGSNAPSHOT_INFO[0]['share'],
fake.COLLATED_CGSNAPSHOT_INFO[0]['snapshot'],
fake.VSERVER1,
vserver_client,
mock.ANY),
mock.call(fake.COLLATED_CGSNAPSHOT_INFO[1]['share'],
fake.COLLATED_CGSNAPSHOT_INFO[1]['snapshot'],
fake.VSERVER1,
vserver_client,
mock.ANY),
])
@ -2034,6 +2155,68 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
mock_get_vserver.assert_called_once_with(
share_server=fake.SHARE_SERVER)
def test_adjust_qos_policy_with_volume_resize_no_cluster_creds(self):
self.library._have_cluster_creds = False
self.mock_object(share_types, 'get_extra_specs_from_share')
retval = self.library._adjust_qos_policy_with_volume_resize(
fake.SHARE, 10, mock.Mock())
self.assertIsNone(retval)
share_types.get_extra_specs_from_share.assert_not_called()
def test_adjust_qos_policy_with_volume_resize_no_qos_on_share(self):
self.library._have_cluster_creds = True
self.mock_object(share_types, 'get_extra_specs_from_share')
vserver_client = mock.Mock()
self.mock_object(vserver_client, 'get_volume',
mock.Mock(return_value=fake.FLEXVOL_WITHOUT_QOS))
retval = self.library._adjust_qos_policy_with_volume_resize(
fake.SHARE, 10, vserver_client)
self.assertIsNone(retval)
share_types.get_extra_specs_from_share.assert_not_called()
def test_adjust_qos_policy_with_volume_resize_no_size_dependent_qos(self):
self.library._have_cluster_creds = True
self.mock_object(share_types, 'get_extra_specs_from_share',
mock.Mock(return_value=fake.EXTRA_SPEC_WITH_QOS))
vserver_client = mock.Mock()
self.mock_object(vserver_client, 'get_volume',
mock.Mock(return_value=fake.FLEXVOL_WITH_QOS))
self.mock_object(self.library, '_get_max_throughput')
self.mock_object(self.library._client, 'qos_policy_group_modify')
retval = self.library._adjust_qos_policy_with_volume_resize(
fake.SHARE, 10, vserver_client)
self.assertIsNone(retval)
share_types.get_extra_specs_from_share.assert_called_once_with(
fake.SHARE)
self.library._get_max_throughput.assert_not_called()
self.library._client.qos_policy_group_modify.assert_not_called()
def test_adjust_qos_policy_with_volume_resize(self):
self.library._have_cluster_creds = True
self.mock_object(
share_types, 'get_extra_specs_from_share',
mock.Mock(return_value=fake.EXTRA_SPEC_WITH_SIZE_DEPENDENT_QOS))
vserver_client = mock.Mock()
self.mock_object(vserver_client, 'get_volume',
mock.Mock(return_value=fake.FLEXVOL_WITH_QOS))
self.mock_object(self.library._client, 'qos_policy_group_modify')
retval = self.library._adjust_qos_policy_with_volume_resize(
fake.SHARE, 10, vserver_client)
expected_max_throughput = '10000B/s'
self.assertIsNone(retval)
share_types.get_extra_specs_from_share.assert_called_once_with(
fake.SHARE)
self.library._client.qos_policy_group_modify.assert_called_once_with(
fake.QOS_POLICY_GROUP_NAME, expected_max_throughput)
def test_extend_share(self):
vserver_client = mock.Mock()
@ -2041,6 +2224,9 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
'_get_vserver',
mock.Mock(return_value=(fake.VSERVER1,
vserver_client)))
mock_adjust_qos_policy = self.mock_object(
self.library, '_adjust_qos_policy_with_volume_resize')
mock_set_volume_size = self.mock_object(vserver_client,
'set_volume_size')
new_size = fake.SHARE['size'] * 2
@ -2048,6 +2234,8 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
self.library.extend_share(fake.SHARE, new_size)
mock_set_volume_size.assert_called_once_with(fake.SHARE_NAME, new_size)
mock_adjust_qos_policy.assert_called_once_with(
fake.SHARE, new_size, vserver_client)
def test_shrink_share(self):
@ -2056,6 +2244,8 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
'_get_vserver',
mock.Mock(return_value=(fake.VSERVER1,
vserver_client)))
mock_adjust_qos_policy = self.mock_object(
self.library, '_adjust_qos_policy_with_volume_resize')
mock_set_volume_size = self.mock_object(vserver_client,
'set_volume_size')
new_size = fake.SHARE['size'] - 1
@ -2063,6 +2253,8 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
self.library.shrink_share(fake.SHARE, new_size)
mock_set_volume_size.assert_called_once_with(fake.SHARE_NAME, new_size)
mock_adjust_qos_policy.assert_called_once_with(
fake.SHARE, new_size, vserver_client)
def test_update_access(self):
@ -2774,6 +2966,7 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
mock.Mock(return_value=mock.Mock()))
self.mock_object(self.library, '_create_export',
mock.Mock(return_value='fake_export_location'))
self.mock_object(self.library, '_handle_qos_on_replication_change')
replicas = self.library.promote_replica(
None, [self.fake_replica, self.fake_replica_2],
@ -2783,7 +2976,6 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
self.fake_replica, self.fake_replica, self.fake_replica_2,
mock.ANY
)
self.assertEqual(2, len(replicas))
actual_replica_1 = list(filter(
lambda x: x['id'] == self.fake_replica['id'], replicas))[0]
@ -2797,6 +2989,7 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
actual_replica_2['export_locations'])
self.assertEqual(constants.STATUS_ACTIVE,
actual_replica_2['access_rules_status'])
self.library._handle_qos_on_replication_change.assert_called_once()
def test_promote_replica_destination_unreachable(self):
self.mock_object(self.library,
@ -2806,6 +2999,8 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
self.mock_object(self.library,
'_get_helper',
mock.Mock(return_value=mock.Mock()))
self.mock_object(self.library, '_handle_qos_on_replication_change')
self.mock_object(self.library, '_create_export',
mock.Mock(return_value='fake_export_location'))
self.mock_object(
@ -2822,6 +3017,8 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
actual_replica['replica_state'])
self.assertEqual(constants.STATUS_ERROR,
actual_replica['status'])
self.assertFalse(
self.library._handle_qos_on_replication_change.called)
def test_promote_replica_more_than_two_replicas(self):
fake_replica_3 = copy.deepcopy(self.fake_replica_2)
@ -2831,6 +3028,7 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
'_get_vserver',
mock.Mock(return_value=(fake.VSERVER1,
mock.Mock())))
self.mock_object(self.library, '_handle_qos_on_replication_change')
self.mock_object(self.library,
'_get_helper',
mock.Mock(return_value=mock.Mock()))
@ -2864,12 +3062,14 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
lambda x: x['id'] == fake_replica_3['id'], replicas))[0]
self.assertEqual(constants.REPLICA_STATE_OUT_OF_SYNC,
actual_replica_3['replica_state'])
self.library._handle_qos_on_replication_change.assert_called_once()
def test_promote_replica_with_access_rules(self):
self.mock_object(self.library,
'_get_vserver',
mock.Mock(return_value=(fake.VSERVER1,
mock.Mock())))
self.mock_object(self.library, '_handle_qos_on_replication_change')
mock_helper = mock.Mock()
self.mock_object(self.library,
'_get_helper',
@ -2891,6 +3091,112 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
mock_helper.update_access.assert_called_once_with(self.fake_replica_2,
share_name,
[fake.SHARE_ACCESS])
self.library._handle_qos_on_replication_change.assert_called_once()
@ddt.data({'extra_specs': {'netapp:snapshot_policy': 'none'},
'have_cluster_creds': True},
# Test Case 2 isn't possible input
{'extra_specs': {'qos': True, 'netapp:maxiops': '3000'},
'have_cluster_creds': False})
@ddt.unpack
def test_handle_qos_on_replication_change_nothing_to_handle(
self, extra_specs, have_cluster_creds):
self.library._have_cluster_creds = have_cluster_creds
self.mock_object(lib_base.LOG, 'exception')
self.mock_object(lib_base.LOG, 'info')
self.mock_object(share_types, 'get_extra_specs_from_share',
mock.Mock(return_value=extra_specs))
retval = self.library._handle_qos_on_replication_change(
self.mock_dm_session, self.fake_replica_2, self.fake_replica,
share_server=fake.SHARE_SERVER)
self.assertIsNone(retval)
lib_base.LOG.exception.assert_not_called()
lib_base.LOG.info.assert_not_called()
def test_handle_qos_on_replication_change_exception(self):
self.library._have_cluster_creds = True
extra_specs = {'qos': True, fake.QOS_EXTRA_SPEC: '3000'}
vserver_client = mock.Mock()
self.mock_object(lib_base.LOG, 'exception')
self.mock_object(lib_base.LOG, 'info')
self.mock_object(share_types, 'get_extra_specs_from_share',
mock.Mock(return_value=extra_specs))
self.mock_object(self.library, '_get_vserver', mock.Mock(
return_value=(fake.VSERVER1, vserver_client)))
self.mock_object(self.library._client, 'qos_policy_group_exists',
mock.Mock(return_value=True))
self.mock_object(self.library._client, 'qos_policy_group_modify',
mock.Mock(side_effect=netapp_api.NaApiError))
retval = self.library._handle_qos_on_replication_change(
self.mock_dm_session, self.fake_replica_2, self.fake_replica,
share_server=fake.SHARE_SERVER)
self.assertIsNone(retval)
(self.mock_dm_session.remove_qos_on_old_active_replica
.assert_called_once_with(self.fake_replica))
lib_base.LOG.exception.assert_called_once()
lib_base.LOG.info.assert_not_called()
vserver_client.set_qos_policy_group_for_volume.assert_not_called()
def test_handle_qos_on_replication_change_modify_existing_policy(self):
self.library._have_cluster_creds = True
extra_specs = {'qos': True, fake.QOS_EXTRA_SPEC: '3000'}
vserver_client = mock.Mock()
volume_name_on_backend = self.library._get_backend_share_name(
self.fake_replica_2['id'])
self.mock_object(lib_base.LOG, 'exception')
self.mock_object(lib_base.LOG, 'info')
self.mock_object(share_types, 'get_extra_specs_from_share',
mock.Mock(return_value=extra_specs))
self.mock_object(self.library, '_get_vserver', mock.Mock(
return_value=(fake.VSERVER1, vserver_client)))
self.mock_object(self.library._client, 'qos_policy_group_exists',
mock.Mock(return_value=True))
self.mock_object(self.library._client, 'qos_policy_group_modify')
self.mock_object(self.library, '_create_qos_policy_group')
retval = self.library._handle_qos_on_replication_change(
self.mock_dm_session, self.fake_replica_2, self.fake_replica,
share_server=fake.SHARE_SERVER)
self.assertIsNone(retval)
self.library._client.qos_policy_group_modify.assert_called_once_with(
'qos_' + volume_name_on_backend, '3000iops')
vserver_client.set_qos_policy_group_for_volume.assert_called_once_with(
volume_name_on_backend, 'qos_' + volume_name_on_backend)
self.library._create_qos_policy_group.assert_not_called()
lib_base.LOG.exception.assert_not_called()
lib_base.LOG.info.assert_called_once()
def test_handle_qos_on_replication_change_create_new_policy(self):
self.library._have_cluster_creds = True
extra_specs = {'qos': True, fake.QOS_EXTRA_SPEC: '3000'}
vserver_client = mock.Mock()
self.mock_object(lib_base.LOG, 'exception')
self.mock_object(lib_base.LOG, 'info')
self.mock_object(share_types, 'get_extra_specs_from_share',
mock.Mock(return_value=extra_specs))
self.mock_object(self.library, '_get_vserver', mock.Mock(
return_value=(fake.VSERVER1, vserver_client)))
self.mock_object(self.library._client, 'qos_policy_group_exists',
mock.Mock(return_value=False))
self.mock_object(self.library._client, 'qos_policy_group_modify')
self.mock_object(self.library, '_create_qos_policy_group')
retval = self.library._handle_qos_on_replication_change(
self.mock_dm_session, self.fake_replica_2, self.fake_replica,
share_server=fake.SHARE_SERVER)
self.assertIsNone(retval)
self.library._create_qos_policy_group.assert_called_once_with(
self.fake_replica_2, fake.VSERVER1, {'maxiops': '3000'})
self.library._client.qos_policy_group_modify.assert_not_called()
lib_base.LOG.exception.assert_not_called()
lib_base.LOG.info.assert_called_once()
def test_convert_destination_replica_to_independent(self):
self.mock_object(self.library,
@ -2958,6 +3264,7 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
'_get_vserver',
mock.Mock(return_value=(fake.VSERVER1,
mock.Mock())))
self.mock_object(self.library, '_handle_qos_on_replication_change')
self.mock_object(self.library,
'_get_helper',
mock.Mock(return_value=fake_helper))
@ -2986,6 +3293,7 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
actual_replica_2['export_locations'])
self.assertEqual(constants.SHARE_INSTANCE_RULES_SYNCING,
actual_replica_2['access_rules_status'])
self.library._handle_qos_on_replication_change.assert_called_once()
def test_convert_destination_replica_to_independent_with_access_rules(
self):
@ -4353,9 +4661,13 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
mock.Mock(side_effect=vol_move_side_effects))
self.mock_object(share_types, 'get_extra_specs_from_share',
mock.Mock(return_value=fake.EXTRA_SPEC))
self.mock_object(self.library, '_get_provisioning_options',
mock.Mock(return_value=fake.PROVISIONING_OPTIONS))
self.mock_object(vserver_client, 'manage_volume')
self.mock_object(
self.library, '_get_provisioning_options',
mock.Mock(return_value=fake.PROVISIONING_OPTIONS_WITH_QOS))
self.mock_object(
self.library, '_modify_or_create_qos_for_existing_share',
mock.Mock(return_value=fake.QOS_POLICY_GROUP_NAME))
self.mock_object(vserver_client, 'modify_volume')
src_share = fake_share.fake_share_instance(id='source-share-instance')
dest_share = fake_share.fake_share_instance(id='dest-share-instance')
@ -4378,8 +4690,8 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
self.library._create_export.assert_called_once_with(
dest_share, fake.SHARE_SERVER, fake.VSERVER1, vserver_client,
clear_current_export_policy=False)
vserver_client.manage_volume.assert_called_once_with(
dest_aggr, 'new_share_name', **fake.PROVISIONING_OPTIONS)
vserver_client.modify_volume.assert_called_once_with(
dest_aggr, 'new_share_name', **fake.PROVISIONING_OPTIONS_WITH_QOS)
mock_info_log.assert_called_once()
if phase != 'completed':
self.assertEqual(2, mock_warning_log.call_count)
@ -4389,3 +4701,81 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
self.assertFalse(mock_warning_log.called)
mock_debug_log.assert_called_once()
mock_move_status_check.assert_called_once()
def test_modify_or_create_qos_for_existing_share_no_qos_extra_specs(self):
vserver_client = mock.Mock()
self.mock_object(self.library, '_get_backend_qos_policy_group_name')
self.mock_object(vserver_client, 'get_volume')
self.mock_object(self.library, '_create_qos_policy_group')
retval = self.library._modify_or_create_qos_for_existing_share(
fake.SHARE, fake.EXTRA_SPEC, fake.VSERVER1, vserver_client)
self.assertIsNone(retval)
self.library._get_backend_qos_policy_group_name.assert_not_called()
vserver_client.get_volume.assert_not_called()
self.library._create_qos_policy_group.assert_not_called()
def test_modify_or_create_qos_for_existing_share_no_existing_qos(self):
vserver_client = mock.Mock()
self.mock_object(self.library, '_get_backend_qos_policy_group_name')
self.mock_object(vserver_client, 'get_volume',
mock.Mock(return_value=fake.FLEXVOL_WITHOUT_QOS))
self.mock_object(self.library, '_create_qos_policy_group')
self.mock_object(self.library._client, 'qos_policy_group_modify')
qos_policy_name = self.library._get_backend_qos_policy_group_name(
fake.SHARE['id'])
retval = self.library._modify_or_create_qos_for_existing_share(
fake.SHARE, fake.EXTRA_SPEC_WITH_QOS, fake.VSERVER1,
vserver_client)
share_obj = {
'size': 2,
'id': fake.SHARE['id'],
}
self.assertEqual(qos_policy_name, retval)
self.library._client.qos_policy_group_modify.assert_not_called()
self.library._create_qos_policy_group.assert_called_once_with(
share_obj, fake.VSERVER1, {'maxiops': '3000'})
@ddt.data(utils.annotated('volume_has_shared_qos_policy', (2, )),
utils.annotated('volume_has_nonshared_qos_policy', (1, )))
def test_modify_or_create_qos_for_existing_share(self, num_workloads):
vserver_client = mock.Mock()
num_workloads = num_workloads[0]
qos_policy = copy.deepcopy(fake.QOS_POLICY_GROUP)
qos_policy['num-workloads'] = num_workloads
extra_specs = fake.EXTRA_SPEC_WITH_QOS
self.mock_object(vserver_client, 'get_volume',
mock.Mock(return_value=fake.FLEXVOL_WITH_QOS))
self.mock_object(self.library._client, 'qos_policy_group_get',
mock.Mock(return_value=qos_policy))
mock_qos_policy_modify = self.mock_object(
self.library._client, 'qos_policy_group_modify')
mock_qos_policy_rename = self.mock_object(
self.library._client, 'qos_policy_group_rename')
mock_create_qos_policy = self.mock_object(
self.library, '_create_qos_policy_group')
new_qos_policy_name = self.library._get_backend_qos_policy_group_name(
fake.SHARE['id'])
retval = self.library._modify_or_create_qos_for_existing_share(
fake.SHARE, extra_specs, fake.VSERVER1, vserver_client)
self.assertEqual(new_qos_policy_name, retval)
if num_workloads == 1:
mock_create_qos_policy.assert_not_called()
mock_qos_policy_modify.assert_called_once_with(
fake.QOS_POLICY_GROUP_NAME, '3000iops')
mock_qos_policy_rename.assert_called_once_with(
fake.QOS_POLICY_GROUP_NAME, new_qos_policy_name)
else:
share_obj = {
'size': 2,
'id': fake.SHARE['id'],
}
mock_create_qos_policy.assert_called_once_with(
share_obj, fake.VSERVER1, {'maxiops': '3000'})
self.library._client.qos_policy_group_modify.assert_not_called()
self.library._client.qos_policy_group_rename.assert_not_called()

8
manila/tests/share/drivers/netapp/dataontap/cluster_mode/test_lib_single_svm.py
View File

@ -165,8 +165,9 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
}
self.assertEqual(expected, result)
def test_handle_housekeeping_tasks(self):
@ddt.data(True, False)
def test_handle_housekeeping_tasks_with_cluster_creds(self, have_creds):
self.library._have_cluster_creds = have_creds
mock_vserver_client = mock.Mock()
self.mock_object(self.library,
'_get_api_client',
@ -179,6 +180,9 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
self.assertTrue(
mock_vserver_client.prune_deleted_nfs_export_policies.called)
self.assertTrue(mock_vserver_client.prune_deleted_snapshots.called)
self.assertIs(
have_creds,
mock_vserver_client.remove_unused_qos_policy_groups.called)
self.assertTrue(mock_super.called)
@ddt.data(True, False)

39
manila/tests/share/drivers/netapp/dataontap/fakes.py
View File

@ -71,6 +71,10 @@ MTU = 1234
DEFAULT_MTU = 1500
MANILA_HOST_NAME = '%(host)s@%(backend)s#%(pool)s' % {
'host': HOST_NAME, 'backend': BACKEND_NAME, 'pool': POOL_NAME}
QOS_EXTRA_SPEC = 'netapp:maxiops'
QOS_SIZE_DEPENDENT_EXTRA_SPEC = 'netapp:maxbpspergib'
QOS_NORMALIZED_SPEC = 'maxiops'
QOS_POLICY_GROUP_NAME = 'fake_qos_policy_group_name'
CLIENT_KWARGS = {
'username': 'admin',
@ -97,6 +101,7 @@ SHARE = {
},
'replica_state': constants.REPLICA_STATE_ACTIVE,
'status': constants.STATUS_AVAILABLE,
'share_server': None,
}
FLEXVOL_TO_MANAGE = {
@ -105,7 +110,19 @@ FLEXVOL_TO_MANAGE = {
'name': FLEXVOL_NAME,
'type': 'rw',
'style': 'flex',
'size': '1610612736', # rounds down to 1 GB
'size': '1610612736', # rounds up to 2 GB
}
FLEXVOL_WITHOUT_QOS = copy.deepcopy(FLEXVOL_TO_MANAGE)
FLEXVOL_WITHOUT_QOS.update({'qos-policy-group-name': None})
FLEXVOL_WITH_QOS = copy.deepcopy(FLEXVOL_TO_MANAGE)
FLEXVOL_WITH_QOS.update({'qos-policy-group-name': QOS_POLICY_GROUP_NAME})
QOS_POLICY_GROUP = {
'policy-group': QOS_POLICY_GROUP_NAME,
'vserver': VSERVER1,
'max-throughput': '3000iops',
'num-workloads': 1,
}
EXTRA_SPEC = {
@ -120,6 +137,18 @@ EXTRA_SPEC = {
'netapp_raid_type': 'raid4',
}
EXTRA_SPEC_WITH_QOS = copy.deepcopy(EXTRA_SPEC)
EXTRA_SPEC_WITH_QOS.update({
'qos': True,
QOS_EXTRA_SPEC: '3000',
})
EXTRA_SPEC_WITH_SIZE_DEPENDENT_QOS = copy.deepcopy(EXTRA_SPEC)
EXTRA_SPEC_WITH_SIZE_DEPENDENT_QOS.update({
'qos': True,
QOS_SIZE_DEPENDENT_EXTRA_SPEC: '1000',
})
PROVISIONING_OPTIONS = {
'thin_provisioned': True,
'snapshot_policy': 'default',
@ -130,6 +159,10 @@ PROVISIONING_OPTIONS = {
'split': True,
}
PROVISIONING_OPTIONS_WITH_QOS = copy.deepcopy(PROVISIONING_OPTIONS)
PROVISIONING_OPTIONS_WITH_QOS.update(
{'qos_policy_group': QOS_POLICY_GROUP_NAME})
PROVISIONING_OPTIONS_BOOLEAN = {
'thin_provisioned': True,
'dedup_enabled': False,
@ -596,6 +629,7 @@ POOLS = [
'snapshot_support': True,
'create_share_from_snapshot_support': True,
'revert_to_snapshot_support': True,
'qos': True,
},
{
'pool_name': AGGREGATES[1],
@ -617,6 +651,7 @@ POOLS = [
'snapshot_support': True,
'create_share_from_snapshot_support': True,
'revert_to_snapshot_support': True,
'qos': True,
},
]
@ -638,6 +673,7 @@ POOLS_VSERVER_CREDS = [
'snapshot_support': True,
'create_share_from_snapshot_support': True,
'revert_to_snapshot_support': True,
'qos': False,
},
{
'pool_name': AGGREGATES[1],
@ -656,6 +692,7 @@ POOLS_VSERVER_CREDS = [
'snapshot_support': True,
'create_share_from_snapshot_support': True,
'revert_to_snapshot_support': True,
'qos': False,
},
]

12
releasenotes/notes/netapp-cdot-quality-of-service-limits-c1fe8601d00cb5a8.yaml Normal file
View File

@ -0,0 +1,12 @@
---
features:
- The NetApp driver now supports Quality of Service extra specs. To create
a share on ONTAP with qos support, set the 'qos' extra-spec in your
share type to True and use one of 'netapp:maxiops' and 'netapp:maxbps'
scoped extra-specs to set absolute limits. To set size based limits,
use scoped extra-specs 'netapp:maxiopspergib' or 'netapp:maxbpspergib'.
QoS policies on the back end are created exclusive to each manila share.
upgrade:
- A new configuration option 'netapp_qos_policy_group_name_template' has
been added to allow overriding the naming of QoS policies created by the
NetApp driver.