openstack/cinder
Code Issues Proposed changes

NetApp ONTAP: Add option to report storage provisioned capacity

Browse Source 
This patch adds option ´netapp_driver_reports_provisioned_capacity´,
for driver to calculate the storage provisioned capacity by queryng
volume sizes directly from the storage system and reporting as a
pool capability ´provisioned_capacity_gb´, instead of relying
on the default behavior of using scheduler and volume service
internal states as done by ´allocated_capacity_gb´.

Implements: blueprint ontap-report-provisioned-capacity
Change-Id: I97625de865a63b0cf725f61d9d2ea3c44740d9e7
This commit is contained in:
Fernando Ferraz 2021-05-11 14:22:43 +00:00
parent 1f96b386b3
commit 6906cd2a7b
10 changed files with 270 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
cinder/tests/unit/volume/drivers/netapp/dataontap/client/fakes.py
View File

@ -1389,6 +1389,34 @@ CLUSTER_PEER_POLICY_GET_RESPONSE = etree.XML("""
</results>
""")
FILE_SIZES_BY_DIR_GET_ITER_RESPONSE = etree.XML("""
<results status="passed">
<attributes-list>
<file-info>
<name>%(name)s</name>
<file-size>1024</file-size>
</file-info>
</attributes-list>
<num-records>1</num-records>
</results>
""" % {
'name': fake.VOLUME_NAME
})
LUN_SIZES_BY_VOLUME_GET_ITER_RESPONSE = etree.XML("""
<results status="passed">
<attributes-list>
<lun-info>
<path>%(path)s</path>
<size>1024</size>
</lun-info>
</attributes-list>
<num-records>1</num-records>
</results>
""" % {
'path': fake.VOLUME_PATH
})
VSERVER_PEER_GET_ITER_RESPONSE = etree.XML("""
<results status="passed">
<attributes-list>

92
cinder/tests/unit/volume/drivers/netapp/dataontap/client/test_client_cmode.py
View File

@ -3379,6 +3379,98 @@ class NetAppCmodeClientTestCase(test.TestCase):
self.client.connection.send_request.assert_has_calls([
mock.call('vserver-peer-accept', vserver_peer_accept_args)])
def test_get_file_sizes_by_dir(self):
api_response = netapp_api.NaElement(
fake_client.FILE_SIZES_BY_DIR_GET_ITER_RESPONSE)
self.mock_object(self.client,
'send_iter_request',
return_value=api_response)
result = self.client.get_file_sizes_by_dir(fake.NETAPP_VOLUME)
get_get_file_sizes_by_dir_get_iter_args = {
'path': '/vol/%s' % fake.NETAPP_VOLUME,
'query': {
'file-info': {
'file-type': 'file',
}
},
'desired-attributes': {
'file-info': {
'name': None,
'file-size': None
}
},
}
self.client.send_iter_request.assert_has_calls([
mock.call('file-list-directory-iter',
get_get_file_sizes_by_dir_get_iter_args,
max_page_length=100)])
expected = [{
'name': fake.VOLUME_NAME,
'file-size': float(1024)
}]
self.assertEqual(expected, result)
def test_get_file_sizes_by_dir_not_found(self):
api_response = netapp_api.NaElement(fake_client.NO_RECORDS_RESPONSE)
self.mock_object(self.client,
'send_iter_request',
return_value=api_response)
result = self.client.get_file_sizes_by_dir(fake.NETAPP_VOLUME)
self.assertEqual([], result)
self.assertTrue(self.client.send_iter_request.called)
def test_get_lun_sizes_by_volume(self):
api_response = netapp_api.NaElement(
fake_client.LUN_SIZES_BY_VOLUME_GET_ITER_RESPONSE)
self.mock_object(self.client,
'send_iter_request',
return_value=api_response)
result = self.client.get_lun_sizes_by_volume(fake.NETAPP_VOLUME)
get_lun_sizes_by_volume_get_iter_args = {
'query': {
'lun-info': {
'volume': fake.NETAPP_VOLUME,
}
},
'desired-attributes': {
'lun-info': {
'path': None,
'size': None
}
},
}
self.client.send_iter_request.assert_has_calls([
mock.call('lun-get-iter', get_lun_sizes_by_volume_get_iter_args,
max_page_length=100)])
expected = [{
'path': fake.VOLUME_PATH,
'size': float(1024)
}]
self.assertEqual(expected, result)
def test_get_lun_sizes_by_volume_not_found(self):
api_response = netapp_api.NaElement(fake_client.NO_RECORDS_RESPONSE)
self.mock_object(self.client,
'send_iter_request',
return_value=api_response)
result = self.client.get_lun_sizes_by_volume(fake.NETAPP_VOLUME)
self.assertEqual([], result)
self.assertTrue(self.client.send_iter_request.called)
def test_get_vserver_peers(self):
api_response = netapp_api.NaElement(

1
cinder/tests/unit/volume/drivers/netapp/dataontap/fakes.py
View File

@ -333,6 +333,7 @@ CLONE_DESTINATION = {
}
VOLUME_NAME = 'volume-fake_volume_id'
VOLUME_PATH = '/vol/%s/%s' % (NETAPP_VOLUME, VOLUME_NAME)
MOUNT_PATH = '168.10.16.11:/' + VOLUME_ID
SNAPSHOT_NAME = 'fake_snapshot_name'
SNAPSHOT_LUN_HANDLE = 'fake_snapshot_lun_handle'

25
cinder/tests/unit/volume/drivers/netapp/dataontap/test_block_cmode.py
View File

@ -353,12 +353,19 @@ class NetAppBlockStorageCmodeLibraryTestCase(test.TestCase):
fake.VOLUME_ID, fake.LUN_ID, fake.LUN_SIZE, fake.LUN_METADATA,
None, False)
@ddt.data({'replication_backends': [], 'cluster_credentials': False},
@ddt.data({'replication_backends': [], 'cluster_credentials': False,
'report_provisioned_capacity': False},
{'replication_backends': ['target_1', 'target_2'],
'cluster_credentials': True})
'cluster_credentials': True,
'report_provisioned_capacity': True})
@ddt.unpack
def test_get_pool_stats(self, replication_backends, cluster_credentials):
def test_get_pool_stats(self, replication_backends, cluster_credentials,
report_provisioned_capacity):
self.library.using_cluster_credentials = cluster_credentials
conf = self.library.configuration
conf.netapp_driver_reports_provisioned_capacity = (
report_provisioned_capacity)
ssc = {
'vola': {
'pool_name': 'vola',
@ -391,9 +398,19 @@ class NetAppBlockStorageCmodeLibraryTestCase(test.TestCase):
'size-total': 10737418240.0,
'size-available': 2147483648.0,
}
luns_provisioned_cap = [{
'path': '/vol/volume-ae947c9b-2392-4956-b373-aaac4521f37e',
'size': 5368709120.0 # 5GB
}, {
'path': '/vol/snapshot-527eedad-a431-483d-b0ca-18995dd65b66',
'size': 1073741824.0 # 1GB
}]
self.mock_object(self.zapi_client,
'get_flexvol_capacity',
return_value=mock_capacities)
self.mock_object(self.zapi_client,
'get_lun_sizes_by_volume',
return_value=luns_provisioned_cap)
self.mock_object(self.zapi_client,
'get_flexvol_dedupe_used_percent',
return_value=55.0)
@ -440,6 +457,8 @@ class NetAppBlockStorageCmodeLibraryTestCase(test.TestCase):
'online_extend_support': True,
'netapp_is_flexgroup': 'false',
}]
if report_provisioned_capacity:
expected[0].update({'provisioned_capacity_gb': 5.0})
expected[0].update({'QoS_support': cluster_credentials})
if not cluster_credentials:

26
cinder/tests/unit/volume/drivers/netapp/dataontap/test_nfs_cmode.py
View File

@ -157,16 +157,22 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase):
self.assertEqual(expected_stats, self.driver._stats)
@ddt.data({'replication_backends': [],
'cluster_credentials': False, 'is_fg': False},
'cluster_credentials': False, 'is_fg': False,
'report_provisioned_capacity': True},
{'replication_backends': ['target_1', 'target_2'],
'cluster_credentials': True, 'is_fg': False},
'cluster_credentials': True, 'is_fg': False,
'report_provisioned_capacity': False},
{'replication_backends': ['target_1', 'target_2'],
'cluster_credentials': True, 'is_fg': True}
'cluster_credentials': True, 'is_fg': True,
'report_provisioned_capacity': False}
)
@ddt.unpack
def test_get_pool_stats(self, replication_backends, cluster_credentials,
is_fg):
is_fg, report_provisioned_capacity):
self.driver.using_cluster_credentials = cluster_credentials
conf = self.driver.configuration
conf.netapp_driver_reports_provisioned_capacity = (
report_provisioned_capacity)
self.driver.zapi_client = mock.Mock()
ssc = {
'vola': {
@ -204,9 +210,19 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase):
'total_capacity_gb': total_capacity_gb,
'free_capacity_gb': free_capacity_gb,
}
files_provisioned_cap = [{
'name': 'volume-ae947c9b-2392-4956-b373-aaac4521f37e',
'file-size': 5368709120.0 # 5GB
}, {
'name': 'snapshot-527eedad-a431-483d-b0ca-18995dd65b66',
'file-size': 1073741824.0 # 1GB
}]
self.mock_object(self.driver,
'_get_share_capacity_info',
return_value=capacity)
self.mock_object(self.driver.zapi_client,
'get_file_sizes_by_dir',
return_value=files_provisioned_cap)
self.mock_object(self.driver.zapi_client,
'get_flexvol_dedupe_used_percent',
return_value=55.0)
@ -255,6 +271,8 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase):
'online_extend_support': False,
'netapp_is_flexgroup': 'false',
}]
if report_provisioned_capacity:
expected[0].update({'provisioned_capacity_gb': 5.0})
expected[0].update({'QoS_support': cluster_credentials})
if not cluster_credentials:

13
cinder/volume/drivers/netapp/dataontap/block_cmode.py
View File

@ -333,6 +333,19 @@ class NetAppBlockStorageCmodeLibrary(block_base.NetAppBlockStorageLibrary,
size_available_gb = capacity['size-available'] / units.Gi
pool['free_capacity_gb'] = na_utils.round_down(size_available_gb)
if self.configuration.netapp_driver_reports_provisioned_capacity:
luns = self.zapi_client.get_lun_sizes_by_volume(
ssc_vol_name)
provisioned_cap = 0
for lun in luns:
lun_name = lun['path'].split('/')[-1]
# Filtering luns that matches the volume name template to
# exclude snapshots
if volume_utils.extract_id_from_volume_name(lun_name):
provisioned_cap = provisioned_cap + lun['size']
pool['provisioned_capacity_gb'] = na_utils.round_down(
float(provisioned_cap) / units.Gi)
if self.using_cluster_credentials:
dedupe_used = self.zapi_client.get_flexvol_dedupe_used_percent(
ssc_vol_name)

65
cinder/volume/drivers/netapp/dataontap/client/client_cmode.py
View File

@ -390,6 +390,71 @@ class Client(client_base.Client):
LOG.debug('No iSCSI service found for vserver %s', self.vserver)
return None
def get_lun_sizes_by_volume(self, volume_name):
""""Gets the list of LUNs and their sizes from a given volume name"""
api_args = {
'query': {
'lun-info': {
'volume': volume_name
}
},
'desired-attributes': {
'lun-info': {
'path': None,
'size': None
}
}
}
result = self.send_iter_request(
'lun-get-iter', api_args, max_page_length=100)
if not self._has_records(result):
return []
attributes_list = result.get_child_by_name('attributes-list')
luns = []
for lun_info in attributes_list.get_children():
luns.append({
'path': lun_info.get_child_content('path'),
'size': float(lun_info.get_child_content('size'))
})
return luns
def get_file_sizes_by_dir(self, dir_path):
"""Gets the list of files and their sizes from a given directory."""
api_args = {
'path': '/vol/%s' % dir_path,
'query': {
'file-info': {
'file-type': 'file'
}
},
'desired-attributes': {
'file-info': {
'name': None,
'file-size': None
}
}
}
result = self.send_iter_request(
'file-list-directory-iter', api_args, max_page_length=100)
if not self._has_records(result):
return []
attributes_list = result.get_child_by_name('attributes-list')
files = []
for file_info in attributes_list.get_children():
files.append({
'name': file_info.get_child_content('name'),
'file-size': float(file_info.get_child_content('file-size'))
})
return files
def get_lun_list(self):
"""Gets the list of LUNs on filer.

9
cinder/volume/drivers/netapp/dataontap/nfs_cmode.py
View File

@ -27,6 +27,7 @@ import uuid
from oslo_log import log as logging
from oslo_service import loopingcall
from oslo_utils import excutils
from oslo_utils import units
import six
from cinder import exception
@ -390,6 +391,14 @@ class NetAppCmodeNfsDriver(nfs_base.NetAppNfsDriver,
nfs_share = ssc_vol_info['pool_name']
capacity = self._get_share_capacity_info(nfs_share)
pool.update(capacity)
if self.configuration.netapp_driver_reports_provisioned_capacity:
files = self.zapi_client.get_file_sizes_by_dir(ssc_vol_name)
provisioned_cap = 0
for f in files:
if volume_utils.extract_id_from_volume_name(f['name']):
provisioned_cap = provisioned_cap + f['file-size']
pool['provisioned_capacity_gb'] = na_utils.round_down(
float(provisioned_cap) / units.Gi)
if self.using_cluster_credentials and not is_flexgroup:
dedupe_used = self.zapi_client.get_flexvol_dedupe_used_percent(

12
cinder/volume/drivers/netapp/options.py
View File

@ -83,7 +83,17 @@ netapp_provisioning_opts = [
help=('This option determines if storage space is reserved '
'for LUN allocation. If enabled, LUNs are thick '
'provisioned. If space reservation is disabled, '
'storage space is allocated on demand.')), ]
'storage space is allocated on demand.')),
cfg.BoolOpt('netapp_driver_reports_provisioned_capacity',
default=False,
help=('Set to True for Cinder to query the storage system in '
'order to calculate volumes provisioned size, otherwise '
'provisioned_capacity_gb will corresponds to the '
'value of allocated_capacity_gb (calculated by Cinder '
'Core code). Enabling this feature increases '
'the number of API calls to the storage and '
'requires more processing on host, which may impact '
'volume report overall performance.')), ]
netapp_cluster_opts = [
cfg.StrOpt('netapp_vserver',

7
releasenotes/notes/ontap-add-provisioned-capacity-option-2f8122663eec51ae.yaml Normal file
View File

@ -0,0 +1,7 @@
---
features:
- |
NetApp ONTAP driver: added option
´netapp_driver_reports_provisioned_capacity´, which enables the driver
to calculate and report provisioned capacity to Cinder Scheduler based
on volumes sizes in the storage system.