NetApp ONTAP: Add option to report storage provisioned capacity

This patch adds option ´netapp_driver_reports_provisioned_capacity´,
for driver to calculate the storage provisioned capacity by queryng
volume sizes directly from the storage system and reporting as a
pool capability ´provisioned_capacity_gb´, instead of relying
on the default behavior of using scheduler and volume service
internal states as done by ´allocated_capacity_gb´.

Implements: blueprint ontap-report-provisioned-capacity
Change-Id: I97625de865a63b0cf725f61d9d2ea3c44740d9e7

cinder/tests/unit/volume/drivers/netapp/dataontap/client/fakes.py

@@ -1389,6 +1389,34 @@ CLUSTER_PEER_POLICY_GET_RESPONSE = etree.XML("""
   </results>
 """)
 
+FILE_SIZES_BY_DIR_GET_ITER_RESPONSE = etree.XML("""
+  <results status="passed">
+    <attributes-list>
+      <file-info>
+        <name>%(name)s</name>
+        <file-size>1024</file-size>
+      </file-info>
+    </attributes-list>
+    <num-records>1</num-records>
+  </results>
+""" % {
+    'name': fake.VOLUME_NAME
+})
+
+LUN_SIZES_BY_VOLUME_GET_ITER_RESPONSE = etree.XML("""
+  <results status="passed">
+    <attributes-list>
+      <lun-info>
+        <path>%(path)s</path>
+        <size>1024</size>
+      </lun-info>
+    </attributes-list>
+    <num-records>1</num-records>
+  </results>
+""" % {
+    'path': fake.VOLUME_PATH
+})
+
 VSERVER_PEER_GET_ITER_RESPONSE = etree.XML("""
   <results status="passed">
     <attributes-list>

cinder/tests/unit/volume/drivers/netapp/dataontap/client/test_client_cmode.py

@@ -3379,6 +3379,98 @@ class NetAppCmodeClientTestCase(test.TestCase):
         self.client.connection.send_request.assert_has_calls([
             mock.call('vserver-peer-accept', vserver_peer_accept_args)])
 
+    def test_get_file_sizes_by_dir(self):
+
+        api_response = netapp_api.NaElement(
+            fake_client.FILE_SIZES_BY_DIR_GET_ITER_RESPONSE)
+        self.mock_object(self.client,
+                         'send_iter_request',
+                         return_value=api_response)
+
+        result = self.client.get_file_sizes_by_dir(fake.NETAPP_VOLUME)
+
+        get_get_file_sizes_by_dir_get_iter_args = {
+            'path': '/vol/%s' % fake.NETAPP_VOLUME,
+            'query': {
+                'file-info': {
+                    'file-type': 'file',
+                }
+            },
+            'desired-attributes': {
+                'file-info': {
+                    'name': None,
+                    'file-size': None
+                }
+            },
+        }
+        self.client.send_iter_request.assert_has_calls([
+            mock.call('file-list-directory-iter',
+                      get_get_file_sizes_by_dir_get_iter_args,
+                      max_page_length=100)])
+
+        expected = [{
+            'name': fake.VOLUME_NAME,
+            'file-size': float(1024)
+        }]
+        self.assertEqual(expected, result)
+
+    def test_get_file_sizes_by_dir_not_found(self):
+
+        api_response = netapp_api.NaElement(fake_client.NO_RECORDS_RESPONSE)
+        self.mock_object(self.client,
+                         'send_iter_request',
+                         return_value=api_response)
+
+        result = self.client.get_file_sizes_by_dir(fake.NETAPP_VOLUME)
+
+        self.assertEqual([], result)
+        self.assertTrue(self.client.send_iter_request.called)
+
+    def test_get_lun_sizes_by_volume(self):
+
+        api_response = netapp_api.NaElement(
+            fake_client.LUN_SIZES_BY_VOLUME_GET_ITER_RESPONSE)
+        self.mock_object(self.client,
+                         'send_iter_request',
+                         return_value=api_response)
+
+        result = self.client.get_lun_sizes_by_volume(fake.NETAPP_VOLUME)
+
+        get_lun_sizes_by_volume_get_iter_args = {
+            'query': {
+                'lun-info': {
+                    'volume': fake.NETAPP_VOLUME,
+                }
+            },
+            'desired-attributes': {
+                'lun-info': {
+                    'path': None,
+                    'size': None
+                }
+            },
+        }
+        self.client.send_iter_request.assert_has_calls([
+            mock.call('lun-get-iter', get_lun_sizes_by_volume_get_iter_args,
+                      max_page_length=100)])
+
+        expected = [{
+            'path': fake.VOLUME_PATH,
+            'size': float(1024)
+        }]
+        self.assertEqual(expected, result)
+
+    def test_get_lun_sizes_by_volume_not_found(self):
+
+        api_response = netapp_api.NaElement(fake_client.NO_RECORDS_RESPONSE)
+        self.mock_object(self.client,
+                         'send_iter_request',
+                         return_value=api_response)
+
+        result = self.client.get_lun_sizes_by_volume(fake.NETAPP_VOLUME)
+
+        self.assertEqual([], result)
+        self.assertTrue(self.client.send_iter_request.called)
+
     def test_get_vserver_peers(self):
 
         api_response = netapp_api.NaElement(

cinder/tests/unit/volume/drivers/netapp/dataontap/fakes.py

@@ -333,6 +333,7 @@ CLONE_DESTINATION = {
 }
 
 VOLUME_NAME = 'volume-fake_volume_id'
+VOLUME_PATH = '/vol/%s/%s' % (NETAPP_VOLUME, VOLUME_NAME)
 MOUNT_PATH = '168.10.16.11:/' + VOLUME_ID
 SNAPSHOT_NAME = 'fake_snapshot_name'
 SNAPSHOT_LUN_HANDLE = 'fake_snapshot_lun_handle'

cinder/tests/unit/volume/drivers/netapp/dataontap/test_block_cmode.py

@@ -353,12 +353,19 @@ class NetAppBlockStorageCmodeLibraryTestCase(test.TestCase):
             fake.VOLUME_ID, fake.LUN_ID, fake.LUN_SIZE, fake.LUN_METADATA,
             None, False)
 
-    @ddt.data({'replication_backends': [], 'cluster_credentials': False},
+    @ddt.data({'replication_backends': [], 'cluster_credentials': False,
+               'report_provisioned_capacity': False},
               {'replication_backends': ['target_1', 'target_2'],
-               'cluster_credentials': True})
+               'cluster_credentials': True,
+               'report_provisioned_capacity': True})
     @ddt.unpack
-    def test_get_pool_stats(self, replication_backends, cluster_credentials):
+    def test_get_pool_stats(self, replication_backends, cluster_credentials,
+                            report_provisioned_capacity):
         self.library.using_cluster_credentials = cluster_credentials
+        conf = self.library.configuration
+        conf.netapp_driver_reports_provisioned_capacity = (
+            report_provisioned_capacity)
+
         ssc = {
             'vola': {
                 'pool_name': 'vola',
@@ -391,9 +398,19 @@ class NetAppBlockStorageCmodeLibraryTestCase(test.TestCase):
             'size-total': 10737418240.0,
             'size-available': 2147483648.0,
         }
+        luns_provisioned_cap = [{
+            'path': '/vol/volume-ae947c9b-2392-4956-b373-aaac4521f37e',
+            'size': 5368709120.0  # 5GB
+        }, {
+            'path': '/vol/snapshot-527eedad-a431-483d-b0ca-18995dd65b66',
+            'size': 1073741824.0  # 1GB
+        }]
         self.mock_object(self.zapi_client,
                          'get_flexvol_capacity',
                          return_value=mock_capacities)
+        self.mock_object(self.zapi_client,
+                         'get_lun_sizes_by_volume',
+                         return_value=luns_provisioned_cap)
         self.mock_object(self.zapi_client,
                          'get_flexvol_dedupe_used_percent',
                          return_value=55.0)
@@ -440,6 +457,8 @@ class NetAppBlockStorageCmodeLibraryTestCase(test.TestCase):
             'online_extend_support': True,
             'netapp_is_flexgroup': 'false',
         }]
+        if report_provisioned_capacity:
+            expected[0].update({'provisioned_capacity_gb': 5.0})
 
         expected[0].update({'QoS_support': cluster_credentials})
         if not cluster_credentials:

cinder/tests/unit/volume/drivers/netapp/dataontap/test_nfs_cmode.py

@@ -157,16 +157,22 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase):
         self.assertEqual(expected_stats, self.driver._stats)
 
     @ddt.data({'replication_backends': [],
-               'cluster_credentials': False, 'is_fg': False},
+               'cluster_credentials': False, 'is_fg': False,
+               'report_provisioned_capacity': True},
               {'replication_backends': ['target_1', 'target_2'],
-               'cluster_credentials': True, 'is_fg': False},
+               'cluster_credentials': True, 'is_fg': False,
+               'report_provisioned_capacity': False},
               {'replication_backends': ['target_1', 'target_2'],
-               'cluster_credentials': True, 'is_fg': True}
+               'cluster_credentials': True, 'is_fg': True,
+               'report_provisioned_capacity': False}
               )
     @ddt.unpack
     def test_get_pool_stats(self, replication_backends, cluster_credentials,
-                            is_fg):
+                            is_fg, report_provisioned_capacity):
         self.driver.using_cluster_credentials = cluster_credentials
+        conf = self.driver.configuration
+        conf.netapp_driver_reports_provisioned_capacity = (
+            report_provisioned_capacity)
         self.driver.zapi_client = mock.Mock()
         ssc = {
             'vola': {
@@ -204,9 +210,19 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase):
             'total_capacity_gb': total_capacity_gb,
             'free_capacity_gb': free_capacity_gb,
         }
+        files_provisioned_cap = [{
+            'name': 'volume-ae947c9b-2392-4956-b373-aaac4521f37e',
+            'file-size': 5368709120.0  # 5GB
+        }, {
+            'name': 'snapshot-527eedad-a431-483d-b0ca-18995dd65b66',
+            'file-size': 1073741824.0  # 1GB
+        }]
         self.mock_object(self.driver,
                          '_get_share_capacity_info',
                          return_value=capacity)
+        self.mock_object(self.driver.zapi_client,
+                         'get_file_sizes_by_dir',
+                         return_value=files_provisioned_cap)
         self.mock_object(self.driver.zapi_client,
                          'get_flexvol_dedupe_used_percent',
                          return_value=55.0)
@@ -255,6 +271,8 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase):
             'online_extend_support': False,
             'netapp_is_flexgroup': 'false',
         }]
+        if report_provisioned_capacity:
+            expected[0].update({'provisioned_capacity_gb': 5.0})
 
         expected[0].update({'QoS_support': cluster_credentials})
         if not cluster_credentials:

cinder/volume/drivers/netapp/dataontap/block_cmode.py

@@ -333,6 +333,19 @@ class NetAppBlockStorageCmodeLibrary(block_base.NetAppBlockStorageLibrary,
             size_available_gb = capacity['size-available'] / units.Gi
             pool['free_capacity_gb'] = na_utils.round_down(size_available_gb)
 
+            if self.configuration.netapp_driver_reports_provisioned_capacity:
+                luns = self.zapi_client.get_lun_sizes_by_volume(
+                    ssc_vol_name)
+                provisioned_cap = 0
+                for lun in luns:
+                    lun_name = lun['path'].split('/')[-1]
+                    # Filtering luns that matches the volume name template to
+                    # exclude snapshots
+                    if volume_utils.extract_id_from_volume_name(lun_name):
+                        provisioned_cap = provisioned_cap + lun['size']
+                pool['provisioned_capacity_gb'] = na_utils.round_down(
+                    float(provisioned_cap) / units.Gi)
+
             if self.using_cluster_credentials:
                 dedupe_used = self.zapi_client.get_flexvol_dedupe_used_percent(
                     ssc_vol_name)

cinder/volume/drivers/netapp/dataontap/client/client_cmode.py

@@ -390,6 +390,71 @@ class Client(client_base.Client):
         LOG.debug('No iSCSI service found for vserver %s', self.vserver)
         return None
 
+    def get_lun_sizes_by_volume(self, volume_name):
+        """"Gets the list of LUNs and their sizes from a given volume name"""
+
+        api_args = {
+            'query': {
+                'lun-info': {
+                    'volume': volume_name
+                }
+            },
+            'desired-attributes': {
+                'lun-info': {
+                    'path': None,
+                    'size': None
+                }
+            }
+        }
+        result = self.send_iter_request(
+            'lun-get-iter', api_args, max_page_length=100)
+
+        if not self._has_records(result):
+            return []
+
+        attributes_list = result.get_child_by_name('attributes-list')
+
+        luns = []
+        for lun_info in attributes_list.get_children():
+            luns.append({
+                'path': lun_info.get_child_content('path'),
+                'size': float(lun_info.get_child_content('size'))
+            })
+        return luns
+
+    def get_file_sizes_by_dir(self, dir_path):
+        """Gets the list of files and their sizes from a given directory."""
+
+        api_args = {
+            'path': '/vol/%s' % dir_path,
+            'query': {
+                'file-info': {
+                    'file-type': 'file'
+                }
+            },
+            'desired-attributes': {
+                'file-info': {
+                    'name': None,
+                    'file-size': None
+                }
+            }
+        }
+        result = self.send_iter_request(
+            'file-list-directory-iter', api_args, max_page_length=100)
+
+        if not self._has_records(result):
+            return []
+
+        attributes_list = result.get_child_by_name('attributes-list')
+
+        files = []
+        for file_info in attributes_list.get_children():
+            files.append({
+                'name': file_info.get_child_content('name'),
+                'file-size': float(file_info.get_child_content('file-size'))
+            })
+        return files
+
     def get_lun_list(self):
         """Gets the list of LUNs on filer.
 

cinder/volume/drivers/netapp/dataontap/nfs_cmode.py

@@ -27,6 +27,7 @@ import uuid
 from oslo_log import log as logging
 from oslo_service import loopingcall
 from oslo_utils import excutils
+from oslo_utils import units
 import six
 
 from cinder import exception
@@ -390,6 +391,14 @@ class NetAppCmodeNfsDriver(nfs_base.NetAppNfsDriver,
             nfs_share = ssc_vol_info['pool_name']
             capacity = self._get_share_capacity_info(nfs_share)
             pool.update(capacity)
+            if self.configuration.netapp_driver_reports_provisioned_capacity:
+                files = self.zapi_client.get_file_sizes_by_dir(ssc_vol_name)
+                provisioned_cap = 0
+                for f in files:
+                    if volume_utils.extract_id_from_volume_name(f['name']):
+                        provisioned_cap = provisioned_cap + f['file-size']
+                pool['provisioned_capacity_gb'] = na_utils.round_down(
+                    float(provisioned_cap) / units.Gi)
 
             if self.using_cluster_credentials and not is_flexgroup:
                 dedupe_used = self.zapi_client.get_flexvol_dedupe_used_percent(

cinder/volume/drivers/netapp/options.py

@@ -83,7 +83,17 @@ netapp_provisioning_opts = [
                help=('This option determines if storage space is reserved '
                      'for LUN allocation. If enabled, LUNs are thick '
                      'provisioned. If space reservation is disabled, '
-                     'storage space is allocated on demand.')), ]
+                     'storage space is allocated on demand.')),
+    cfg.BoolOpt('netapp_driver_reports_provisioned_capacity',
+                default=False,
+                help=('Set to True for Cinder to query the storage system in '
+                      'order to calculate volumes provisioned size, otherwise '
+                      'provisioned_capacity_gb will corresponds to the '
+                      'value of allocated_capacity_gb (calculated by Cinder '
+                      'Core code). Enabling this feature increases '
+                      'the number of API calls to the storage and '
+                      'requires more processing on host, which may impact '
+                      'volume report overall performance.')), ]
 
 netapp_cluster_opts = [
     cfg.StrOpt('netapp_vserver',

releasenotes/notes/ontap-add-provisioned-capacity-option-2f8122663eec51ae.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    NetApp ONTAP driver: added option
+    ´netapp_driver_reports_provisioned_capacity´, which enables the driver
+    to calculate and report provisioned capacity to Cinder Scheduler based
+    on volumes sizes in the storage system.