Dell PowerScale: Added support of thin provisioning

Implements: blueprint powerscale-thin-provisioning
Change-Id: I197796713304ff6695f39b1b39817bfbb4c7927e
Signed-off-by: Nilesh Thathagar <nilesh.thathagar@dell.com>

doc/source/admin/share_back_ends_feature_support_mapping.rst

@@ -284,7 +284,7 @@ More information: :ref:`capabilities_and_extra_specs`
 +----------------------------------------+-----------+------------+--------+-------------+-------------------+--------------------+-----+----------------------------+--------------------+--------------------+--------------+--------------+-------------------------+--------------------------+
 |                EMC Unity               |     N     |      T     |   \-   |      \-     |         N         |          \-        | \-  |              N             |          S         |          \-        |       P      |       Q      |           \-            |           \-             |
 +----------------------------------------+-----------+------------+--------+-------------+-------------------+--------------------+-----+----------------------------+--------------------+--------------------+--------------+--------------+-------------------------+--------------------------+
-|               EMC Isilon               |     \-    |      K     |   \-   |      \-     |        \-         |          L         | \-  |              K             |          \-        |          \-        |       P      |      \-      |           \-            |           \-             |
+|               EMC Isilon               |     \-    |      K     |   \-   |      \-     |         F         |          \-        | \-  |              K             |          \-        |          \-        |       P      |      \-      |           \-            |           \-             |
 +----------------------------------------+-----------+------------+--------+-------------+-------------------+--------------------+-----+----------------------------+--------------------+--------------------+--------------+--------------+-------------------------+--------------------------+
 |          Dell EMC PowerStore           |     \-    |      B     |   \-   |      \-     |         B         |          \-        | \-  |              B             |          B         |          \-        |       B      |      \-      |           \-            |           \-             |
 +----------------------------------------+-----------+------------+--------+-------------+-------------------+--------------------+-----+----------------------------+--------------------+--------------------+--------------+--------------+-------------------------+--------------------------+

doc/source/configuration/shared-file-systems/drivers/emc-isilon-driver.rst

@@ -62,6 +62,16 @@ Systems service configuration file for the Isilon driver:
    emc_nas_login = <username>
    emc_nas_password = <password>
 
+Thin Provisioning
+~~~~~~~~~~~~~~~~~
+
+PowerScale systems have thin provisioning enabled by default.
+Add the parameter below to set an advisory limit.
+
+.. code-block:: ini
+
+    powerscale_threshold_limit = <threshold percentage value>
+
 Restrictions
 ~~~~~~~~~~~~
 

manila/share/drivers/dell_emc/plugins/isilon/isilon.py

@@ -33,8 +33,9 @@ from manila.share.drivers.dell_emc.plugins.isilon import isilon_api
     1.0.0 - Fix Http auth issue, SSL verification error and etc
     1.0.1 - Add support for update share stats
     1.0.2 - Add support for ensure shares
+    1.0.3 - Add support for thin provisioning
 """
-VERSION = "1.0.2"
+VERSION = "1.0.3"
 
 CONF = cfg.CONF
 
@@ -44,7 +45,11 @@ POWERSCALE_OPTS = [
     cfg.StrOpt('powerscale_dir_permission',
                default='0777',
                help='Predefined ACL value or POSIX mode '
-                    'for PowerScale directories.')
+                    'for PowerScale directories.'),
+    cfg.IntOpt('powerscale_threshold_limit',
+               default=0,
+               help='Specifies the threshold limit (in percentage) '
+                    'for triggering SmartQuotas alerts in PowerScale')
 ]
 
 
@@ -78,6 +83,7 @@ class IsilonStorageConnection(base.StorageConnection):
         self.reserved_snapshot_percentage = None
         self.reserved_share_extend_percentage = None
         self.max_over_subscription_ratio = None
+        self._threshold_limit = 0
 
     def _get_container_path(self, share):
         """Return path to a container."""
@@ -298,6 +304,7 @@ class IsilonStorageConnection(base.StorageConnection):
         self._username = config.safe_get("emc_nas_login")
         self._password = config.safe_get("emc_nas_password")
         self._root_dir = config.safe_get("emc_nas_root_dir")
+        self._threshold_limit = config.safe_get("powerscale_threshold_limit")
 
         # validate IP, username and password
         if not all([self._server,
@@ -316,7 +323,8 @@ class IsilonStorageConnection(base.StorageConnection):
         self._isilon_api = isilon_api.IsilonApi(
             self._server_url, self._username, self._password,
             self._verify_ssl_cert, self._ssl_cert_path,
-            self._dir_permission)
+            self._dir_permission,
+            self._threshold_limit)
 
         if not self._isilon_api.is_path_existent(self._root_dir):
             self._create_directory(self._root_dir, recursive=True)
@@ -344,7 +352,6 @@ class IsilonStorageConnection(base.StorageConnection):
         """Retrieve stats info from share."""
         stats_dict['driver_version'] = VERSION
         stats_dict['storage_protocol'] = 'NFS_CIFS'
-
         # PowerScale does not support pools.
         # To align with manila scheduler 'pool-aware' strategic,
         # report with one pool structure.
@@ -357,13 +364,15 @@ class IsilonStorageConnection(base.StorageConnection):
             'reserved_share_extend_percentage':
                 self.reserved_share_extend_percentage,
             'max_over_subscription_ratio':
-                self.max_over_subscription_ratio
+                self.max_over_subscription_ratio,
+            'thin_provisioning': True,
         }
         spaces = self._isilon_api.get_space_stats()
         if spaces:
             pool_stat['total_capacity_gb'] = spaces['total'] // units.Gi
             pool_stat['free_capacity_gb'] = spaces['free'] // units.Gi
-            pool_stat['allocated_capacity_gb'] = spaces['used'] // units.Gi
+        allocated_space = self._isilon_api.get_allocated_space()
+        pool_stat['allocated_capacity_gb'] = allocated_space
 
         stats_dict['pools'] = [pool_stat]
 

manila/share/drivers/dell_emc/plugins/isilon/isilon_api.py

@@ -32,7 +32,8 @@ class IsilonApi(object):
     def __init__(self, api_url, username, password,
                  verify_ssl_cert=False,
                  ssl_cert_path=None,
-                 dir_permission=None):
+                 dir_permission=None,
+                 threshold_limit=0):
         self.host_url = api_url
         self.session = requests.session()
         self.username = username
@@ -40,6 +41,7 @@ class IsilonApi(object):
         self.verify_ssl_cert = verify_ssl_cert
         self.certificate_path = ssl_cert_path
         self.dir_permission = dir_permission
+        self.threshold_limit = threshold_limit
 
         # Create session
         self.session_token = None
@@ -279,6 +281,9 @@ class IsilonApi(object):
 
     def quota_create(self, path, quota_type, size):
         thresholds = {'hard': size}
+        if self.threshold_limit > 0:
+            advisory_size = round((size * self.threshold_limit) / 100)
+            thresholds['advisory'] = int(advisory_size)
         data = {
             'path': path,
             'type': quota_type,
@@ -315,6 +320,9 @@ class IsilonApi(object):
 
     def quota_modify_size(self, quota_id, new_size):
         data = {'thresholds': {'hard': new_size}}
+        if self.threshold_limit > 0:
+            advisory_size = round((new_size * self.threshold_limit) / 100)
+            data.get('thresholds')['advisory'] = int(advisory_size)
         response = self.send_put_request(
             '{0}/platform/1/quota/quotas/{1}'.format(self.host_url, quota_id),
             data=data
@@ -393,6 +401,22 @@ class IsilonApi(object):
                 spaces['used'] = stat['value']
         return spaces
 
+    def get_allocated_space(self):
+        url = '{0}/platform/1/quota/quotas'.format(self.host_url)
+        r = self.send_get_request(url)
+        allocated_capacity = 0
+        if r.status_code != 200:
+            raise exception.ShareBackendException(
+                msg=_('Failed to get share quotas from PowerScale.')
+            )
+        quotas = r.json()['quotas']
+        for quota in quotas:
+            if quota['thresholds']['hard'] is not None:
+                allocated_capacity += quota['thresholds']['hard']
+        if allocated_capacity > 0:
+            return round(allocated_capacity / (1024 ** 3), 2)
+        return allocated_capacity
+
     def get_cluster_version(self):
         url = '{0}/platform/12/cluster/version'.format(self.host_url)
         r = self.send_get_request(url)

manila/tests/share/drivers/dell_emc/plugins/isilon/test_isilon.py

@@ -516,8 +516,8 @@ class IsilonTest(test.TestCase):
         self._mock_isilon_api.get_space_stats.return_value = {
             'total': 1000 * units.Gi,
             'free': 100 * units.Gi,
-            'used': 1 * units.Gi,
         }
+        self._mock_isilon_api.get_allocated_space.return_value = 2110.0
         stats_dict = {'share_backend_name': 'PowerScale_backend'}
         self.storage_connection.update_share_stats(stats_dict)
 
@@ -527,10 +527,11 @@ class IsilonTest(test.TestCase):
             'reserved_snapshot_percentage': 0,
             'reserved_share_extend_percentage': 0,
             'max_over_subscription_ratio': None,
+            'thin_provisioning': True,
             'total_capacity_gb': 1000,
             'free_capacity_gb': 100,
-            'allocated_capacity_gb': 1,
+            'allocated_capacity_gb': 2110.0,
-            'qos': False
+            'qos': False,
         }
         expected_stats = {
             'share_backend_name': 'PowerScale_backend',

manila/tests/share/drivers/dell_emc/plugins/isilon/test_isilon_api.py

@@ -42,6 +42,11 @@ class IsilonApiTest(test.TestCase):
             self._mock_url, self.username, self.password,
             dir_permission=self.dir_permission
         )
+        self.isilon_api_threshold = isilon_api.IsilonApi(
+            self._mock_url, self.username, self.password,
+            dir_permission=self.dir_permission,
+            threshold_limit=80
+        )
 
     @mock.patch('manila.share.drivers.dell_emc.plugins.isilon.'
                 'isilon_api.IsilonApi.create_session')
@@ -575,6 +580,30 @@ class IsilonApiTest(test.TestCase):
         call_body = m.request_history[0].body
         self.assertEqual(expected_request_json, json.loads(call_body))
 
+    @requests_mock.mock()
+    def test_quota_create_with_threshold(self, m):
+        quota_path = '/ifs/manila/test'
+        quota_size = 100
+        self.assertEqual(0, len(m.request_history))
+        m.post(self._mock_url + '/platform/1/quota/quotas', status_code=201)
+        self.isilon_api_threshold.quota_create(quota_path,
+                                               'directory',
+                                               quota_size)
+        advisory_size = round(
+            (quota_size * self.isilon_api_threshold.threshold_limit) / 100)
+        self.assertEqual(1, len(m.request_history))
+        expected_request_json = {
+            'path': quota_path,
+            'type': 'directory',
+            'include_snapshots': False,
+            'thresholds_include_overhead': False,
+            'enforced': True,
+            'thresholds': {'hard': quota_size,
+                           'advisory': advisory_size},
+        }
+        call_body = m.request_history[0].body
+        self.assertEqual(expected_request_json, json.loads(call_body))
+
     @requests_mock.mock()
     def test_quota_create__path_does_not_exist(self, m):
         quota_path = '/ifs/test2'
@@ -628,6 +657,22 @@ class IsilonApiTest(test.TestCase):
         request_body = m.request_history[0].body
         self.assertEqual(expected_request_body, json.loads(request_body))
 
+    @requests_mock.mock()
+    def test_quota_modify_with_threshold(self, m):
+        self.assertEqual(0, len(m.request_history))
+        quota_id = "ADEF1G"
+        new_size = 1024
+        advisory_size = round(
+            (new_size * self.isilon_api_threshold.threshold_limit) / 100)
+        m.put('{0}/platform/1/quota/quotas/{1}'.format(
+            self._mock_url, quota_id), status_code=204)
+        self.isilon_api_threshold.quota_modify_size(quota_id, new_size)
+        self.assertEqual(1, len(m.request_history))
+        expected_request_body = {'thresholds': {'hard': new_size,
+                                                'advisory': advisory_size}}
+        request_body = m.request_history[0].body
+        self.assertEqual(expected_request_body, json.loads(request_body))
+
     @requests_mock.mock()
     def test_quota_modify__given_id_does_not_exist(self, m):
         quota_id = 'ADE2F'
@@ -886,6 +931,46 @@ class IsilonApiTest(test.TestCase):
         self.assertRaises(exception.ShareBackendException,
                           self.isilon_api.get_space_stats)
 
+    def test_get_allocated_space_success(self):
+        self.isilon_api.send_get_request = mock.MagicMock()
+        self.isilon_api.send_get_request.return_value.status_code = 200
+        self.isilon_api.send_get_request.return_value.json.return_value = {
+            'quotas': [
+                {
+                    'path': '/ifs/home',
+                    'thresholds': {
+                        'hard': None
+                    }
+                },
+                {
+                    'path': '/ifs/manila/CI-1d52ed66-a1ee-4b19-8f56-3706b',
+                    'thresholds': {
+                        'hard': 2147483648000
+                    }
+                },
+                {
+                    'path': '/ifs/manila/CI-0b622133-8b58-4a9f-ad1a-b8247',
+                    'thresholds': {
+                        'hard': 107374182400
+                    }
+                },
+                {
+                    'path': '/ifs/nilesh',
+                    'thresholds': {
+                        'hard': 10737418240
+                    }
+                }
+            ]
+        }
+        result = self.isilon_api.get_allocated_space()
+        self.assertEqual(result, 2110.0)
+
+    def test_get_allocated_space_failure(self):
+        self.isilon_api.send_get_request = mock.MagicMock()
+        self.isilon_api.send_get_request.return_value.status_code = 400
+        self.assertRaises(exception.ShareBackendException,
+                          self.isilon_api.get_allocated_space)
+
     def test_get_cluster_version_success(self):
         self.isilon_api.send_get_request = mock.MagicMock()
         self.isilon_api.send_get_request.return_value.status_code = 200

releasenotes/notes/bp-dell-powerscale-thin-provisioning-71a8c25322d67a6b.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    Added support for thin provisioning on Dell PowerScale.