Merge "HP 3PAR reports capabilities"

doc/source/devref/hp_3par_driver.rst

@@ -88,6 +88,8 @@ file for the HP 3PAR driver:
 - `hp3par_share_ip_address` = <IP address to use for share export location>
 - `hp3par_san_ip` = <IP address for SSH access to the SAN controller>
 - `hp3par_api_url` = <3PAR WS API Server URL>
+- `hp3par_username` = <3PAR username with the 'edit' role>
+- `hp3par_password` = <3PAR password for the user specified in hp3par_username>
 - `hp3par_san_login` = <Username for SSH access to the SAN controller>
 - `hp3par_san_password` = <Password for SSH access to the SAN controller>
 - `hp3par_debug` = <False or True for extra debug logging>
@@ -131,6 +133,27 @@ Another common Manila extra-spec used to determine where a share is created
 is `share_backend_name`. When this extra-spec is defined in the share type,
 the share will be created on a backend with a matching share_backend_name.
 
+The HP 3PAR driver automatically reports capabilities based on the FPG used
+for each backend. Share types with extra specs can be created by an
+administrator to control which share types are allowed to use FPGs with or
+without specific capabilities. The following extra-specs are used with
+the capabilities filter and the HP 3PAR driver:
+
+- `hp3par_flash_cache` = '<is> True' or '<is> False'
+- `thin_provisioning` = '<is> True' or '<is> False'
+- `dedupe` = '<is> True' or '<is> False'
+
+`hp3par_flash_cache` will be reported as True for backends that have
+3PAR's Adaptive Flash Cache enabled.
+
+`thin_provisioning` will be reported as True for backends that use thin
+provisioned volumes. FPGs that use fully provisioned volumes will report
+False. Backends that use thin provisioning also support Manila's
+over-subscription feature.
+
+`dedupe` will be reported as True for backends that use deduplication
+technology.
+
 Scoped extra-specs are used to influence vendor-specific implementation
 details. Scoped extra-specs use a prefix followed by a colon.  For HP 3PAR
 these extra-specs have a prefix of `hpe3par`.

manila/share/drivers/hp/hp_3par_driver.py

@@ -35,6 +35,13 @@ HP3PAR_OPTS = [
                default='',
                help="3PAR WSAPI Server Url like "
                     "https://<3par ip>:8080/api/v1"),
+    cfg.StrOpt('hp3par_username',
+               default='',
+               help="3PAR username with the 'edit' role"),
+    cfg.StrOpt('hp3par_password',
+               default='',
+               help="3PAR password for the user specified in hp3par_username",
+               secret=True),
     cfg.StrOpt('hp3par_san_ip',
                default='',
                help="IP address of SAN controller"),
@@ -101,6 +108,8 @@ class HP3ParShareDriver(driver.ShareDriver):
                   "hp3par_share_ip_address is not set."))
 
         mediator = hp_3par_mediator.HP3ParMediator(
+            hp3par_username=self.configuration.hp3par_username,
+            hp3par_password=self.configuration.hp3par_password,
             hp3par_api_url=self.configuration.hp3par_api_url,
             hp3par_debug=self.configuration.hp3par_debug,
             hp3par_san_ip=self.configuration.hp3par_san_ip,
@@ -261,21 +270,12 @@ class HP3ParShareDriver(driver.ShareDriver):
     def _update_share_stats(self):
         """Retrieve stats info from share group."""
 
-        if not self._hp3par:
-            LOG.info(
-                _LI("Skipping share statistics update. Setup has not "
-                    "completed."))
-            total_capacity_gb = 0
-            free_capacity_gb = 0
-        else:
-            capacity_stats = self._hp3par.get_capacity(self.fpg)
-            LOG.debug("Share capacity = %s.", capacity_stats)
-            total_capacity_gb = capacity_stats['total_capacity_gb']
-            free_capacity_gb = capacity_stats['free_capacity_gb']
-
         backend_name = self.configuration.safe_get(
             'share_backend_name') or "HP_3PAR"
 
+        max_over_subscription_ratio = self.configuration.safe_get(
+            'max_over_subscription_ratio')
+
         reserved_share_percentage = self.configuration.safe_get(
             'reserved_share_percentage')
         if reserved_share_percentage is None:
@@ -287,10 +287,22 @@ class HP3ParShareDriver(driver.ShareDriver):
             'vendor_name': 'HP',
             'driver_version': self.VERSION,
             'storage_protocol': 'NFS_CIFS',
-            'total_capacity_gb': total_capacity_gb,
-            'free_capacity_gb': free_capacity_gb,
+            'total_capacity_gb': 0,
+            'free_capacity_gb': 0,
+            'provisioned_capacity_gb': 0,
             'reserved_percentage': reserved_share_percentage,
+            'max_over_subscription_ratio': max_over_subscription_ratio,
             'QoS_support': False,
+            'thin_provisioning': True,  # 3PAR default is thin
         }
 
+        if not self._hp3par:
+            LOG.info(
+                _LI("Skipping capacity and capabilities update. Setup has not "
+                    "completed."))
+        else:
+            fpg_status = self._hp3par.get_fpg_status(self.fpg)
+            LOG.debug("FPG status = %s.", fpg_status)
+            stats.update(fpg_status)
+
         super(HP3ParShareDriver, self)._update_share_stats(stats)

manila/share/drivers/hp/hp_3par_mediator.py

@@ -24,7 +24,7 @@ from oslo_utils import units
 import six
 
 from manila import exception
-from manila.i18n import _, _LI
+from manila.i18n import _, _LI, _LW
 
 hp3parclient = importutils.try_import("hp3parclient")
 if hp3parclient:
@@ -37,6 +37,11 @@ MIN_SMB_CA_VERSION = (3, 2, 2)
 DENY = '-'
 ALLOW = '+'
 OPEN_STACK_MANILA_FSHARE = 'OpenStack Manila fshare'
+FULL = 1
+THIN = 2
+DEDUPE = 6
+ENABLED = 1
+DISABLED = 2
 CACHE = 'cache'
 CONTINUOUS_AVAIL = 'continuous_avail'
 ACCESS_BASED_ENUM = 'access_based_enum'
@@ -53,6 +58,8 @@ class HP3ParMediator(object):
 
     def __init__(self, **kwargs):
 
+        self.hp3par_username = kwargs.get('hp3par_username')
+        self.hp3par_password = kwargs.get('hp3par_password')
         self.hp3par_api_url = kwargs.get('hp3par_api_url')
         self.hp3par_debug = kwargs.get('hp3par_debug')
         self.hp3par_san_ip = kwargs.get('hp3par_san_ip')
@@ -136,27 +143,104 @@ class HP3ParMediator(object):
         if self.hp3par_debug:
             self._client.debug_rest(True)  # Includes SSH debug (setSSH above)
 
-    def get_capacity(self, fpg):
+    def _wsapi_login(self):
+        try:
+            self._client.login(self.hp3par_username, self.hp3par_password)
+        except Exception as e:
+            msg = (_("Failed to Login to 3PAR (%(url)s) as %(user)s "
+                     "because: %(err)s") %
+                   {'url': self.hp3par_api_url,
+                    'user': self.hp3par_username,
+                    'err': six.text_type(e)})
+            LOG.error(msg)
+            raise exception.ShareBackendException(msg=msg)
+
+    def _wsapi_logout(self):
+        try:
+            self._client.http.unauthenticate()
+        except Exception as e:
+            msg = _LW("Failed to Logout from 3PAR (%(url)s) because %(err)s")
+            LOG.warning(msg, {'url': self.hp3par_api_url,
+                              'err': six.text_type(e)})
+            # don't raise exception on logout()
+
+    def get_provisioned_gb(self, fpg):
+        total_mb = 0
+        try:
+            result = self._client.getfsquota(fpg=fpg)
+        except Exception as e:
+            result = {'message': six.text_type(e)}
+
+        error_msg = result.get('message')
+        if error_msg:
+            message = (_('Error while getting fsquotas for FPG '
+                         '%(fpg)s: %(msg)s') %
+                       {'fpg': fpg, 'msg': error_msg})
+            LOG.error(message)
+            raise exception.ShareBackendException(msg=message)
+
+        for fsquota in result['members']:
+            total_mb += float(fsquota['hardBlock'])
+        return total_mb / units.Ki
+
+    def get_fpg_status(self, fpg):
+        """Get capacity and capabilities for FPG."""
+
         try:
             result = self._client.getfpg(fpg)
         except Exception as e:
             msg = (_('Failed to get capacity for fpg %(fpg)s: %(e)s') %
                    {'fpg': fpg, 'e': six.text_type(e)})
             LOG.exception(msg)
-            raise exception.ShareBackendException(message=msg)
+            raise exception.ShareBackendException(msg=msg)
 
         if result['total'] != 1:
             msg = (_('Failed to get capacity for fpg %s.') % fpg)
             LOG.exception(msg)
-            raise exception.ShareBackendException(message=msg)
+            raise exception.ShareBackendException(msg=msg)
+
+        member = result['members'][0]
+        total_capacity_gb = float(member['capacityKiB']) / units.Mi
+        free_capacity_gb = float(member['availCapacityKiB']) / units.Mi
+
+        volumes = member['vvs']
+        if isinstance(volumes, list):
+            volume = volumes[0]  # Use first name from list
         else:
-            member = result['members'][0]
-            total_capacity_gb = int(member['capacityKiB']) / units.Mi
-            free_capacity_gb = int(member['availCapacityKiB']) / units.Mi
-            return {
-                'total_capacity_gb': total_capacity_gb,
-                'free_capacity_gb': free_capacity_gb
-            }
+            volume = volumes  # There is just a name
+
+        self._wsapi_login()
+        try:
+            volume_info = self._client.getVolume(volume)
+            volume_set = self._client.getVolumeSet(fpg)
+        finally:
+            self._wsapi_logout()
+
+        provisioning_type = volume_info['provisioningType']
+        if provisioning_type not in (THIN, FULL, DEDUPE):
+            msg = (_('Unexpected provisioning type for FPG %(fpg)s: '
+                     '%(ptype)s.') % {'fpg': fpg, 'ptype': provisioning_type})
+            LOG.exception(msg)
+            raise exception.ShareBackendException(msg=msg)
+
+        dedupe = provisioning_type == DEDUPE
+        thin_provisioning = provisioning_type in (THIN, DEDUPE)
+
+        flash_cache_policy = volume_set.get('flashCachePolicy', DISABLED)
+        hp3par_flash_cache = flash_cache_policy == ENABLED
+
+        status = {
+            'total_capacity_gb': total_capacity_gb,
+            'free_capacity_gb': free_capacity_gb,
+            'thin_provisioning': thin_provisioning,
+            'dedupe': dedupe,
+            'hp3par_flash_cache': hp3par_flash_cache,
+        }
+
+        if thin_provisioning:
+            status['provisioned_capacity_gb'] = self.get_provisioned_gb(fpg)
+
+        return status
 
     @staticmethod
     def ensure_supported_protocol(share_proto):

manila/tests/share/drivers/hp/test_hp_3par_driver.py

@@ -34,6 +34,8 @@ class HP3ParDriverTestCase(test.TestCase):
         self.conf = mock.Mock()
         self.conf.driver_handles_share_servers = False
         self.conf.hp3par_debug = constants.EXPECTED_HP_DEBUG
+        self.conf.hp3par_username = constants.USERNAME
+        self.conf.hp3par_password = constants.PASSWORD
         self.conf.hp3par_api_url = constants.API_URL
         self.conf.hp3par_san_login = constants.SAN_LOGIN
         self.conf.hp3par_san_password = constants.SAN_PASSWORD
@@ -70,9 +72,11 @@ class HP3ParDriverTestCase(test.TestCase):
         self.mock_mediator_constructor.assert_has_calls([
             mock.call(hp3par_san_ssh_port=conf.hp3par_san_ssh_port,
                       hp3par_san_password=conf.hp3par_san_password,
+                      hp3par_username=conf.hp3par_username,
                       hp3par_san_login=conf.hp3par_san_login,
                       hp3par_debug=conf.hp3par_debug,
                       hp3par_api_url=conf.hp3par_api_url,
+                      hp3par_password=conf.hp3par_password,
                       hp3par_san_ip=conf.hp3par_san_ip,
                       hp3par_fstore_per_share=conf.hp3par_fstore_per_share,
                       ssh_conn_timeout=conf.ssh_conn_timeout)])
@@ -96,9 +100,11 @@ class HP3ParDriverTestCase(test.TestCase):
         self.mock_mediator_constructor.assert_has_calls([
             mock.call(hp3par_san_ssh_port=conf.hp3par_san_ssh_port,
                       hp3par_san_password=conf.hp3par_san_password,
+                      hp3par_username=conf.hp3par_username,
                       hp3par_san_login=conf.hp3par_san_login,
                       hp3par_debug=conf.hp3par_debug,
                       hp3par_api_url=conf.hp3par_api_url,
+                      hp3par_password=conf.hp3par_password,
                       hp3par_san_ip=conf.hp3par_san_ip,
                       hp3par_fstore_per_share=conf.hp3par_fstore_per_share,
                       ssh_conn_timeout=conf.ssh_conn_timeout)])
@@ -118,9 +124,11 @@ class HP3ParDriverTestCase(test.TestCase):
         self.mock_mediator_constructor.assert_has_calls([
             mock.call(hp3par_san_ssh_port=conf.hp3par_san_ssh_port,
                       hp3par_san_password=conf.hp3par_san_password,
+                      hp3par_username=conf.hp3par_username,
                       hp3par_san_login=conf.hp3par_san_login,
                       hp3par_debug=conf.hp3par_debug,
                       hp3par_api_url=conf.hp3par_api_url,
+                      hp3par_password=conf.hp3par_password,
                       hp3par_san_ip=conf.hp3par_san_ip,
                       hp3par_fstore_per_share=conf.hp3par_fstore_per_share,
                       ssh_conn_timeout=conf.ssh_conn_timeout)])
@@ -437,28 +445,64 @@ class HP3ParDriverTestCase(test.TestCase):
         expected_capacity = constants.EXPECTED_SIZE_2
         expected_version = self.driver.VERSION
 
-        self.mock_mediator.get_capacity.return_value = {
+        self.mock_mediator.get_fpg_status.return_value = {
             'free_capacity_gb': expected_free,
-            'total_capacity_gb': expected_capacity
+            'total_capacity_gb': expected_capacity,
+            'thin_provisioning': True,
+            'dedupe': False,
+            'hpe3par_flash_cache': False,
         }
 
         expected_result = {
-            'driver_handles_share_servers': False,
             'QoS_support': False,
+            'driver_handles_share_servers': False,
             'driver_version': expected_version,
             'free_capacity_gb': expected_free,
+            'max_over_subscription_ratio': None,
+            'pools': None,
+            'provisioned_capacity_gb': 0,
             'reserved_percentage': 0,
             'share_backend_name': 'HP_3PAR',
             'storage_protocol': 'NFS_CIFS',
             'total_capacity_gb': expected_capacity,
             'vendor_name': 'HP',
-            'pools': None,
+            'thin_provisioning': True,
+            'dedupe': False,
+            'hpe3par_flash_cache': False,
         }
 
         result = self.driver.get_share_stats(refresh=True)
         self.assertEqual(expected_result, result)
 
         expected_calls = [
-            mock.call.get_capacity(constants.EXPECTED_FPG)
+            mock.call.get_fpg_status(constants.EXPECTED_FPG)
         ]
         self.mock_mediator.assert_has_calls(expected_calls)
+        self.assertTrue(self.mock_mediator.get_fpg_status.called)
+
+    def test_driver_get_share_stats_premature(self):
+        """Driver init stats before init_driver completed."""
+
+        expected_version = self.driver.VERSION
+
+        self.mock_mediator.get_fpg_status.return_value = {'not_called': 1}
+
+        expected_result = {
+            'QoS_support': False,
+            'driver_handles_share_servers': False,
+            'driver_version': expected_version,
+            'free_capacity_gb': 0,
+            'max_over_subscription_ratio': None,
+            'pools': None,
+            'provisioned_capacity_gb': 0,
+            'reserved_percentage': 0,
+            'share_backend_name': 'HP_3PAR',
+            'storage_protocol': 'NFS_CIFS',
+            'thin_provisioning': True,
+            'total_capacity_gb': 0,
+            'vendor_name': 'HP',
+        }
+
+        result = self.driver.get_share_stats(refresh=True)
+        self.assertEqual(expected_result, result)
+        self.assertFalse(self.mock_mediator.get_fpg_status.called)

manila/tests/share/drivers/hp/test_hp_3par_mediator.py

@@ -25,6 +25,7 @@ from manila import test
 from manila.tests.share.drivers.hp import test_hp_3par_constants as constants
 
 from oslo_utils import units
+import six
 
 CLIENT_VERSION_MIN_OK = hp3parmediator.MIN_CLIENT_VERSION
 TEST_WSAPI_VERSION_STR = '30201292'
@@ -53,6 +54,8 @@ class HP3ParMediatorTestCase(test.TestCase):
 
         # Set the mediator to use in tests.
         self.mediator = hp3parmediator.HP3ParMediator(
+            hp3par_username=constants.USERNAME,
+            hp3par_password=constants.PASSWORD,
             hp3par_api_url=constants.API_URL,
             hp3par_debug=constants.EXPECTED_HP_DEBUG,
             hp3par_san_ip=constants.EXPECTED_IP_1234,
@@ -155,6 +158,34 @@ class HP3ParMediatorTestCase(test.TestCase):
         ]
         self.mock_client.assert_has_calls(expected_calls)
 
+    def test_mediator_client_login_error(self):
+        """Test exception during login."""
+        self.init_mediator()
+
+        self.mock_client.login.side_effect = constants.FAKE_EXCEPTION
+
+        self.assertRaises(exception.ShareBackendException,
+                          self.mediator._wsapi_login)
+
+        expected_calls = [mock.call.login(constants.USERNAME,
+                                          constants.PASSWORD)]
+        self.mock_client.assert_has_calls(expected_calls)
+
+    def test_mediator_client_logout_error(self):
+        """Test exception during logout."""
+        self.init_mediator()
+
+        mock_log = self.mock_object(hp3parmediator, 'LOG')
+        fake_exception = constants.FAKE_EXCEPTION
+        self.mock_client.http.unauthenticate.side_effect = fake_exception
+
+        self.mediator._wsapi_logout()
+
+        # Warning is logged (no exception thrown).
+        self.assertTrue(mock_log.warning.called)
+        expected_calls = [mock.call.http.unauthenticate()]
+        self.mock_client.assert_has_calls(expected_calls)
+
     def test_mediator_client_version_unsupported(self):
         """Try a client with version less than minimum."""
 
@@ -856,7 +887,8 @@ class HP3ParMediatorTestCase(test.TestCase):
         self.assertTrue(mock_log.debug.called)
         self.assertTrue(mock_log.exception.called)
 
-    def test_mediator_get_capacity(self):
+    @ddt.data(six.text_type('volname.1'), ['volname.2', 'volname.3'])
+    def test_mediator_get_fpg_status(self, volume_name_or_list):
         """Mediator converts client stats to capacity result."""
         expected_capacity = constants.EXPECTED_SIZE_2
         expected_free = constants.EXPECTED_SIZE_1
@@ -867,24 +899,120 @@ class HP3ParMediatorTestCase(test.TestCase):
             'members': [
                 {
                     'capacityKiB': str(expected_capacity * units.Mi),
-                    'availCapacityKiB': str(expected_free * units.Mi)
+                    'availCapacityKiB': str(expected_free * units.Mi),
+                    'vvs': volume_name_or_list,
                 }
             ],
             'message': None,
         }
 
-        expected_result = {
-            'free_capacity_gb': expected_free,
-            'total_capacity_gb': expected_capacity,
+        self.mock_client.getfsquota.return_value = {
+            'total': 3,
+            'members': [
+                {'hardBlock': 1 * units.Ki},
+                {'hardBlock': 2 * units.Ki},
+                {'hardBlock': 3 * units.Ki},
+            ],
+            'message': None,
         }
 
-        result = self.mediator.get_capacity(constants.EXPECTED_FPG)
+        self.mock_client.getVolume.return_value = {
+            'provisioningType': hp3parmediator.DEDUPE}
+
+        expected_result = {
+            'free_capacity_gb': expected_free,
+            'hp3par_flash_cache': False,
+            'dedupe': True,
+            'thin_provisioning': True,
+            'total_capacity_gb': expected_capacity,
+            'provisioned_capacity_gb': 6,
+        }
+
+        result = self.mediator.get_fpg_status(constants.EXPECTED_FPG)
         self.assertEqual(expected_result, result)
         expected_calls = [
             mock.call.getfpg(constants.EXPECTED_FPG)
         ]
         self.mock_client.assert_has_calls(expected_calls)
 
+    def test_mediator_get_fpg_status_exception(self):
+        """Exception during get_fpg_status call to getfpg."""
+        self.init_mediator()
+
+        self.mock_client.getfpg.side_effect = constants.FAKE_EXCEPTION
+
+        self.assertRaises(exception.ShareBackendException,
+                          self.mediator.get_fpg_status,
+                          constants.EXPECTED_FPG)
+
+        expected_calls = [mock.call.getfpg(constants.EXPECTED_FPG)]
+        self.mock_client.assert_has_calls(expected_calls)
+
+    def test_mediator_get_fpg_status_error(self):
+        """Unexpected result from getfpg during get_fpg_status."""
+        self.init_mediator()
+
+        self.mock_client.getfpg.return_value = {'total': 0}
+
+        self.assertRaises(exception.ShareBackendException,
+                          self.mediator.get_fpg_status,
+                          constants.EXPECTED_FPG)
+
+        expected_calls = [mock.call.getfpg(constants.EXPECTED_FPG)]
+        self.mock_client.assert_has_calls(expected_calls)
+
+    def test_mediator_get_fpg_status_bad_prov_type(self):
+        """Test get_fpg_status handling of unexpected provisioning type."""
+        self.init_mediator()
+
+        self.mock_client.getfpg.return_value = {
+            'total': 1,
+            'members': [
+                {
+                    'capacityKiB': '1',
+                    'availCapacityKiB': '1',
+                    'vvs': 'foo',
+                }
+            ],
+            'message': None,
+        }
+        self.mock_client.getVolume.return_value = {
+            'provisioningType': 'BOGUS'}
+
+        self.assertRaises(exception.ShareBackendException,
+                          self.mediator.get_fpg_status,
+                          constants.EXPECTED_FPG)
+
+        expected_calls = [mock.call.getfpg(constants.EXPECTED_FPG)]
+        self.mock_client.assert_has_calls(expected_calls)
+
+    def test_mediator_get_provisioned_error(self):
+        """Test error during get provisioned GB."""
+        self.init_mediator()
+
+        error_return = {'message': 'Some error happened.'}
+        self.mock_client.getfsquota.return_value = error_return
+
+        self.assertRaises(exception.ShareBackendException,
+                          self.mediator.get_provisioned_gb,
+                          constants.EXPECTED_FPG)
+
+        expected_calls = [mock.call.getfsquota(fpg=constants.EXPECTED_FPG)]
+        self.mock_client.assert_has_calls(expected_calls)
+
+    def test_mediator_get_provisioned_exception(self):
+        """Test exception during get provisioned GB."""
+        self.init_mediator()
+
+        self.mock_client.getfsquota.side_effect = constants.FAKE_EXCEPTION
+
+        self.assertRaises(exception.ShareBackendException,
+                          self.mediator.get_provisioned_gb,
+                          constants.EXPECTED_FPG)
+
+        expected_calls = [mock.call.getfsquota(fpg=constants.EXPECTED_FPG)]
+        self.mock_client.assert_has_calls(expected_calls)
+
     def test_mediator_allow_user_access_cifs(self):
         """"Allow user access to cifs share."""
         self.init_mediator()