[NetApp] Fix issues with share server migration

This patch is a follow up patch of the main change[1] that adds
support for share server migration in NetApp driver.
It fixes two issues:
- Data motion 'get_backend_configuration' now avoids sending vserver
  configuration, available only in DHSS=False mode, when driver is
  configured with DHSS=True.
- After migrating a share server, all volumes appears with different
  autosize configuration and need to be manually updated to the
  original values. Provisioning options are also being updated in the
  same method.

[1] https://review.opendev.org/#/c/747048/

Change-Id: I28a47417ec5dda0ed1f6c64fae37f5af6ca057e6
Signed-off-by: Douglas Viroel <viroel@gmail.com>

manila/share/drivers/netapp/dataontap/client/client_cmode.py

@@ -1920,7 +1920,7 @@ class NetAppCmodeClient(client_base.NetAppBaseClient):
                       language=None, dedup_enabled=False,
                       compression_enabled=False, max_files=None,
                       qos_policy_group=None, hide_snapdir=None,
-                      **options):
+                      autosize_attributes=None, **options):
         """Update backend volume for a share as necessary."""
         api_args = {
             'query': {
@@ -1936,6 +1936,9 @@ class NetAppCmodeClient(client_base.NetAppBaseClient):
                     'volume-inode-attributes': {},
                     'volume-language-attributes': {},
                     'volume-snapshot-attributes': {},
+                    'volume-autosize-attributes': (autosize_attributes
+                                                   if autosize_attributes
+                                                   else {}),
                     'volume-space-attributes': {
                         'space-guarantee': ('none' if thin_provisioned else
                                             'volume'),
@@ -2137,6 +2140,26 @@ class NetAppCmodeClient(client_base.NetAppBaseClient):
         result = self.send_iter_request('volume-get-iter', api_args)
         return self._has_records(result)
 
+    @na_utils.trace
+    def get_volume_autosize_attributes(self, volume_name):
+        """Returns autosize attributes for a given volume name."""
+        api_args = {
+            'volume': volume_name,
+        }
+
+        result = self.send_request('volume-autosize-get', api_args)
+        # NOTE(dviroel): 'is-enabled' is deprecated since ONTAP 8.2, use 'mode'
+        # to identify if autosize is enabled or not.
+        return {
+            'mode': result.get_child_content('mode'),
+            'grow-threshold-percent': result.get_child_content(
+                'grow-threshold-percent'),
+            'shrink-threshold-percent': result.get_child_content(
+                'shrink-threshold-percent'),
+            'maximum-size': result.get_child_content('maximum-size'),
+            'minimum-size': result.get_child_content('minimum-size'),
+        }
+
     @na_utils.trace
     def get_volume(self, volume_name):
         """Returns the volume with the specified name, if present."""

manila/share/drivers/netapp/dataontap/cluster_mode/data_motion.py

@@ -55,6 +55,10 @@ def get_backend_configuration(backend_name):
 
     config = configuration.Configuration(driver.share_opts,
                                          config_group=backend_name)
+    if config.driver_handles_share_servers:
+        # NOTE(dviroel): avoid using a pre-create vserver on DHSS == True mode
+        # when retrieving remote backend configuration.
+        config.netapp_vserver = None
     config.append_config_values(na_opts.netapp_cluster_opts)
     config.append_config_values(na_opts.netapp_connection_opts)
     config.append_config_values(na_opts.netapp_basicauth_opts)

manila/share/drivers/netapp/dataontap/cluster_mode/lib_multi_svm.py

@@ -26,6 +26,7 @@ import re
 from oslo_log import log
 from oslo_serialization import jsonutils
 from oslo_utils import excutils
+from oslo_utils import units
 
 from manila import exception
 from manila.i18n import _
@@ -1210,9 +1211,15 @@ class NetAppCmodeMultiSVMFileStorageLibrary(
         share_updates = {}
         for instance in share_instances:
             # Get the volume to find out the associated aggregate
+            # Update post-migration info that can't be replicated
             try:
                 share_name = self._get_backend_share_name(instance['id'])
                 volume = dest_client.get_volume(share_name)
+                dest_aggregate = volume.get('aggregate')
+                # Update share attributes according with share extra specs
+                self._update_share_attributes_after_server_migration(
+                    instance, src_client, dest_aggregate, dest_client)
+
             except Exception:
                 msg_args = {
                     'src': source_share_server['id'],
@@ -1284,3 +1291,30 @@ class NetAppCmodeMultiSVMFileStorageLibrary(
                                             snapshots):
         # TODO(dviroel): get snapmirror info to infer the progress
         return {'total_progress': 0}
+
+    def _update_share_attributes_after_server_migration(
+            self, src_share_instance, src_client, dest_aggregate, dest_client):
+        """Updates destination share instance with share type extra specs."""
+        extra_specs = share_types.get_extra_specs_from_share(
+            src_share_instance)
+        provisioning_options = self._get_provisioning_options(extra_specs)
+        volume_name = self._get_backend_share_name(src_share_instance['id'])
+        # NOTE(dviroel): Need to retrieve current autosize attributes since
+        # they aren't being updated by SVM DR.
+        autosize_attrs = src_client.get_volume_autosize_attributes(volume_name)
+        # NOTE(dviroel): In order to modify maximum and minimum size, we must
+        # convert from Kbytes to bytes.
+        for key in ('minimum-size', 'maximum-size'):
+            autosize_attrs[key] = int(autosize_attrs[key]) * units.Ki
+        provisioning_options['autosize_attributes'] = autosize_attrs
+        # NOTE(dviroel): SVM DR already creates a copy of the snapshot policies
+        # at the destination, using a different name. If we update the snapshot
+        # policy in these volumes, might end up with an error if the policy
+        # still does not exist in the destination cluster. Administrators will
+        # have the opportunity to add the snapshot policy after a successful
+        # migration.
+        provisioning_options.pop('snapshot_policy', None)
+
+        # Modify volume to match extra specs
+        dest_client.modify_volume(dest_aggregate, volume_name,
+                                  **provisioning_options)

manila/tests/share/drivers/netapp/dataontap/client/fakes.py

@@ -167,6 +167,15 @@ QOS_POLICY_GROUP = {
     'num-workloads': 1,
 }
 
+VOLUME_AUTOSIZE_ATTRS = {
+    'mode': 'off',
+    'grow-threshold-percent': '85',
+    'shrink-threshold-percent': '50',
+    'maximum-size': '1258288',
+    'minimum-size': '1048576',
+}
+
+
 NO_RECORDS_RESPONSE = etree.XML("""
   <results status="passed">
     <num-records>0</num-records>
@@ -2007,6 +2016,22 @@ GET_AGGREGATE_FOR_VOLUME_RESPONSE = etree.XML("""
     'share': SHARE_NAME
 })
 
+VOLUME_AUTOSIZE_GET_RESPONSE = etree.XML("""
+  <results status="passed">
+    <grow-threshold-percent>%(grow_percent)s</grow-threshold-percent>
+    <is-enabled>false</is-enabled>
+    <maximum-size>%(max_size)s</maximum-size>
+    <minimum-size>%(min_size)s</minimum-size>
+    <mode>%(mode)s</mode>
+    <shrink-threshold-percent>%(shrink_percent)s</shrink-threshold-percent>
+  </results>
+""" % {'grow_percent': VOLUME_AUTOSIZE_ATTRS.get('grow-threshold-percent'),
+       'max_size': VOLUME_AUTOSIZE_ATTRS.get('maximum-size'),
+       'min_size': VOLUME_AUTOSIZE_ATTRS.get('minimum-size'),
+       'mode': VOLUME_AUTOSIZE_ATTRS.get('mode'),
+       'shrink_percent': VOLUME_AUTOSIZE_ATTRS.get(
+           'shrink-threshold-percent')})
+
 GET_VOLUME_FOR_ENCRYPTED_RESPONSE = etree.XML("""
   <results status="passed">
     <attributes-list>

manila/tests/share/drivers/netapp/dataontap/client/test_client_cmode.py

@@ -3201,6 +3201,7 @@ class NetAppClientCmodeTestCase(test.TestCase):
                     'volume-space-attributes': {
                         'space-guarantee': 'volume',
                     },
+                    'volume-autosize-attributes': {},
                 },
             },
         }
@@ -3216,16 +3217,18 @@ class NetAppClientCmodeTestCase(test.TestCase):
         mock_update_volume_efficiency_attributes = self.mock_object(
             self.client, 'update_volume_efficiency_attributes')
 
-        self.client.modify_volume(fake.SHARE_AGGREGATE_NAME,
+        self.client.modify_volume(
-                                  fake.SHARE_NAME,
+            fake.SHARE_AGGREGATE_NAME,
-                                  thin_provisioned=True,
+            fake.SHARE_NAME,
-                                  snapshot_policy=fake.SNAPSHOT_POLICY_NAME,
+            thin_provisioned=True,
-                                  language=fake.LANGUAGE,
+            snapshot_policy=fake.SNAPSHOT_POLICY_NAME,
-                                  dedup_enabled=True,
+            language=fake.LANGUAGE,
-                                  compression_enabled=False,
+            dedup_enabled=True,
-                                  max_files=fake.MAX_FILES,
+            compression_enabled=False,
-                                  qos_policy_group=fake.QOS_POLICY_GROUP_NAME,
+            max_files=fake.MAX_FILES,
-                                  hide_snapdir=True)
+            qos_policy_group=fake.QOS_POLICY_GROUP_NAME,
+            autosize_attributes=fake.VOLUME_AUTOSIZE_ATTRS,
+            hide_snapdir=True)
 
         volume_modify_iter_api_args = {
             'query': {
@@ -3254,7 +3257,7 @@ class NetAppClientCmodeTestCase(test.TestCase):
                     'volume-qos-attributes': {
                         'policy-group-name': fake.QOS_POLICY_GROUP_NAME,
                     },
-
+                    'volume-autosize-attributes': fake.VOLUME_AUTOSIZE_ATTRS,
                 },
             },
         }
@@ -7424,3 +7427,21 @@ class NetAppClientCmodeTestCase(test.TestCase):
         self.assertEqual(expected_result, result)
         self.client.send_iter_request.assert_called_once_with(
             'cifs-share-get-iter', cifs_share_get_iter_args)
+
+    def test_get_volume_autosize_attributes(self):
+        api_response = netapp_api.NaElement(fake.VOLUME_AUTOSIZE_GET_RESPONSE)
+        self.mock_object(self.client,
+                         'send_request',
+                         mock.Mock(return_value=api_response))
+
+        result = self.client.get_volume_autosize_attributes(fake.SHARE_NAME)
+
+        expected_result = {}
+        expected_keys = ['mode', 'grow-threshold-percent', 'minimum-size',
+                         'shrink-threshold-percent', 'maximum-size']
+        for key in expected_keys:
+            expected_result[key] = fake.VOLUME_AUTOSIZE_ATTRS[key]
+
+        self.assertEqual(expected_result, result)
+        self.client.send_request.assert_called_once_with(
+            'volume-autosize-get', {'volume': fake.SHARE_NAME})

manila/tests/share/drivers/netapp/dataontap/cluster_mode/test_lib_multi_svm.py

@@ -21,6 +21,7 @@ from unittest import mock
 import ddt
 from oslo_log import log
 from oslo_serialization import jsonutils
+from oslo_utils import units
 
 from manila.common import constants
 from manila import context
@@ -2173,6 +2174,8 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
         self.mock_object(self.library, '_create_export',
                          mock.Mock(return_value=fake.NFS_EXPORTS))
         self.mock_object(self.library, '_delete_share')
+        mock_update_share_attrs = self.mock_object(
+            self.library, '_update_share_attributes_after_server_migration')
 
         result = self.library.share_server_migration_complete(
             None,
@@ -2229,6 +2232,9 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
         )
         self.mock_dest_client.get_volume.assert_called_once_with(
             fake_share_name)
+        mock_update_share_attrs.assert_called_once_with(
+            fake.SHARE_INSTANCE, self.mock_src_client,
+            fake_volume['aggregate'], self.mock_dest_client)
         self.library._delete_share.assert_called_once_with(
             fake.SHARE_INSTANCE, self.mock_src_client, remove_export=True)
 
@@ -2505,3 +2511,36 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
         self.library._get_node_data_port.assert_has_calls(get_node_port_calls)
         self.library._client.create_port_and_broadcast_domain.assert_has_calls(
             create_port_calls)
+
+    def test___update_share_attributes_after_server_migration(self):
+        fake_aggregate = 'fake_aggr_0'
+        mock_get_extra_spec = self.mock_object(
+            share_types, "get_extra_specs_from_share",
+            mock.Mock(return_value=fake.EXTRA_SPEC))
+        mock__get_provisioning_opts = self.mock_object(
+            self.library, '_get_provisioning_options',
+            mock.Mock(return_value=copy.deepcopy(fake.PROVISIONING_OPTIONS)))
+        fake_share_name = self.library._get_backend_share_name(
+            fake.SHARE_INSTANCE['id'])
+        mock_get_vol_autosize_attrs = self.mock_object(
+            self.mock_src_client, 'get_volume_autosize_attributes',
+            mock.Mock(return_value=fake.VOLUME_AUTOSIZE_ATTRS)
+        )
+        fake_provisioning_opts = copy.copy(fake.PROVISIONING_OPTIONS)
+        fake_autosize_attrs = copy.copy(fake.VOLUME_AUTOSIZE_ATTRS)
+        for key in ('minimum-size', 'maximum-size'):
+            fake_autosize_attrs[key] = int(fake_autosize_attrs[key]) * units.Ki
+        fake_provisioning_opts['autosize_attributes'] = fake_autosize_attrs
+        mock_modify_volume = self.mock_object(self.mock_dest_client,
+                                              'modify_volume')
+        fake_provisioning_opts.pop('snapshot_policy', None)
+
+        self.library._update_share_attributes_after_server_migration(
+            fake.SHARE_INSTANCE, self.mock_src_client, fake_aggregate,
+            self.mock_dest_client)
+
+        mock_get_extra_spec.assert_called_once_with(fake.SHARE_INSTANCE)
+        mock__get_provisioning_opts.assert_called_once_with(fake.EXTRA_SPEC)
+        mock_get_vol_autosize_attrs.assert_called_once_with(fake_share_name)
+        mock_modify_volume.assert_called_once_with(
+            fake_aggregate, fake_share_name, **fake_provisioning_opts)

manila/tests/share/drivers/netapp/dataontap/fakes.py

@@ -386,6 +386,14 @@ REMAPPED_OVERLAPPING_EXTRA_SPEC = {
     'netapp:thin_provisioned': 'false',
 }
 
+VOLUME_AUTOSIZE_ATTRS = {
+    'mode': 'off',
+    'grow-threshold-percent': '85',
+    'shrink-threshold-percent': '50',
+    'maximum-size': '1258288',
+    'minimum-size': '1048576',
+}
+
 USER_NETWORK_ALLOCATIONS = [
     {
         'id': '132dbb10-9a36-46f2-8d89-3d909830c356',