Merge "Add interface port configuration in EMC VNX driver"

doc/source/devref/emc_vnx_driver.rst

@@ -187,6 +187,7 @@ for the VNX driver:
     emc_nas_login = <user>
     emc_nas_server_container = <Data Mover name>
     emc_nas_pool_name = <pool name>
+    emc_interface_ports = <Comma separated ports list>
     share_driver = manila.share.drivers.emc.driver.EMCShareDriver
 
 - `emc_share_backend` is the plugin name. Set it to `vnx` for the VNX driver.
@@ -198,6 +199,11 @@ for the VNX driver:
   share service.
 - `emc_nas_pool_name` is the pool name user wants to create volume from. The
   pools can be created using Unisphere for VNX.
+- `emc_interface_ports` is comma separated list specifying the ports(devices) of
+  Data Mover that can be used for share server interface.
+  Members of the list can be Unix-style glob expressions (supports Unix shell-style
+  wildcards). This list is optional. In the absence of this option, any of the ports
+  on the Data Mover can be used.
 
 Restart of :term:`manila-share` service is needed for the configuration changes to take
 effect.

manila/share/drivers/emc/driver.py

@@ -53,6 +53,10 @@ EMC_NAS_OPTS = [
                help='EMC pool names.'),
     cfg.StrOpt('emc_nas_root_dir',
                help='The root directory where shares will be located.'),
+    cfg.ListOpt('emc_interface_ports',
+                help='Comma separated list specifying the ports that can be '
+                     'used for share server interfaces. Members of the list '
+                     'can be Unix-style glob expressions.'),
 ]
 
 CONF = cfg.CONF

manila/share/drivers/emc/plugins/vnx/connection.py

@@ -19,13 +19,13 @@ import random
 
 from oslo_log import log
 from oslo_utils import excutils
-from oslo_utils import fnmatch
 from oslo_utils import units
 
 from manila.common import constants as const
 from manila import exception
 from manila.i18n import _
 from manila.i18n import _LE
+from manila.i18n import _LI
 from manila.i18n import _LW
 from manila.share.drivers.emc.plugins import base as driver
 from manila.share.drivers.emc.plugins.vnx import constants
@@ -53,6 +53,7 @@ class VNXStorageConnection(driver.StorageConnection):
         self.pool_conf = None
         self.reserved_percentage = None
         self.driver_handles_share_servers = True
+        self.port_conf = None
 
     def create_share(self, context, share, share_server=None):
         """Create a share and export it based on protocol used."""
@@ -458,33 +459,20 @@ class VNXStorageConnection(driver.StorageConnection):
                 raise exception.EMCVnxXMLAPIError(err=message)
 
             real_pools = set([item for item in backend_pools])
-
             conf_pools = set([item.strip() for item in pools.split(",")])
-
+            matched_pools, unmatched_pools = vnx_utils.do_match_any(
-            for pool in real_pools:
+                real_pools, conf_pools)
-                for matcher in conf_pools:
-                    if fnmatch.fnmatchcase(pool, matcher):
-                        matched_pools.add(pool)
-
-            nonexistent_pools = real_pools.difference(matched_pools)
 
             if not matched_pools:
-                msg = (_("All the specified storage pools to be managed "
+                msg = (_("None of the specified storage pools to be managed "
-                         "do not exist. Please check your configuration "
+                         "exist. Please check your configuration "
                          "emc_nas_pool_names in manila.conf. "
-                         "The available pools in the backend are %s") %
+                         "The available pools in the backend are %s.") %
                        ",".join(real_pools))
                 raise exception.InvalidParameterValue(err=msg)
-            if nonexistent_pools:
+
-                LOG.warning(_LW("The following specified storage pools "
+            LOG.info(_LI("Storage pools: %s will be managed."),
-                                "do not exist: %(unexist)s. "
+                     ",".join(matched_pools))
-                                "This host will only manage the storage "
-                                "pools: %(exist)s"),
-                            {'unexist': ",".join(nonexistent_pools),
-                             'exist': ",".join(matched_pools)})
-            else:
-                LOG.debug("Storage pools: %s will be managed.",
-                          ",".join(matched_pools))
         else:
             LOG.debug("No storage pool is specified, so all pools "
                       "in storage system will be managed.")
@@ -506,6 +494,32 @@ class VNXStorageConnection(driver.StorageConnection):
         configuration = emc_share_driver.configuration
 
         self.manager = manager.StorageObjectManager(configuration)
+        self.port_conf = emc_share_driver.configuration.safe_get(
+            'emc_interface_ports')
+
+    def get_managed_ports(self):
+        # Get the real ports(devices) list from the backend storage
+        real_ports = self._get_physical_devices(self.mover_name)
+
+        if not self.port_conf:
+            LOG.debug("No ports are specified, so any of the ports on the "
+                      "Data Mover can be used.")
+            return real_ports
+
+        matched_ports, unmanaged_ports = vnx_utils.do_match_any(
+            real_ports, self.port_conf)
+
+        if not matched_ports:
+            msg = (_("None of the specified network ports exist. "
+                     "Please check your configuration emc_interface_ports "
+                     "in manila.conf. The available ports on the Data Mover "
+                     "are %s.") %
+                   ",".join(real_ports))
+            raise exception.BadConfigurationException(reason=msg)
+
+        LOG.debug("Ports: %s can be used.", ",".join(matched_ports))
+
+        return list(matched_ports)
 
     def update_share_stats(self, stats_dict):
         """Communicate with EMCNASClient to get the stats."""
@@ -593,7 +607,7 @@ class VNXStorageConnection(driver.StorageConnection):
 
             netmask = utils.cidr_to_netmask(network_info['cidr'])
 
-            devices = self._get_physical_devices(self.mover_name)
+            devices = self.get_managed_ports()
 
             for net_info in network_info['network_allocations']:
                 random.shuffle(devices)

manila/share/drivers/emc/plugins/vnx/utils.py

@@ -17,6 +17,7 @@ import types
 
 from oslo_config import cfg
 from oslo_log import log
+from oslo_utils import fnmatch
 from oslo_utils import timeutils
 
 CONF = cfg.CONF
@@ -58,3 +59,25 @@ def log_enter_exit(func):
         return ret
 
     return inner
+
+
+def do_match_any(full, matcher_list):
+    """Finds items that match any of the matchers.
+
+    :param full: Full item list
+    :param matcher_list: The list of matchers. Each matcher supports
+                         Unix shell-style wildcards
+    :return: The matched items set and the unmatched items set
+    """
+    matched = set()
+    not_matched = set()
+
+    full = set([item.strip() for item in full])
+    matcher_list = set([item.strip() for item in matcher_list])
+
+    for matcher in matcher_list:
+        for item in full:
+            if fnmatch.fnmatchcase(item, matcher):
+                matched.add(item)
+    not_matched = full - matched
+    return matched, not_matched

manila/tests/share/drivers/emc/plugins/vnx/fakes.py

@@ -1039,7 +1039,7 @@ class MoverTestData(StorageObjectTestData):
             '    Link: Down\n'
             '    0:  cge-1-3  IRQ: 27\n'
             '    speed=auto duplex=auto txflowctl=disable rxflowctl=disable\n'
-            '    Link: Down\n'
+            '    Link: Up\n'
             'Slot: 4\n'
             '  PLX PCI-Express Switch  Controller\n'
             '    1:  PLX PEX8648  IRQ: 10\n'

manila/tests/share/drivers/emc/plugins/vnx/test_connection.py

@@ -606,6 +606,7 @@ class StorageConnectionTestCase(test.TestCase):
         ]
         xml_req_mock.assert_has_calls(expected_calls)
 
+    @utils.patch_get_managed_ports(return_value=['cge-1-0'])
     def test_setup_server(self):
         hook = utils.RequestSideEffect()
         hook.append(self.vdm.resp_get_but_not_found())
@@ -620,7 +621,6 @@ class StorageConnectionTestCase(test.TestCase):
         self.connection.manager.connectors['XML'].request = xml_req_mock
 
         ssh_hook = utils.SSHSideEffect()
-        ssh_hook.append(self.mover.output_get_physical_devices())
         ssh_hook.append()
         ssh_cmd_mock = mock.Mock(side_effect=ssh_hook)
         self.connection.manager.connectors['SSH'].run_ssh = ssh_cmd_mock
@@ -647,11 +647,11 @@ class StorageConnectionTestCase(test.TestCase):
         xml_req_mock.assert_has_calls(expected_calls)
 
         ssh_calls = [
-            mock.call(self.mover.cmd_get_physical_devices(), False),
             mock.call(self.vdm.cmd_attach_nfs_interface(), False),
         ]
         ssh_cmd_mock.assert_has_calls(ssh_calls)
 
+    @utils.patch_get_managed_ports(return_value=['cge-1-0'])
     def test_setup_server_with_existing_vdm(self):
         hook = utils.RequestSideEffect()
         hook.append(self.vdm.resp_get_succeed())
@@ -664,11 +664,9 @@ class StorageConnectionTestCase(test.TestCase):
         self.connection.manager.connectors['XML'].request = xml_req_mock
 
         ssh_hook = utils.SSHSideEffect()
-        ssh_hook.append(self.mover.output_get_physical_devices())
         ssh_hook.append()
         ssh_cmd_mock = mock.Mock(side_effect=ssh_hook)
         self.connection.manager.connectors['SSH'].run_ssh = ssh_cmd_mock
-
         self.connection.setup_server(fakes.NETWORK_INFO, None)
 
         if_name_1 = fakes.FakeData.network_allocations_id1[-12:]
@@ -689,7 +687,6 @@ class StorageConnectionTestCase(test.TestCase):
         xml_req_mock.assert_has_calls(expected_calls)
 
         ssh_calls = [
-            mock.call(self.mover.cmd_get_physical_devices(), False),
             mock.call(self.vdm.cmd_attach_nfs_interface(), False),
         ]
         ssh_cmd_mock.assert_has_calls(ssh_calls)
@@ -702,6 +699,8 @@ class StorageConnectionTestCase(test.TestCase):
                           self.connection.setup_server,
                           network_info, None)
 
+    @utils.patch_get_managed_ports(
+        side_effect=exception.EMCVnxXMLAPIError())
     def test_setup_server_without_valid_physical_device(self):
         hook = utils.RequestSideEffect()
         hook.append(self.vdm.resp_get_but_not_found())
@@ -712,9 +711,7 @@ class StorageConnectionTestCase(test.TestCase):
         hook.append(self.vdm.resp_task_succeed())
         xml_req_mock = utils.EMCMock(side_effect=hook)
         self.connection.manager.connectors['XML'].request = xml_req_mock
-
         ssh_hook = utils.SSHSideEffect()
-        ssh_hook.append(self.mover.fake_output)
         ssh_hook.append(self.vdm.output_get_interfaces(nfs_interface=''))
         ssh_cmd_mock = mock.Mock(side_effect=ssh_hook)
         self.connection.manager.connectors['SSH'].run_ssh = ssh_cmd_mock
@@ -734,11 +731,11 @@ class StorageConnectionTestCase(test.TestCase):
         xml_req_mock.assert_has_calls(expected_calls)
 
         ssh_calls = [
-            mock.call(self.mover.cmd_get_physical_devices(), False),
             mock.call(self.vdm.cmd_get_interfaces(), False),
         ]
         ssh_cmd_mock.assert_has_calls(ssh_calls)
 
+    @utils.patch_get_managed_ports(return_value=['cge-1-0'])
     def test_setup_server_with_exception(self):
         hook = utils.RequestSideEffect()
         hook.append(self.vdm.resp_get_but_not_found())
@@ -754,7 +751,6 @@ class StorageConnectionTestCase(test.TestCase):
         self.connection.manager.connectors['XML'].request = xml_req_mock
 
         ssh_hook = utils.SSHSideEffect()
-        ssh_hook.append(self.mover.output_get_physical_devices())
         ssh_hook.append(self.vdm.output_get_interfaces(nfs_interface=''))
         ssh_cmd_mock = mock.Mock(side_effect=ssh_hook)
         self.connection.manager.connectors['SSH'].run_ssh = ssh_cmd_mock
@@ -785,7 +781,6 @@ class StorageConnectionTestCase(test.TestCase):
         xml_req_mock.assert_has_calls(expected_calls)
 
         ssh_calls = [
-            mock.call(self.mover.cmd_get_physical_devices(), False),
             mock.call(self.vdm.cmd_get_interfaces(), False),
         ]
         ssh_cmd_mock.assert_has_calls(ssh_calls)
@@ -1407,3 +1402,48 @@ class StorageConnectionTestCase(test.TestCase):
             mock.call(self.pool.req_get()),
         ]
         xml_req_mock.assert_has_calls(expected_calls)
+
+    @ddt.data({'port_conf': None,
+               'managed_ports': ['cge-1-0', 'cge-1-3']},
+              {'port_conf': '*',
+               'managed_ports': ['cge-1-0', 'cge-1-3']},
+              {'port_conf': ['cge-1-*'],
+               'managed_ports': ['cge-1-0', 'cge-1-3']},
+              {'port_conf': ['cge-1-3'],
+               'managed_ports': ['cge-1-3']})
+    @ddt.unpack
+    def test_get_managed_ports_one_port(self, port_conf, managed_ports):
+        hook = utils.SSHSideEffect()
+        hook.append(self.mover.output_get_physical_devices())
+
+        ssh_cmd_mock = mock.Mock(side_effect=hook)
+        expected_calls = [
+            mock.call(self.mover.cmd_get_physical_devices(), False),
+        ]
+        self.connection.manager.connectors['SSH'].run_ssh = ssh_cmd_mock
+        self.connection.port_conf = port_conf
+        ports = self.connection.get_managed_ports()
+        self.assertIsInstance(ports, list)
+        self.assertEqual(sorted(managed_ports), sorted(ports))
+        ssh_cmd_mock.assert_has_calls(expected_calls)
+
+    def test_get_managed_ports_no_valid_port(self):
+        hook = utils.SSHSideEffect()
+        hook.append(self.mover.output_get_physical_devices())
+
+        ssh_cmd_mock = mock.Mock(side_effect=hook)
+        self.connection.manager.connectors['SSH'].run_ssh = ssh_cmd_mock
+        self.connection.port_conf = ['cge-2-0']
+
+        self.assertRaises(exception.BadConfigurationException,
+                          self.connection.get_managed_ports)
+
+    def test_get_managed_ports_query_devices_failed(self):
+        hook = utils.SSHSideEffect()
+        hook.append(self.mover.fake_output)
+        ssh_cmd_mock = mock.Mock(side_effect=hook)
+        self.connection.manager.connectors['SSH'].run_ssh = ssh_cmd_mock
+        self.connection.port_conf = ['cge-2-0']
+
+        self.assertRaises(exception.EMCVnxXMLAPIError,
+                          self.connection.get_managed_ports)

manila/tests/share/drivers/emc/plugins/vnx/test_utils.py

@@ -0,0 +1,44 @@
+# Copyright (c) 2016 EMC Corporation.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import ddt
+
+from manila.share.drivers.emc.plugins.vnx import utils
+from manila import test
+
+
+@ddt.ddt
+class VNXUtilsTestCase(test.TestCase):
+
+    @ddt.data({'full': ['cge-1-0', 'cge-1-1', 'cge-3-0',
+                        'cge-3-1', 'cge-12-3'],
+               'matchers': ['cge-?-0', 'cge-3*', 'foo'],
+               'matched': set(['cge-1-0', 'cge-3-0',
+                               'cge-3-1']),
+               'unmatched': set(['cge-1-1', 'cge-12-3'])},
+              {'full': ['cge-1-0', 'cge-1-1'],
+               'matchers': ['cge-1-0'],
+               'matched': set(['cge-1-0']),
+               'unmatched': set(['cge-1-1'])},
+              {'full': ['cge-1-0', 'cge-1-1'],
+               'matchers': ['foo'],
+               'matched': set([]),
+               'unmatched': set(['cge-1-0', 'cge-1-1'])})
+    @ddt.unpack
+    def test_do_match_any(self, full, matchers, matched, unmatched):
+        real_matched, real_unmatched = utils.do_match_any(
+            full, matchers)
+        self.assertEqual(matched, real_matched)
+        self.assertEqual(unmatched, real_unmatched)

manila/tests/share/drivers/emc/plugins/vnx/utils.py

@@ -150,3 +150,9 @@ class EMCNFSShareMock(mock.Mock):
                     return False
 
         return True
+
+
+def patch_get_managed_ports(*arg, **kwargs):
+    return mock.patch('manila.share.drivers.emc.plugins.vnx.connection.'
+                      'VNXStorageConnection.get_managed_ports',
+                      mock.Mock(*arg, **kwargs))

releasenotes/notes/emc_vnx_interface_ports_configuration-00d454b3003ef981.yaml

@@ -0,0 +1,5 @@
+---
+fixes:
+  - EMC VNX driver supports interface ports configuration now.
+    The ports of Data Mover that can be used by share server
+    interfaces are configurable.