Adding InfiniBand Support

InfiniBand is computer-networking communications standard
used in high-performance computing, features very high
throughput and very low latency. Where ethernet uses MAC
as unique identifier assigned to network interfaces,
InfiniBand uses GUID. The difference is
that MAC is 6 bytes and GID is 8 bytes. Moreover to be
able to PXE boot on InfiniBand network we should use DHCP
over InfiniBand https://tools.ietf.org/html/rfc4390.
The major changes to allow it is to generate client-id per
GID and add it as DHCP option to the neutron port.

This patch update the neutron port with CLient-ID
DHCP option when ironic port.extra has client-id
paramater.

Closes-Bug: #1532534

Change-Id: Ifad453977e5d3be64b34e544f269835a72b4d73f

ironic/common/neutron.py

@@ -121,6 +121,12 @@ def add_ports_to_network(task, network_uuid, is_flat=False):
         binding_profile = {'local_link_information':
                            [portmap[ironic_port.uuid]]}
         body['port']['binding:profile'] = binding_profile
+        client_id = ironic_port.extra.get('client-id')
+        if client_id:
+            client_id_opt = {'opt_name': 'client-id', 'opt_value': client_id}
+            extra_dhcp_opts = body['port'].get('extra_dhcp_opts', [])
+            extra_dhcp_opts.append(client_id_opt)
+            body['port']['extra_dhcp_opts'] = extra_dhcp_opts
         try:
             port = client.create_port(body)
         except neutron_exceptions.NeutronClientException as e:

ironic/common/pxe_utils.py

@@ -27,7 +27,6 @@ from ironic.common import exception
 from ironic.common.i18n import _
 from ironic.common import utils
 from ironic.drivers.modules import deploy_utils
-from ironic.drivers import utils as driver_utils
 
 CONF = cfg.CONF
 
@@ -91,8 +90,9 @@ def _link_mac_pxe_configs(task):
         utils.create_link_without_raise(relative_source_path, mac_path)
 
     pxe_config_file_path = get_pxe_config_file_path(task.node.uuid)
-    for mac in driver_utils.get_node_mac_addresses(task):
+    for port in task.ports:
-        create_link(_get_pxe_mac_path(mac))
+        client_id = port.extra.get('client-id')
+        create_link(_get_pxe_mac_path(port.address, client_id=client_id))
 
 
 def _link_ip_address_pxe_configs(task, hex_form):
@@ -123,17 +123,22 @@ def _link_ip_address_pxe_configs(task, hex_form):
                                         ip_address_path)
 
 
-def _get_pxe_mac_path(mac, delimiter='-'):
+def _get_pxe_mac_path(mac, delimiter='-', client_id=None):
     """Convert a MAC address into a PXE config file name.
 
     :param mac: A MAC address string in the format xx:xx:xx:xx:xx:xx.
     :param delimiter: The MAC address delimiter. Defaults to dash ('-').
+    :param client_id: client_id indicate InfiniBand port.
+                      Defaults is None (Ethernet)
     :returns: the path to the config file.
 
     """
     mac_file_name = mac.replace(':', delimiter).lower()
     if not CONF.pxe.ipxe_enabled:
-        mac_file_name = '01-' + mac_file_name
+        hw_type = '01-'
+        if client_id:
+            hw_type = '20-'
+        mac_file_name = hw_type + mac_file_name
 
     return os.path.join(get_root_dir(), PXE_CFG_DIR_NAME, mac_file_name)
 
@@ -273,8 +278,10 @@ def clean_up_pxe_config(task):
             # Cleaning up config files created for elilo.
             ironic_utils.unlink_without_raise(hex_ip_path)
     else:
-        for mac in driver_utils.get_node_mac_addresses(task):
+        for port in task.ports:
-            ironic_utils.unlink_without_raise(_get_pxe_mac_path(mac))
+            client_id = port.extra.get('client-id')
+            ironic_utils.unlink_without_raise(
+                _get_pxe_mac_path(port.address, client_id=client_id))
 
     utils.rmtree_without_raise(os.path.join(get_root_dir(),
                                             task.node.uuid))

ironic/conductor/manager.py

@@ -1630,7 +1630,8 @@ class ConductorManager(base_manager.BaseConductorManager):
     @messaging.expected_exceptions(exception.NodeLocked,
                                    exception.FailedToUpdateMacOnPort,
                                    exception.MACAlreadyExists,
-                                   exception.InvalidState)
+                                   exception.InvalidState,
+                                   exception.FailedToUpdateDHCPOptOnPort)
     def update_port(self, context, port_obj):
         """Update a port.
 
@@ -1703,6 +1704,31 @@ class ConductorManager(base_manager.BaseConductorManager):
                         "address."),
                         {'port': port_uuid, 'instance': node.instance_uuid})
 
+            if 'extra' in port_obj.obj_what_changed():
+                orignal_port = objects.Port.get_by_id(context, port_obj.id)
+                updated_client_id = port_obj.extra.get('client-id')
+                if (orignal_port.extra.get('client-id') !=
+                    updated_client_id):
+                    vif = port_obj.extra.get('vif_port_id')
+                    # DHCP Option with opt_value=None will remove it
+                    # from the neutron port
+                    if vif:
+                        api = dhcp_factory.DHCPFactory()
+                        client_id_opt = {'opt_name': 'client-id',
+                                         'opt_value': updated_client_id}
+
+                        api.provider.update_port_dhcp_opts(
+                            vif, [client_id_opt], token=context.auth_token)
+                    # Log warning if there is no vif_port_id and an instance
+                    # is associated with the node.
+                    elif node.instance_uuid:
+                        LOG.warning(_LW(
+                            "No VIF found for instance %(instance)s "
+                            "port %(port)s when attempting to update port "
+                            "client-id."),
+                            {'port': port_uuid,
+                             'instance': node.instance_uuid})
+
             port_obj.save()
 
             return port_obj

ironic/drivers/modules/network/neutron.py

@@ -162,6 +162,7 @@ class NeutronNetwork(base.NetworkInterface):
                       '%(node_id)s',
                       {'vif_port_id': vif_port_id, 'node_id': node.uuid})
             local_link_info = []
+            client_id_opt = None
             if isinstance(port_like_obj, objects.Portgroup):
                 pg_ports = [p for p in task.ports
                             if p.portgroup_id == port_like_obj.id]
@@ -171,6 +172,10 @@ class NeutronNetwork(base.NetworkInterface):
                 # We iterate only on ports or portgroups, no need to check
                 # that it is a port
                 local_link_info.append(portmap[port_like_obj.uuid])
+                client_id = port_like_obj.extra.get('client-id')
+                if client_id:
+                    client_id_opt = (
+                        {'opt_name': 'client-id', 'opt_value': client_id})
             body = {
                 'port': {
                     'device_owner': 'baremetal:none',
@@ -183,6 +188,8 @@ class NeutronNetwork(base.NetworkInterface):
                     },
                 }
             }
+            if client_id_opt:
+                body['port']['extra_dhcp_opts'] = [client_id_opt]
 
             try:
                 client.update_port(vif_port_id, body)

ironic/tests/unit/common/test_neutron.py

@@ -112,6 +112,9 @@ class TestNeutronClient(base.TestCase):
 
 class TestNeutronNetworkActions(db_base.DbTestCase):
 
+    _CLIENT_ID = (
+        '20:00:55:04:01:fe:80:00:00:00:00:00:00:00:02:c9:02:00:23:13:92')
+
     def setUp(self):
         super(TestNeutronNetworkActions, self).setUp()
         mgr_utils.mock_the_extension_manager(driver='fake')
@@ -133,7 +136,7 @@ class TestNeutronNetworkActions(db_base.DbTestCase):
         patcher.start()
         self.addCleanup(patcher.stop)
 
-    def test_add_ports_to_vlan_network(self):
+    def _test_add_ports_to_vlan_network(self, is_client_id):
         # Ports will be created only if pxe_enabled is True
         object_utils.create_test_port(
             self.context, node_id=self.node.id,
@@ -142,6 +145,11 @@ class TestNeutronNetworkActions(db_base.DbTestCase):
             pxe_enabled=False
         )
         port = self.ports[0]
+        if is_client_id:
+            extra = port.extra
+            extra['client-id'] = self._CLIENT_ID
+            port.extra = extra
+            port.save()
         expected_body = {
             'port': {
                 'network_id': self.network_uuid,
@@ -156,6 +164,9 @@ class TestNeutronNetworkActions(db_base.DbTestCase):
                 }
             }
         }
+        if is_client_id:
+            expected_body['port']['extra_dhcp_opts'] = (
+                [{'opt_name': 'client-id', 'opt_value': self._CLIENT_ID}])
         # Ensure we can create ports
         self.client_mock.create_port.return_value = {
             'port': self.neutron_port}
@@ -166,8 +177,19 @@ class TestNeutronNetworkActions(db_base.DbTestCase):
             self.client_mock.create_port.assert_called_once_with(
                 expected_body)
 
-    def test_add_ports_to_flat_network(self):
+    def test_add_ports_to_vlan_network(self):
+        self._test_add_ports_to_vlan_network(is_client_id=False)
+
+    def test_add_ports_with_client_id_to_vlan_network(self):
+        self._test_add_ports_to_vlan_network(is_client_id=True)
+
+    def _test_add_ports_to_flat_network(self, is_client_id):
         port = self.ports[0]
+        if is_client_id:
+            extra = port.extra
+            extra['client-id'] = self._CLIENT_ID
+            port.extra = extra
+            port.save()
         expected_body = {
             'port': {
                 'network_id': self.network_uuid,
@@ -181,6 +203,9 @@ class TestNeutronNetworkActions(db_base.DbTestCase):
                 }
             }
         }
+        if is_client_id:
+            expected_body['port']['extra_dhcp_opts'] = (
+                [{'opt_name': 'client-id', 'opt_value': self._CLIENT_ID}])
         # Ensure we can create ports
         self.client_mock.create_port.return_value = {
             'port': self.neutron_port}
@@ -192,6 +217,12 @@ class TestNeutronNetworkActions(db_base.DbTestCase):
             self.client_mock.create_port.assert_called_once_with(
                 expected_body)
 
+    def test_add_ports_to_flat_network(self):
+        self._test_add_ports_to_flat_network(is_client_id=False)
+
+    def test_add_ports_with_client_id_to_flat_network(self):
+        self._test_add_ports_to_flat_network(is_client_id=True)
+
     def test_add_ports_to_flat_network_no_neutron_port_id(self):
         port = self.ports[0]
         expected_body = {

ironic/tests/unit/common/test_pxe_utils.py

@@ -18,6 +18,7 @@ import os
 
 import mock
 from oslo_config import cfg
+from oslo_utils import uuidutils
 import six
 
 from ironic.common import pxe_utils
@@ -199,25 +200,25 @@ class TestPXEUtils(db_base.DbTestCase):
 
     @mock.patch('ironic.common.utils.create_link_without_raise', autospec=True)
     @mock.patch('ironic_lib.utils.unlink_without_raise', autospec=True)
-    @mock.patch('ironic.drivers.utils.get_node_mac_addresses', autospec=True)
+    def test__write_mac_pxe_configs(self, unlink_mock, create_link_mock):
-    def test__write_mac_pxe_configs(self, get_macs_mock, unlink_mock,
+        port_1 = object_utils.create_test_port(
-                                    create_link_mock):
+            self.context, node_id=self.node.id,
-        macs = [
+            address='11:22:33:44:55:66', uuid=uuidutils.generate_uuid())
-            '00:11:22:33:44:55:66',
+        port_2 = object_utils.create_test_port(
-            '00:11:22:33:44:55:67'
+            self.context, node_id=self.node.id,
-        ]
+            address='11:22:33:44:55:67', uuid=uuidutils.generate_uuid())
-        get_macs_mock.return_value = macs
         create_link_calls = [
             mock.call(u'../1be26c0b-03f2-4d2e-ae87-c02d7f33c123/config',
-                      '/tftpboot/pxelinux.cfg/01-00-11-22-33-44-55-66'),
+                      '/tftpboot/pxelinux.cfg/01-11-22-33-44-55-66'),
             mock.call(u'../1be26c0b-03f2-4d2e-ae87-c02d7f33c123/config',
-                      '/tftpboot/pxelinux.cfg/01-00-11-22-33-44-55-67')
+                      '/tftpboot/pxelinux.cfg/01-11-22-33-44-55-67')
         ]
         unlink_calls = [
-            mock.call('/tftpboot/pxelinux.cfg/01-00-11-22-33-44-55-66'),
+            mock.call('/tftpboot/pxelinux.cfg/01-11-22-33-44-55-66'),
-            mock.call('/tftpboot/pxelinux.cfg/01-00-11-22-33-44-55-67'),
+            mock.call('/tftpboot/pxelinux.cfg/01-11-22-33-44-55-67'),
         ]
         with task_manager.acquire(self.context, self.node.uuid) as task:
+            task.ports = [port_1, port_2]
             pxe_utils._link_mac_pxe_configs(task)
 
         unlink_mock.assert_has_calls(unlink_calls)
@@ -225,26 +226,59 @@ class TestPXEUtils(db_base.DbTestCase):
 
     @mock.patch('ironic.common.utils.create_link_without_raise', autospec=True)
     @mock.patch('ironic_lib.utils.unlink_without_raise', autospec=True)
-    @mock.patch('ironic.drivers.utils.get_node_mac_addresses', autospec=True)
+    def test__write_infiniband_mac_pxe_configs(
-    def test__write_mac_ipxe_configs(self, get_macs_mock, unlink_mock,
+            self, unlink_mock, create_link_mock):
-                                     create_link_mock):
+        client_id1 = (
-        self.config(ipxe_enabled=True, group='pxe')
+            '20:00:55:04:01:fe:80:00:00:00:00:00:00:00:02:c9:02:00:23:13:92')
-        macs = [
+        port_1 = object_utils.create_test_port(
-            '00:11:22:33:44:55:66',
+            self.context, node_id=self.node.id,
-            '00:11:22:33:44:55:67'
+            address='11:22:33:44:55:66', uuid=uuidutils.generate_uuid(),
-        ]
+            extra={'client-id': client_id1})
-        get_macs_mock.return_value = macs
+        client_id2 = (
+            '20:00:55:04:01:fe:80:00:00:00:00:00:00:00:02:c9:02:00:23:45:12')
+        port_2 = object_utils.create_test_port(
+            self.context, node_id=self.node.id,
+            address='11:22:33:44:55:67', uuid=uuidutils.generate_uuid(),
+            extra={'client-id': client_id2})
         create_link_calls = [
             mock.call(u'../1be26c0b-03f2-4d2e-ae87-c02d7f33c123/config',
-                      '/httpboot/pxelinux.cfg/00-11-22-33-44-55-66'),
+                      '/tftpboot/pxelinux.cfg/20-11-22-33-44-55-66'),
             mock.call(u'../1be26c0b-03f2-4d2e-ae87-c02d7f33c123/config',
-                      '/httpboot/pxelinux.cfg/00-11-22-33-44-55-67'),
+                      '/tftpboot/pxelinux.cfg/20-11-22-33-44-55-67')
         ]
         unlink_calls = [
-            mock.call('/httpboot/pxelinux.cfg/00-11-22-33-44-55-66'),
+            mock.call('/tftpboot/pxelinux.cfg/20-11-22-33-44-55-66'),
-            mock.call('/httpboot/pxelinux.cfg/00-11-22-33-44-55-67'),
+            mock.call('/tftpboot/pxelinux.cfg/20-11-22-33-44-55-67'),
         ]
         with task_manager.acquire(self.context, self.node.uuid) as task:
+            task.ports = [port_1, port_2]
+            pxe_utils._link_mac_pxe_configs(task)
+
+        unlink_mock.assert_has_calls(unlink_calls)
+        create_link_mock.assert_has_calls(create_link_calls)
+
+    @mock.patch('ironic.common.utils.create_link_without_raise', autospec=True)
+    @mock.patch('ironic_lib.utils.unlink_without_raise', autospec=True)
+    def test__write_mac_ipxe_configs(self, unlink_mock, create_link_mock):
+        self.config(ipxe_enabled=True, group='pxe')
+        port_1 = object_utils.create_test_port(
+            self.context, node_id=self.node.id,
+            address='11:22:33:44:55:66', uuid=uuidutils.generate_uuid())
+        port_2 = object_utils.create_test_port(
+            self.context, node_id=self.node.id,
+            address='11:22:33:44:55:67', uuid=uuidutils.generate_uuid())
+        create_link_calls = [
+            mock.call(u'../1be26c0b-03f2-4d2e-ae87-c02d7f33c123/config',
+                      '/httpboot/pxelinux.cfg/11-22-33-44-55-66'),
+            mock.call(u'../1be26c0b-03f2-4d2e-ae87-c02d7f33c123/config',
+                      '/httpboot/pxelinux.cfg/11-22-33-44-55-67'),
+        ]
+        unlink_calls = [
+            mock.call('/httpboot/pxelinux.cfg/11-22-33-44-55-66'),
+            mock.call('/httpboot/pxelinux.cfg/11-22-33-44-55-67'),
+        ]
+        with task_manager.acquire(self.context, self.node.uuid) as task:
+            task.ports = [port_1, port_2]
             pxe_utils._link_mac_pxe_configs(task)
 
         unlink_mock.assert_has_calls(unlink_calls)

ironic/tests/unit/conductor/test_manager.py

@@ -2806,6 +2806,65 @@ class UpdatePortTestCase(mgr_utils.ServiceSetUpMixin,
         mac_update_mock.assert_called_once_with('fake-id', new_address,
                                                 token=self.context.auth_token)
 
+    @mock.patch('ironic.dhcp.neutron.NeutronDHCPApi.update_port_dhcp_opts')
+    def test_update_port_client_id(self, dhcp_update_mock):
+        node = obj_utils.create_test_node(self.context, driver='fake')
+        port = obj_utils.create_test_port(self.context,
+                                          node_id=node.id,
+                                          extra={'vif_port_id': 'fake-id',
+                                                 'client-id': 'fake1'})
+        expected_extra = {'vif_port_id': 'fake-id', 'client-id': 'fake2'}
+        expected_dhcp_opts = [{'opt_name': 'client-id', 'opt_value': 'fake2'}]
+        port.extra = expected_extra
+        res = self.service.update_port(self.context, port)
+        self.assertEqual(expected_extra, res.extra)
+        dhcp_update_mock.assert_called_once_with('fake-id', expected_dhcp_opts,
+                                                 token=self.context.auth_token)
+
+    @mock.patch('ironic.dhcp.neutron.NeutronDHCPApi.update_port_dhcp_opts')
+    def test_update_port_vif(self, dhcp_update_mock):
+        node = obj_utils.create_test_node(self.context, driver='fake')
+        port = obj_utils.create_test_port(self.context,
+                                          node_id=node.id,
+                                          extra={'vif_port_id': 'fake-id',
+                                                 'client-id': 'fake1'})
+        expected_extra = {'vif_port_id': 'new_ake-id', 'client-id': 'fake1'}
+        port.extra = expected_extra
+        res = self.service.update_port(self.context, port)
+        self.assertEqual(expected_extra, res.extra)
+        self.assertFalse(dhcp_update_mock.called)
+
+    @mock.patch('ironic.dhcp.neutron.NeutronDHCPApi.update_port_dhcp_opts')
+    def test_update_port_client_id_fail(self, dhcp_update_mock):
+        node = obj_utils.create_test_node(self.context, driver='fake')
+        expected_extra = {'vif_port_id': 'fake-id', 'client-id': 'fake1'}
+        port = obj_utils.create_test_port(self.context,
+                                          node_id=node.id,
+                                          extra=expected_extra)
+        extra = {'vif_port_id': 'fake-id', 'client-id': 'fake2'}
+        port.extra = extra
+        dhcp_update_mock.side_effect = (
+            exception.FailedToUpdateDHCPOptOnPort(port_id=port.uuid))
+        exc = self.assertRaises(messaging.rpc.ExpectedException,
+                                self.service.update_port,
+                                self.context, port)
+        # Compare true exception hidden by @messaging.expected_exceptions
+        self.assertEqual(
+            exception.FailedToUpdateDHCPOptOnPort, exc.exc_info[0])
+        port.refresh()
+        self.assertEqual(expected_extra, port.extra)
+
+    @mock.patch('ironic.dhcp.neutron.NeutronDHCPApi.update_port_dhcp_opts')
+    def test_update_port_client_id_no_vif_id(self, dhcp_update_mock):
+        node = obj_utils.create_test_node(self.context, driver='fake')
+        port = obj_utils.create_test_port(self.context, node_id=node.id)
+
+        expected_extra = {'client-id': 'fake2'}
+        port.extra = expected_extra
+        res = self.service.update_port(self.context, port)
+        self.assertEqual(expected_extra, res.extra)
+        self.assertFalse(dhcp_update_mock.called)
+
     def test_update_port_node_deleting_state(self):
         node = obj_utils.create_test_node(self.context, driver='fake',
                                           provision_state=states.DELETING)

ironic/tests/unit/drivers/modules/network/test_neutron.py

@@ -26,6 +26,8 @@ from ironic.tests.unit.db import base as db_base
 from ironic.tests.unit.objects import utils
 
 CONF = cfg.CONF
+CLIENT_ID1 = '20:00:55:04:01:fe:80:00:00:00:00:00:00:00:02:c9:02:00:23:13:92'
+CLIENT_ID2 = '20:00:55:04:01:fe:80:00:00:00:00:00:00:00:02:c9:02:00:23:13:93'
 
 
 class NeutronInterfaceTestCase(db_base.DbTestCase):
@@ -136,7 +138,7 @@ class NeutronInterfaceTestCase(db_base.DbTestCase):
             client_mock.assert_called_once_with(task.context.auth_token)
 
     @mock.patch.object(neutron_common, 'get_client')
-    def _test_configure_tenant_networks(self, client_mock):
+    def _test_configure_tenant_networks(self, client_mock, is_client_id=False):
         upd_mock = mock.Mock()
         client_mock.return_value.update_port = upd_mock
         second_port = utils.create_test_port(
@@ -147,6 +149,15 @@ class NeutronInterfaceTestCase(db_base.DbTestCase):
                                    'port_id': 'Ethernet1/1',
                                    'switch_info': 'switch2'}
         )
+        if is_client_id:
+            client_ids = (CLIENT_ID1, CLIENT_ID2)
+            ports = (self.port, second_port)
+            for port, client_id in zip(ports, client_ids):
+                extra = port.extra
+                extra['client-id'] = client_id
+                port.extra = extra
+                port.save()
+
         expected_body = {
             'port': {
                 'device_owner': 'baremetal:none',
@@ -164,6 +175,11 @@ class NeutronInterfaceTestCase(db_base.DbTestCase):
         port2_body['port']['binding:profile'] = {
             'local_link_information': [second_port.local_link_connection]
         }
+        if is_client_id:
+            port1_body['port']['extra_dhcp_opts'] = (
+                [{'opt_name': 'client-id', 'opt_value': client_ids[0]}])
+            port2_body['port']['extra_dhcp_opts'] = (
+                [{'opt_name': 'client-id', 'opt_value': client_ids[1]}])
         with task_manager.acquire(self.context, self.node.id) as task:
             self.interface.configure_tenant_networks(task)
             client_mock.assert_called_once_with(task.context.auth_token)
@@ -181,6 +197,11 @@ class NeutronInterfaceTestCase(db_base.DbTestCase):
     def test_configure_tenant_networks_no_instance_uuid(self):
         self._test_configure_tenant_networks()
 
+    def test_configure_tenant_networks_with_client_id(self):
+        self.node.instance_uuid = uuidutils.generate_uuid()
+        self.node.save()
+        self._test_configure_tenant_networks(is_client_id=True)
+
     @mock.patch.object(neutron_common, 'get_client')
     def test_configure_tenant_networks_with_portgroups(self, client_mock):
         pg = utils.create_test_portgroup(

releasenotes/notes/add_infiniband_support-f497767f77277a1a.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - Add support for InfiniBand network to allow
+    Hardware inspection and PXE boot over InfiniBand.