diff --git a/cyborg.conf.intelnic.sample b/cyborg.conf.intelnic.sample new file mode 100644 index 00000000..f46bc1ea --- /dev/null +++ b/cyborg.conf.intelnic.sample @@ -0,0 +1,69 @@ + +[DEFAULT] +auth_strategy = +transport_url = rabbit://stackrabbit:123456@192.168.0.186:5672/ +use_syslog = False +state_path = /var/lib/cyborg +debug = True + +[database] +connection = mysql+pymysql://root:123456@127.0.0.1/cyborg?charset=utf8 + +[keystone] +region_name = RegionOne + +[service_catalog] +cafile = /opt/stack/data/ca-bundle.pem +project_domain_id = default +user_domain_id = default +project_name = service +password = 123456 +username = cyborg +auth_url = http://192.168.0.186/identity +auth_type = password + +[nova] +project_domain_name = Default +project_name = service +user_domain_name = Default +password = 123456 +username = nova +auth_url = http://192.168.0.186/identity +auth_type = password + +[placement] +project_domain_name = Default +project_name = service +user_domain_name = Default +password = 123456 +username = placement +auth_url = http://192.168.0.186/identity +auth_type = password + +[keystone_authtoken] +memcached_servers = localhost:11211 +cafile = /opt/stack/data/ca-bundle.pem +project_domain_name = Default +project_name = service +user_domain_name = Default +password = 123456 +username = cyborg +auth_url = http://192.168.0.186/identity +interface = public +auth_type = password + +[oslo_policy] +policy_file = /etc/cyborg/policy.yaml + +[conductor] +automated_clean = + +[agent] +enabled_drivers = fake_driver,intel_nic_driver + +[nic_devices] +enabled_nic_types = x710_static + +[x710_static] +physical_device_mappings = physnet1:eth2|eth3 +function_device_mappings = GTPv1:eth3|eth2 diff --git a/cyborg/accelerator/common/utils.py b/cyborg/accelerator/common/utils.py index 3779a7f1..e237581f 100644 --- a/cyborg/accelerator/common/utils.py +++ b/cyborg/accelerator/common/utils.py @@ -13,16 +13,82 @@ # under the License. import collections +import os from oslo_serialization import jsonutils +from cyborg.common import exception -def pci_str_to_json(pci_address): + +def pci_str_to_json(pci_address, physnet=None): dbs, func = pci_address.split('.') domain, bus, slot = dbs.split(':') keys = ["domain", "bus", "device", "function"] values = [domain, bus, slot, func] + if physnet: + keys.append("physical_network") + values.append(physnet) bdf_dict = dict(zip(keys, values)) ordered_dict = collections.OrderedDict(sorted(bdf_dict.items())) bdf_json = jsonutils.dumps(ordered_dict) return bdf_json + + +def _get_sysfs_netdev_path(pci_addr, vf_interface=False): + """Get the sysfs path based on the PCI address of the device. + Assumes a networking device - will not check for the existence of the path. + :param pci_addr: the pci addresee of the device(PF or VF). + :param vf_interface: True if the pci_addr is a VF, + False if the pci_addr is a PF. + :returns: the sysfs path corresponds to the pci_addr. + """ + if vf_interface: + return "/sys/bus/pci/devices/%s/physfn/net" % pci_addr + return "/sys/bus/pci/devices/%s/net" % pci_addr + + +def get_ifname_by_pci_address(pci_addr, vf_interface=False): + """Get the interface name based on the pci address. + The returned interface name is either the parent PF's or that of the PF + itself based on the argument of vf_interface. + :param pci_addr: the pci address of the device(PF or VF) + :param vf_interface: True if the pci_addr is a VF, + False if the pci_addr is a PF. + :returns: the interface name corresponds to the pci_addr. + """ + dev_path = _get_sysfs_netdev_path(pci_addr, vf_interface) + try: + dev_info = os.listdir(dev_path) + return dev_info.pop() + except Exception: + raise exception.PciDeviceNotFoundById(id=pci_addr) + + +def parse_mappings(mapping_list): + """Parse mapping devices list + + parses mapping device list in the form: + physnet:pci_dev_1,pci_dev_2 or + function:pci_dev_1,pci_dev_2 + :param mapping_list: list of string pairs in "key:value" format + the key part represents the physnet or function name + the value part is a list of device name separated by "," + :returns: a dict of valid fields. + """ + mapping = {} + for dev_mapping in mapping_list: + try: + physnet_or_function, devices = dev_mapping.split(":", 1) + except ValueError: + raise ValueError(("Invalid mapping: '%s'") % dev_mapping) + physnet_or_function = physnet_or_function.strip() + if not physnet_or_function: + raise ValueError(("Missing key in mapping: '%s'") % dev_mapping) + if physnet_or_function in mapping: + raise ValueError( + ("Key %(physnet_or_function)s in mapping: %(mapping)s " + "not unique") % {'physnet_or_function': physnet_or_function, + 'mapping': dev_mapping}) + mapping[physnet_or_function] = set(dev.strip() for dev in + devices.split("|") if dev.strip()) + return mapping diff --git a/cyborg/accelerator/drivers/fpga/intel/driver.py b/cyborg/accelerator/drivers/fpga/intel/driver.py index 8c8335d5..4edd5c7f 100644 --- a/cyborg/accelerator/drivers/fpga/intel/driver.py +++ b/cyborg/accelerator/drivers/fpga/intel/driver.py @@ -47,7 +47,7 @@ class IntelFPGADriver(FPGADriver): """Base class for FPGA drivers. This is just a virtual FPGA drivers interface. - Vedor should implement their specific drivers. + Vendor should implement their specific drivers. """ VENDOR = "intel" diff --git a/cyborg/accelerator/drivers/nic/__init__.py b/cyborg/accelerator/drivers/nic/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cyborg/accelerator/drivers/nic/base.py b/cyborg/accelerator/drivers/nic/base.py new file mode 100644 index 00000000..ddd11496 --- /dev/null +++ b/cyborg/accelerator/drivers/nic/base.py @@ -0,0 +1,51 @@ +# Copyright 2020 Intel, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +""" +Cyborg NIC driver implementation. +""" + +VENDOR_MAPS = {"0x8086": "intel"} + + +class NICDriver(object): + """Base class for Nic drivers. + + This is just a virtual NIC drivers interface. + Vendor should implement their specific drivers. + """ + + @classmethod + def create(cls, vendor, *args, **kwargs): + for sclass in cls.__subclasses__(): + vendor_name = VENDOR_MAPS.get(vendor, vendor) + if vendor_name == sclass.VENDOR: + return sclass(*args, **kwargs) + raise LookupError("Not find the NIC driver for vendor %s" % vendor) + + def discover(self): + """Discover NIC information of current vendor(Identified by class). + + :return: List of NIC information dict. + """ + raise NotImplementedError() + + @classmethod + def discover_vendors(cls): + """Discover NIC vendors of current node. + + :return: NIC vendor ID list. + """ + raise NotImplementedError() diff --git a/cyborg/accelerator/drivers/nic/intel/__init__.py b/cyborg/accelerator/drivers/nic/intel/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cyborg/accelerator/drivers/nic/intel/driver.py b/cyborg/accelerator/drivers/nic/intel/driver.py new file mode 100644 index 00000000..65752e43 --- /dev/null +++ b/cyborg/accelerator/drivers/nic/intel/driver.py @@ -0,0 +1,34 @@ +# Copyright 2020 Intel, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +""" +Cyborg Intel NIC driver implementation. +""" + +from cyborg.accelerator.drivers.nic.base import NICDriver +from cyborg.accelerator.drivers.nic.intel import sysinfo + + +class IntelNICDriver(NICDriver): + """Class for Intel NIC drivers. + Vendor should implement their specific drivers in this class. + """ + VENDOR = "intel" + + def __init__(self, *args, **kwargs): + pass + + def discover(self): + return sysinfo.nic_tree() diff --git a/cyborg/accelerator/drivers/nic/intel/sysinfo.py b/cyborg/accelerator/drivers/nic/intel/sysinfo.py new file mode 100644 index 00000000..69bc1771 --- /dev/null +++ b/cyborg/accelerator/drivers/nic/intel/sysinfo.py @@ -0,0 +1,253 @@ +# Copyright 2020 Intel, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +""" +Cyborg Intel NIC driver implementation. +""" + + +import glob +import os +import socket + +from oslo_config import cfg +from oslo_log import log as logging +from oslo_serialization import jsonutils + +from cyborg.accelerator.common import utils +import cyborg.conf + +from cyborg.objects.driver_objects import driver_attach_handle +from cyborg.objects.driver_objects import driver_attribute +from cyborg.objects.driver_objects import driver_controlpath_id +from cyborg.objects.driver_objects import driver_deployable +from cyborg.objects.driver_objects import driver_device + +LOG = logging.getLogger(__name__) + +PCI_DEVICES_PATH_PATTERN = "/sys/bus/pci/devices/*" +KNOWN_NICS = [("0x8086", "0x158b"), ("0x8086", "0x1572")] +DRIVER_NAME = "intel" + +VF = "virtfn*" +_SRIOV_TOTALVFS = "sriov_totalvfs" +CONF = cyborg.conf.CONF + + +def _parse_config(): + # parse nic config. + cyborg.conf.devices.register_dynamic_opts(CONF) + try: + pdm = utils.parse_mappings(CONF.x710_static.physical_device_mappings) + fdm = utils.parse_mappings(CONF.x710_static.function_device_mappings) + except cfg.NoSuchOptError: + return None, None + else: + return pdm, fdm + + +def get_physical_network_and_traits(pci_info, physnet_device_mappings, + function_device_mappings, pf_nic=None): + traits = [] + physnet = None + func_name = None + + if pf_nic: + pf_addr = pf_nic["device"] + traits.append("CUSTOM_VF") + else: + pf_addr = pci_info["PCI_SLOT_NAME"] + traits.append("CUSTOM_PF") + if not pf_addr: + LOG.error("Incorrect report data received. Missing PF info.") + + pf_ifname = utils.get_ifname_by_pci_address(pf_addr) + + if physnet_device_mappings: + for key, devices in physnet_device_mappings.items(): + if pf_ifname in devices: + physnet = key + break + if function_device_mappings: + for key, devices in function_device_mappings.items(): + if pf_ifname in devices: + func_name = key + break + if func_name: + traits.append("CUSTOM_" + func_name.upper()) + if physnet: + traits.append("CUSTOM_" + physnet.upper()) + + return {"traits": traits, "physical_network": physnet} + + +def read_line(filename): + with open(filename) as f: + return f.readline().strip() + + +def find_nics_by_know_list(): + return set(filter( + lambda p: ( + read_line(os.path.join(p, "vendor")), + read_line(os.path.join(p, "device")) + ) in KNOWN_NICS, + glob.glob(PCI_DEVICES_PATH_PATTERN))) + + +def pci_attributes(path): + with open(os.path.join(path, "uevent")) as f: + attributes = dict(map( + lambda p: p.strip().split("="), + f.readlines() + )) + + with open(os.path.join(path, "vendor")) as f: + attributes["VENDOR"] = f.readline().strip() + + with open(os.path.join(path, "device")) as f: + attributes["PRODUCT_ID"] = f.readline().strip() + + return attributes + + +def nic_gen(path, physnet_device_mappings=None, function_device_mappings=None, + pf_nic=None): + pci_info = pci_attributes(path) + nic = { + "name": "_".join((socket.gethostname(), pci_info["PCI_SLOT_NAME"])), + "device": pci_info["PCI_SLOT_NAME"], + "type": "NIC", + "vendor": pci_info["VENDOR"], + "product_id": pci_info["PRODUCT_ID"], + "rc": "CUSTOM_NIC", + "stub": False, + } + # TODO(Xinran): need check device id and call get_traits differently. + updates = get_physical_network_and_traits(pci_info, + physnet_device_mappings, + function_device_mappings, + pf_nic) + + nic.update(updates) + return nic + + +def all_pfs_with_vf(): + return set(filter( + lambda p: glob.glob(os.path.join(p, VF)), + find_nics_by_know_list())) + + +def all_vfs_in_pf(pf_path): + return map( + lambda p: + os.path.join( + os.path.dirname(os.path.dirname(p)), + os.path.basename(os.readlink(p))), + glob.glob(os.path.join(pf_path, VF))) + + +def nic_tree(): + physnet_device_mappings, function_device_mappings = _parse_config() + nics = [] + pfs_has_vf = all_pfs_with_vf() + for n in find_nics_by_know_list(): + nic = nic_gen(n, physnet_device_mappings, function_device_mappings) + if n in pfs_has_vf: + vfs = [] + for vf in all_vfs_in_pf(n): + vf_nic = nic_gen(vf, physnet_device_mappings, + function_device_mappings, nic) + vfs.append(vf_nic) + nic["vfs"] = vfs + nics.append(_generate_driver_device(nic)) + return nics + + +def _generate_driver_device(nic): + driver_device_obj = driver_device.DriverDevice() + driver_device_obj.vendor = nic["vendor"] + driver_device_obj.stub = nic["stub"] + driver_device_obj.model = nic.get("model", "miss_model_info") + driver_device_obj.vendor_board_info = nic.get( + "vendor_board_info", + "miss_vb_info") + std_board_info = {"product_id": nic.get("product_id", None)} + driver_device_obj.std_board_info = jsonutils.dumps(std_board_info) + driver_device_obj.type = nic["type"] + driver_device_obj.controlpath_id = _generate_controlpath_id(nic) + driver_device_obj.deployable_list = _generate_dep_list(nic) + return driver_device_obj + + +def _generate_controlpath_id(nic): + driver_cpid = driver_controlpath_id.DriverControlPathID() + driver_cpid.cpid_type = "PCI" + driver_cpid.cpid_info = utils.pci_str_to_json(nic["device"]) + return driver_cpid + + +def _generate_dep_list(nic): + dep_list = [] + # pf without sriov enabled. + if "vfs" not in nic: + driver_dep = driver_deployable.DriverDeployable() + driver_dep.num_accelerators = 1 + driver_dep.attach_handle_list = [ + _generate_attach_handle(nic)] + driver_dep.name = nic["name"] + driver_dep.driver_name = DRIVER_NAME + driver_dep.attribute_list = _generate_attribute_list(nic) + dep_list = [driver_dep] + # pf with sriov enabled, may have several vfs. + else: + for vf in nic["vfs"]: + driver_dep = driver_deployable.DriverDeployable() + driver_dep.num_accelerators = 1 + driver_dep.attach_handle_list = [ + _generate_attach_handle(vf)] + driver_dep.name = vf["name"] + driver_dep.driver_name = DRIVER_NAME + driver_dep.attribute_list = _generate_attribute_list(vf) + dep_list.append(driver_dep) + return dep_list + + +def _generate_attach_handle(nic): + driver_ah = driver_attach_handle.DriverAttachHandle() + driver_ah.attach_type = "PCI" + driver_ah.attach_info = utils.pci_str_to_json(nic["device"], + nic["physical_network"]) + driver_ah.in_use = False + return driver_ah + + +def _generate_attribute_list(nic): + attr_list = [] + for k, v in nic.items(): + if k == "rc": + driver_attr = driver_attribute.DriverAttribute() + driver_attr.key, driver_attr.value = k, v + attr_list.append(driver_attr) + if k == "traits": + values = nic.get(k, []) + for idx, val in enumerate(values): + driver_attr = driver_attribute.DriverAttribute() + driver_attr.key = "trait" + str(idx) + driver_attr.value = val + attr_list.append(driver_attr) + + return attr_list diff --git a/cyborg/common/constants.py b/cyborg/common/constants.py index e28462d7..c8aedd4b 100644 --- a/cyborg/common/constants.py +++ b/cyborg/common/constants.py @@ -20,6 +20,7 @@ DEVICE_GPU = 'GPU' DEVICE_FPGA = 'FPGA' DEVICE_AICHIP = 'AICHIP' DEVICE_QAT = 'QAT' +DEVICE_NIC = 'NIC' ARQ_STATES = (ARQ_INITIAL, ARQ_BIND_STARTED, ARQ_BOUND, ARQ_UNBOUND, @@ -58,7 +59,7 @@ ARQ_STATES_TRANSFORM_MATRIX = { # Device type -DEVICE_TYPE = (DEVICE_GPU, DEVICE_FPGA, DEVICE_AICHIP, DEVICE_QAT) +DEVICE_TYPE = (DEVICE_GPU, DEVICE_FPGA, DEVICE_AICHIP, DEVICE_QAT, DEVICE_NIC) # Attach handle type @@ -76,7 +77,8 @@ RESOURCES = { "FPGA": orc.FPGA, "PGPU": orc.PGPU, "VGPU": orc.VGPU, - "QAT": "CUSTOM_QAT" + "QAT": "CUSTOM_QAT", + "NIC": "CUSTOM_NIC" } @@ -90,8 +92,8 @@ ACCEL_SPECS = ( SUPPORT_RESOURCES = ( - FPGA, GPU, VGPU, PGPU, QAT) = ( - "FPGA", "GPU", "VGPU", "PGPU", "CUSTOM_QAT" + FPGA, GPU, VGPU, PGPU, QAT, NIC) = ( + "FPGA", "GPU", "VGPU", "PGPU", "CUSTOM_QAT", "CUSTOM_NIC" ) diff --git a/cyborg/common/exception.py b/cyborg/common/exception.py index 9fd2e89d..b16243f0 100644 --- a/cyborg/common/exception.py +++ b/cyborg/common/exception.py @@ -415,3 +415,7 @@ class NotAcceptable(CyborgException): class FPGAProgramError(CyborgException): _msg_fmt = _("FPGA programming failed with return %(ret)s.") + + +class PciDeviceNotFoundById(NotFound): + _msg_fmt = _("PCI device %(id)s not found") diff --git a/cyborg/conf/__init__.py b/cyborg/conf/__init__.py index e1a5889d..c90243f8 100644 --- a/cyborg/conf/__init__.py +++ b/cyborg/conf/__init__.py @@ -19,6 +19,7 @@ from cyborg.conf import agent from cyborg.conf import api from cyborg.conf import database from cyborg.conf import default +from cyborg.conf import devices from cyborg.conf import glance from cyborg.conf import keystone from cyborg.conf import nova @@ -31,6 +32,7 @@ api.register_opts(CONF) agent.register_opts(CONF) database.register_opts(CONF) default.register_opts(CONF) +devices.register_opts(CONF) service_token.register_opts(CONF) glance.register_opts(CONF) keystone.register_opts(CONF) diff --git a/cyborg/conf/devices.py b/cyborg/conf/devices.py new file mode 100644 index 00000000..73439e5b --- /dev/null +++ b/cyborg/conf/devices.py @@ -0,0 +1,57 @@ +# Copyright 2020 Intel, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + + +nic_group = cfg.OptGroup( + name='nic_devices', + title='nic device ID options', + help="""This is used to config specific nic devices. + """) + +nic_opts = [ + cfg.ListOpt('enabled_nic_types', + default=[], + help=" ") +] + + +def register_opts(conf): + conf.register_group(nic_group) + conf.register_opts(nic_opts, group=nic_group) + + +def register_dynamic_opts(conf): + """Register dynamically-generated options and groups. + + This must be called by the service that wishes to use the options **after** + the initial configuration has been loaded. + """ + opts = [ + cfg.ListOpt('physical_device_mappings', default=[], + item_type=cfg.types.String()), + cfg.ListOpt('function_device_mappings', default=[], + item_type=cfg.types.String()), + ] + + # Register the '[nic_type]/physical_device_mappings' and + # '[nic_type]/function_device_mappings' opts, implicitly + # registering the '[nic_type]' groups in the process + for nic_type in conf.nic_devices.enabled_nic_types: + conf.register_opts(opts, group=nic_type) + + +def list_opts(): + return {nic_group: nic_opts} diff --git a/cyborg/db/sqlalchemy/alembic/versions/899cead40bc9_add_nic_type.py b/cyborg/db/sqlalchemy/alembic/versions/899cead40bc9_add_nic_type.py new file mode 100644 index 00000000..52654f10 --- /dev/null +++ b/cyborg/db/sqlalchemy/alembic/versions/899cead40bc9_add_nic_type.py @@ -0,0 +1,22 @@ +"""add_nic_type + +Revision ID: 899cead40bc9 +Revises: 7e6f1f107f2b +Create Date: 2020-09-18 02:33:42.640673 + +""" + +# revision identifiers, used by Alembic. +revision = '899cead40bc9' +down_revision = '7e6f1f107f2b' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + new_device_type = sa.Enum('GPU', 'FPGA', 'AICHIP', 'QAT', 'NIC', + name='device_type') + op.alter_column('devices', 'type', + existing_type=new_device_type, + nullable=False) diff --git a/cyborg/db/sqlalchemy/models.py b/cyborg/db/sqlalchemy/models.py index dfc4e45d..bc74bb79 100644 --- a/cyborg/db/sqlalchemy/models.py +++ b/cyborg/db/sqlalchemy/models.py @@ -81,7 +81,7 @@ class Device(Base): id = Column(Integer, primary_key=True) uuid = Column(String(36), nullable=False, unique=True) - type = Column(Enum('GPU', 'FPGA', 'AICHIP', 'QAT', + type = Column(Enum('GPU', 'FPGA', 'AICHIP', 'QAT', 'NIC', name='device_type'), nullable=False) vendor = Column(String(255), nullable=False) model = Column(String(255), nullable=False) diff --git a/cyborg/tests/unit/accelerator/drivers/fpga/intel/prepare_test_data.py b/cyborg/tests/unit/accelerator/drivers/fpga/intel/prepare_test_data.py index a4d605c7..ecfd2bb2 100755 --- a/cyborg/tests/unit/accelerator/drivers/fpga/intel/prepare_test_data.py +++ b/cyborg/tests/unit/accelerator/drivers/fpga/intel/prepare_test_data.py @@ -13,7 +13,6 @@ # License for the specific language governing permissions and limitations # under the License. -import argparse import copy import glob import os @@ -282,22 +281,3 @@ def create_fake_sysfs(prefix=""): os.makedirs(sys_class_fpga) create_devices_path_and_files(FPGA_TREE, sys_device, sys_class_fpga) create_devices_soft_link(sys_class_fpga) - - -def main(): - create_fake_sysfs() - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Generate a fake sysfs for intel FPGA.") - group = parser.add_mutually_exclusive_group() - group.add_argument("-v", "--verbose", action="store_true") - group.add_argument("-q", "--quiet", action="store_true") - parser.add_argument("-p", "--prefix", type=str, - default="/tmp", dest="p", - help='Set the prefix path of the fake sysfs. ' - 'default "/tmp"') - args = parser.parse_args() - - create_fake_sysfs(args.p) diff --git a/cyborg/tests/unit/accelerator/drivers/nic/__init__.py b/cyborg/tests/unit/accelerator/drivers/nic/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cyborg/tests/unit/accelerator/drivers/nic/intel/__init__.py b/cyborg/tests/unit/accelerator/drivers/nic/intel/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cyborg/tests/unit/accelerator/drivers/nic/intel/prepare_test_data.py b/cyborg/tests/unit/accelerator/drivers/nic/intel/prepare_test_data.py new file mode 100644 index 00000000..124c4782 --- /dev/null +++ b/cyborg/tests/unit/accelerator/drivers/nic/intel/prepare_test_data.py @@ -0,0 +1,245 @@ +#!/usr/bin/python +# Copyright 2021 Intel, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy +import os +import shutil + + +PF0_ADDR = "0000:05:00.0" +PF1_ADDR = "0000:06:00.0" +VF0_ADDR = "0000:05:01.0" +NIC_TREE = { + "dev.0": {"bdf": PF0_ADDR, + "vfs": {"dev.2": {"bdf": VF0_ADDR}}}, + "dev.1": {"bdf": PF1_ADDR}} + +SYS_DEVICES = "sys/devices" +PCI_DEVICES_PATH = "sys/bus/pci/devices" + +DEV_PREFIX = "intel-nic" + +NIC_DEVICE_COMMON_SUB_DIR = ["power", "msi_irqs"] + +NIC_DEVICE_COMMON_CONTENT = { + "broken_parity_status": "0", + "class": "0x0b4000", + "config": "", + "consistent_dma_mask_bits": "64", + "d3cold_allowed": "1", + "device": "0x158b", + "dma_mask_bits": "64", + "driver_override": "(null)", + "enable": "1", + "irq": "33", + "local_cpulist": "0-7,16-23", + "local_cpus": "000000,00000000,00000000,00000000,00ff00ff", + "max_link_speed": "5 GT/s", + "max_link_width": "16", + "modalias": "pci:v00008086d000037C8sv00008086sd00000002bc0Bsc40i00", + "msi_bus": "1", + "numa_node": "0", + "resource0": "", + "subsystem_device": "0x0002", + "subsystem_vendor": "0x8086", + "vendor": "0x8086"} + +NIC_DEVICES_SPECIAL_COMMON_CONTENT = { + "dev.0": { + "resource": [ + "0x0000000000000000 0x0000000000000000 0x0000000000000000", + "0x0000000000000000 0x0000000000000000 0x0000000000000000", + "0x00000000d1340000 0x00000000d137ffff 0x0000000000140204", + "0x0000000000000000 0x0000000000000000 0x0000000000000000", + "0x00000000d1300000 0x00000000d133ffff 0x0000000000140204", + "0x0000000000000000 0x0000000000000000 0x0000000000000000", + "0x0000000000000000 0x0000000000000000 0x0000000000000000", + "0x00000000d1390000 0x00000000d139ffff 0x0000000000140204", + "0x0000000000000000 0x0000000000000000 0x0000000000000000", + "0x00000000d1390000 0x00000000d139ffff 0x0000000000140204", + "0x0000000000000000 0x0000000000000000 0x0000000000000000", + "0x0000000000000000 0x0000000000000000 0x0000000000000000", + "0x0000000000000000 0x0000000000000000 0x0000000000000000"], + "resource2": "", + "resource4": "", + "sriov_numvfs": "1", + "sriov_totalvfs": "1", + "uevent": [ + "DRIVER=c6xx", + "PCI_CLASS=B4000", + "PCI_ID=8086:37C8", + "PCI_SUBSYS_ID=8086:0002", + "PCI_SLOT_NAME=0000:05:00.0", + "MODALIAS=pci:v00008086d000037C8sv00008086sd00000002bc0Bsc40i00"], + }, + "dev.1": { + "resource": [ + "0x00000000fbc00000 0x00000000fbc7ffff 0x000000000014220c", + "0x0000000000000000 0x0000000000000000 0x0000000000000000", + "0x00000000fbc80000 0x00000000fbcfffff 0x000000000014220c", + "0x0000000000000000 0x0000000000000000 0x0000000000000000", + "0x0000000000000000 0x0000000000000000 0x0000000000000000", + "0x0000000000000000 0x0000000000000000 0x0000000000000000", + "0x0000000000000000 0x0000000000000000 0x0000000000000000", + "0x00000000fbd00000 0x00000000fbd7ffff 0x000000000014220c", + "0x0000000000000000 0x0000000000000000 0x0000000000000000", + "0x0000000000000000 0x0000000000000000 0x0000000000000000", + "0x0000000000000000 0x0000000000000000 0x0000000000000000", + "0x0000000000000000 0x0000000000000000 0x0000000000000000", + "0x0000000000000000 0x0000000000000000 0x0000000000000000"], + "resource2": "", + "sriov_numvfs": "0", + "sriov_totalvfs": "0", + "uevent": [ + "DRIVER=c6xx", + "PCI_CLASS=B4000", + "PCI_ID=8086:37C8", + "PCI_SUBSYS_ID=8086:0002", + "PCI_SLOT_NAME=0000:06:00.0", + "MODALIAS=pci:v00008086d000037C8sv00008086sd00000002bc0Bsc40i00"], + }, + "dev.2": { + "d3cold_allowed": "0", + "device": "0x37c9", + "modalias": "pci:v00008086d000037C9sv00008086sd00000000bc0Bsc40i00", + "irq": "0", + "resource": [ + "0x00000000c6100000 0x00000000c617ffff 0x000000000014220c", + "0x0000000000000000 0x0000000000000000 0x0000000000000000", + "0x0000000000000000 0x0000000000000000 0x0000000000000000", + "0x0000000000000000 0x0000000000000000 0x0000000000000000", + "0x0000000000000000 0x0000000000000000 0x0000000000000000", + "0x0000000000000000 0x0000000000000000 0x0000000000000000", + "0x0000000000000000 0x0000000000000000 0x0000000000000000", + "0x0000000000000000 0x0000000000000000 0x0000000000000000", + "0x0000000000000000 0x0000000000000000 0x0000000000000000", + "0x0000000000000000 0x0000000000000000 0x0000000000000000", + "0x0000000000000000 0x0000000000000000 0x0000000000000000", + "0x0000000000000000 0x0000000000000000 0x0000000000000000", + "0x0000000000000000 0x0000000000000000 0x0000000000000000"], + "uevent": [ + "DRIVER=c6xx", + "PCI_CLASS=B4000", + "PCI_ID=8086:37C8", + "PCI_SUBSYS_ID=8086:0002", + "PCI_SLOT_NAME=0000:05:01.0", + "MODALIAS=pci:v00008086d000037C8sv00008086sd00000002bc0Bsc40i00"], + } +} + +NIC_DEVICE_COMMON_SOFT_LINK = { + "driver": "../../../../../../bus/pci/drivers/c6xx", + "iommu": "../../../../../virtual/iommu/dmar1", + "subsystem": "../../../../../../bus/pci" +} + +NIC_DEVICES_SPECIAL_SOFT_LINK = { + "dev.0": { + "driver": "../../../../../..//bus/pci/drivers/c6xx", + "iommu_group": "../../../../../../kernel/iommu_groups/20", + # "virtfn0": "../0000:05:01.0/", + }, + "dev.1": { + "driver": "../../../../../..//bus/pci/drivers/c6xx", + "iommu_group": "../../../../../../kernel/iommu_groups/21", + }, + "dev.2": { + "iommu_group": "../../../../../../kernel/iommu_groups/67", + # "physfn": "../0000:05:00.0/", + } +} + +NIC_DEVICE_PF_SOFT_LINK = { + "virtfn": lambda k, v: (k + str(int(v.rsplit(".", 1)[-1])), + "/".join(["..", v])) +} + +NIC_DEVICE_VF_SOFT_LINK = { + "physfn": lambda k, v: (k, "/".join(["..", v])) +} + + +def gen_nic_content(path, dev): + content = copy.copy(NIC_DEVICE_COMMON_CONTENT) + content.update(NIC_DEVICES_SPECIAL_COMMON_CONTENT[dev]) + for k, v in content.items(): + p = os.path.join(path, k) + if not v: + os.mknod(p) + elif type(v) is str: + with open(p, 'a') as f: + f.write(v + "\n") + elif type(v) is list: + with open(p, 'a') as f: + f.writelines([l + "\n" for l in v]) + + +def gen_nic_sub_dir(path): + for d in NIC_DEVICE_COMMON_SUB_DIR: + p = os.path.join(path, d) + os.makedirs(p) + + +def gen_nic_pf_soft_link(path, bdf): + for k, v in NIC_DEVICE_PF_SOFT_LINK.items(): + if callable(v): + k, v = v(k, bdf) + os.symlink(v, os.path.join(path, k)) + + +def gen_nic_common_soft_link(path, bdf): + for k, v in NIC_DEVICE_COMMON_SOFT_LINK.items(): + os.symlink(v, os.path.join(path, k)) + + +def gen_nic_vf_soft_link(path, bdf): + for k, v in NIC_DEVICE_VF_SOFT_LINK.items(): + if callable(v): + k, v = v(k, bdf) + os.symlink(v, os.path.join(path, k)) + + +def create_devices_path_and_files(tree, device_path, vf=False, pfinfo=None): + for k, v in tree.items(): + bdf = v["bdf"] + pci_path = "pci" + bdf.rsplit(":", 1)[0] + bdf_path = os.path.join(device_path, pci_path, bdf) + ln = "-".join([DEV_PREFIX, k]) + dev_path = os.path.join(bdf_path, "nic", ln) + os.makedirs(dev_path) + gen_nic_content(bdf_path, k) + gen_nic_sub_dir(bdf_path) + if vf: + gen_nic_pf_soft_link(pfinfo["path"], bdf) + gen_nic_vf_soft_link(bdf_path, pfinfo["bdf"]) + pfinfo = {"path": bdf_path, "bdf": bdf} + if "vfs" in v: + create_devices_path_and_files( + v["vfs"], device_path, True, pfinfo) + os.symlink("../../../" + bdf, os.path.join(dev_path, "device")) + pci_dev = os.path.join(device_path.split(SYS_DEVICES)[0], + PCI_DEVICES_PATH) + if not os.path.exists(pci_dev): + os.makedirs(pci_dev) + os.symlink("../../.." + bdf_path.split("sys")[-1], + os.path.join(pci_dev, bdf)) + + +def create_fake_sysfs(prefix=""): + sys_device = os.path.join(prefix, SYS_DEVICES) + basedir = os.path.dirname(sys_device) + if os.path.exists(basedir): + shutil.rmtree(basedir, ignore_errors=False, onerror=None) + create_devices_path_and_files(NIC_TREE, sys_device) diff --git a/cyborg/tests/unit/accelerator/drivers/nic/intel/test_driver.py b/cyborg/tests/unit/accelerator/drivers/nic/intel/test_driver.py new file mode 100644 index 00000000..c44771e9 --- /dev/null +++ b/cyborg/tests/unit/accelerator/drivers/nic/intel/test_driver.py @@ -0,0 +1,140 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +from unittest import mock + +from cyborg.accelerator.drivers.nic.intel import sysinfo + +from cyborg.accelerator.drivers.nic.intel.driver import IntelNICDriver +from cyborg.tests import base +from cyborg.tests.unit.accelerator.drivers.nic.intel import prepare_test_data + +import fixtures + + +class TestIntelNICDriver(base.TestCase): + def setUp(self): + super(TestIntelNICDriver, self).setUp() + self.pcipath = sysinfo.PCI_DEVICES_PATH_PATTERN + tmp_sys_dir = self.useFixture(fixtures.TempDir()) + prepare_test_data.create_fake_sysfs(tmp_sys_dir.path) + tmp_path = tmp_sys_dir.path + sysinfo.PCI_DEVICES_PATH_PATTERN = os.path.join( + tmp_path, sysinfo.PCI_DEVICES_PATH_PATTERN.split("/", 1)[-1]) + + def tearDown(self): + super(TestIntelNICDriver, self).tearDown() + sysinfo.PCI_DEVICES_PATH = self.pcipath + + @mock.patch("cyborg.accelerator.common.utils.get_ifname_by_pci_address") + def test_discover(self, mock_device_ifname): + mock_device_ifname.return_value = "ethx" + attach_handle_list = [ + [ + {'attach_type': 'PCI', + 'attach_info': '{"bus": "05", ' + '"device": "01", ' + '"domain": "0000", ' + '"function": "0"}', + 'in_use': False} + ], + [ + {'attach_type': 'PCI', + 'attach_info': '{"bus": "06", ' + '"device": "00", ' + '"domain": "0000", ' + '"function": "0"}', + 'in_use': False} + ] + ] + attribute_list = [ + [ + { + "key": "rc", + "value": "CUSTOM_NIC" + }, + { + "key": "trait0", + "value": "CUSTOM_VF" + } + ], + [ + { + "key": "rc", + "value": "CUSTOM_NIC" + }, + { + "key": "trait0", + "value": "CUSTOM_PF" + } + ], + ] + expected = [{'vendor': '0x8086', + 'type': 'NIC', + 'deployable_list': + [ + {'num_accelerators': 1, + 'name': '0000:05:01.0', + 'attach_handle_list': attach_handle_list[0], + 'attribute_list':attribute_list[0] + }, + ], + 'controlpath_id': + { + 'cpid_info': '{"bus": "05", ' + '"device": "00", ' + '"domain": "0000", ' + '"function": "0"}', + 'cpid_type': 'PCI'} + }, + {'vendor': '0x8086', + 'type': 'NIC', + 'deployable_list': + [ + {'num_accelerators': 1, + 'name': '0000:06:00.0', + 'attach_handle_list': attach_handle_list[1], + 'attribute_list':attribute_list[1] + }, + ], + 'controlpath_id': + { + 'cpid_info': '{"bus": "06", ' + '"device": "00", ' + '"domain": "0000", ' + '"function": "0"}', + 'cpid_type': 'PCI'} + } + ] + intel = IntelNICDriver() + nics = intel.discover() + list.sort(nics, key=lambda x: x._obj_deployable_list[0].name) + self.assertEqual(2, len(nics)) + for i in range(len(nics)): + nic_dict = nics[i].as_dict() + nic_dep_list = nic_dict['deployable_list'] + nic_attach_handle_list = \ + nic_dep_list[0].as_dict()['attach_handle_list'] + nic_attribute_list = \ + nic_dep_list[0].as_dict()['attribute_list'] + self.assertEqual(expected[i]['vendor'], nic_dict['vendor']) + self.assertEqual(expected[i]['controlpath_id'], + nic_dict['controlpath_id']) + self.assertEqual(expected[i]['deployable_list'][0] + ['num_accelerators'], + nic_dep_list[0].as_dict()['num_accelerators']) + self.assertEqual(1, len(nic_attach_handle_list)) + self.assertEqual(attach_handle_list[i][0], + nic_attach_handle_list[0].as_dict()) + self.assertEqual(attribute_list[i][0], + nic_attribute_list[0].as_dict()) diff --git a/cyborg/tests/unit/accelerator/drivers/nic/test_base.py b/cyborg/tests/unit/accelerator/drivers/nic/test_base.py new file mode 100644 index 00000000..e72143d7 --- /dev/null +++ b/cyborg/tests/unit/accelerator/drivers/nic/test_base.py @@ -0,0 +1,31 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from cyborg.accelerator.drivers.nic.base import NICDriver +from cyborg.accelerator.drivers.nic.intel.driver import IntelNICDriver # noqa + +from cyborg.tests import base + + +class TestNICDriver(base.TestCase): + def test_create(self): + NICDriver.create("intel") + self.assertRaises(LookupError, NICDriver.create, "other") + + def test_discover(self): + d = NICDriver() + self.assertRaises(NotImplementedError, d.discover) + + def test_discover_vendors(self): + d = NICDriver() + self.assertRaises(NotImplementedError, d.discover_vendors) diff --git a/doc/source/reference/driver-table.rst b/doc/source/reference/driver-table.rst index 30563bc7..7bb25d2e 100644 --- a/doc/source/reference/driver-table.rst +++ b/doc/source/reference/driver-table.rst @@ -37,6 +37,11 @@ - The driver for Inspur FPGA Cards. - None - Test results reported at Aug 2020. Please reference: `Inspur FPGA Driver Test Report `_ + * - Intel NIC Driver + - None + - The driver for Intel NIC Cards. + - None + - Test results reported at Feb 2021. Please reference: `Intel NIC Driver Test Report `_ .. note:: Temporary Test Report: This is a temporary test report, it is only valid for a short time, if you encounter problems, please contact the diff --git a/releasenotes/notes/intel-nic-driver-f93adad86a23ceb9.yaml b/releasenotes/notes/intel-nic-driver-f93adad86a23ceb9.yaml new file mode 100644 index 00000000..0664f433 --- /dev/null +++ b/releasenotes/notes/intel-nic-driver-f93adad86a23ceb9.yaml @@ -0,0 +1,10 @@ +--- +features: + - | + The Intel nic driver defines the Intel x710 NIC's data model in + Cyborg. It also proposes a standard configuration format to manage + networking related devices. The Intel X710 NIC supports DDP(Dynamic + Device Personalization) which provides the ability to reconfigure the + packet processing pipeline to support a broader range of traffic types. + It also supports SR-IOV technology, each physical card can be virtualized + into mulitiple VFs. diff --git a/setup.cfg b/setup.cfg index 38a88bcc..e1a58710 100644 --- a/setup.cfg +++ b/setup.cfg @@ -53,6 +53,7 @@ cyborg.accelerator.driver = fake_driver = cyborg.accelerator.drivers.fake:FakeDriver huawei_ascend_driver = cyborg.accelerator.drivers.aichip.huawei.ascend:AscendDriver intel_qat_driver = cyborg.accelerator.drivers.qat.intel.driver:IntelQATDriver + intel_nic_driver = cyborg.accelerator.drivers.nic.intel.driver:IntelNICDriver oslo.config.opts = cyborg = cyborg.conf.opts:list_opts