Add intel NIC driver

This patch implements a new driver for Intel Nic Card. It
can discover the device and report it to Placement service with
CUSTOM_NIC resource class and specific traits.

Operator should specify the device and correspond profile in a
config file so that nic's driver can read the info from this
file.

Please check the test report in the following link:
https://wiki.openstack.org/wiki/Cyborg/TestReport/IntelNic

Change-Id: Ida0ba8f24b9e226da7f3d7a85fc372247e5281a5
Implements: blueprint sriov-smartnic-support
This commit is contained in:
Xinran Wang 2020-09-30 06:10:51 +00:00
parent 54b4fa53ad
commit e3caf5cb0a
23 changed files with 999 additions and 27 deletions

View File

@ -0,0 +1,69 @@
[DEFAULT]
auth_strategy =
transport_url = rabbit://stackrabbit:123456@192.168.0.186:5672/
use_syslog = False
state_path = /var/lib/cyborg
debug = True
[database]
connection = mysql+pymysql://root:123456@127.0.0.1/cyborg?charset=utf8
[keystone]
region_name = RegionOne
[service_catalog]
cafile = /opt/stack/data/ca-bundle.pem
project_domain_id = default
user_domain_id = default
project_name = service
password = 123456
username = cyborg
auth_url = http://192.168.0.186/identity
auth_type = password
[nova]
project_domain_name = Default
project_name = service
user_domain_name = Default
password = 123456
username = nova
auth_url = http://192.168.0.186/identity
auth_type = password
[placement]
project_domain_name = Default
project_name = service
user_domain_name = Default
password = 123456
username = placement
auth_url = http://192.168.0.186/identity
auth_type = password
[keystone_authtoken]
memcached_servers = localhost:11211
cafile = /opt/stack/data/ca-bundle.pem
project_domain_name = Default
project_name = service
user_domain_name = Default
password = 123456
username = cyborg
auth_url = http://192.168.0.186/identity
interface = public
auth_type = password
[oslo_policy]
policy_file = /etc/cyborg/policy.yaml
[conductor]
automated_clean =
[agent]
enabled_drivers = fake_driver,intel_nic_driver
[nic_devices]
enabled_nic_types = x710_static
[x710_static]
physical_device_mappings = physnet1:eth2|eth3
function_device_mappings = GTPv1:eth3|eth2

View File

@ -13,16 +13,82 @@
# under the License.
import collections
import os
from oslo_serialization import jsonutils
from cyborg.common import exception
def pci_str_to_json(pci_address):
def pci_str_to_json(pci_address, physnet=None):
dbs, func = pci_address.split('.')
domain, bus, slot = dbs.split(':')
keys = ["domain", "bus", "device", "function"]
values = [domain, bus, slot, func]
if physnet:
keys.append("physical_network")
values.append(physnet)
bdf_dict = dict(zip(keys, values))
ordered_dict = collections.OrderedDict(sorted(bdf_dict.items()))
bdf_json = jsonutils.dumps(ordered_dict)
return bdf_json
def _get_sysfs_netdev_path(pci_addr, vf_interface=False):
"""Get the sysfs path based on the PCI address of the device.
Assumes a networking device - will not check for the existence of the path.
:param pci_addr: the pci addresee of the device(PF or VF).
:param vf_interface: True if the pci_addr is a VF,
False if the pci_addr is a PF.
:returns: the sysfs path corresponds to the pci_addr.
"""
if vf_interface:
return "/sys/bus/pci/devices/%s/physfn/net" % pci_addr
return "/sys/bus/pci/devices/%s/net" % pci_addr
def get_ifname_by_pci_address(pci_addr, vf_interface=False):
"""Get the interface name based on the pci address.
The returned interface name is either the parent PF's or that of the PF
itself based on the argument of vf_interface.
:param pci_addr: the pci address of the device(PF or VF)
:param vf_interface: True if the pci_addr is a VF,
False if the pci_addr is a PF.
:returns: the interface name corresponds to the pci_addr.
"""
dev_path = _get_sysfs_netdev_path(pci_addr, vf_interface)
try:
dev_info = os.listdir(dev_path)
return dev_info.pop()
except Exception:
raise exception.PciDeviceNotFoundById(id=pci_addr)
def parse_mappings(mapping_list):
"""Parse mapping devices list
parses mapping device list in the form:
physnet:pci_dev_1,pci_dev_2 or
function:pci_dev_1,pci_dev_2
:param mapping_list: list of string pairs in "key:value" format
the key part represents the physnet or function name
the value part is a list of device name separated by ","
:returns: a dict of valid fields.
"""
mapping = {}
for dev_mapping in mapping_list:
try:
physnet_or_function, devices = dev_mapping.split(":", 1)
except ValueError:
raise ValueError(("Invalid mapping: '%s'") % dev_mapping)
physnet_or_function = physnet_or_function.strip()
if not physnet_or_function:
raise ValueError(("Missing key in mapping: '%s'") % dev_mapping)
if physnet_or_function in mapping:
raise ValueError(
("Key %(physnet_or_function)s in mapping: %(mapping)s "
"not unique") % {'physnet_or_function': physnet_or_function,
'mapping': dev_mapping})
mapping[physnet_or_function] = set(dev.strip() for dev in
devices.split("|") if dev.strip())
return mapping

View File

@ -47,7 +47,7 @@ class IntelFPGADriver(FPGADriver):
"""Base class for FPGA drivers.
This is just a virtual FPGA drivers interface.
Vedor should implement their specific drivers.
Vendor should implement their specific drivers.
"""
VENDOR = "intel"

View File

@ -0,0 +1,51 @@
# Copyright 2020 Intel, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Cyborg NIC driver implementation.
"""
VENDOR_MAPS = {"0x8086": "intel"}
class NICDriver(object):
"""Base class for Nic drivers.
This is just a virtual NIC drivers interface.
Vendor should implement their specific drivers.
"""
@classmethod
def create(cls, vendor, *args, **kwargs):
for sclass in cls.__subclasses__():
vendor_name = VENDOR_MAPS.get(vendor, vendor)
if vendor_name == sclass.VENDOR:
return sclass(*args, **kwargs)
raise LookupError("Not find the NIC driver for vendor %s" % vendor)
def discover(self):
"""Discover NIC information of current vendor(Identified by class).
:return: List of NIC information dict.
"""
raise NotImplementedError()
@classmethod
def discover_vendors(cls):
"""Discover NIC vendors of current node.
:return: NIC vendor ID list.
"""
raise NotImplementedError()

View File

@ -0,0 +1,34 @@
# Copyright 2020 Intel, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Cyborg Intel NIC driver implementation.
"""
from cyborg.accelerator.drivers.nic.base import NICDriver
from cyborg.accelerator.drivers.nic.intel import sysinfo
class IntelNICDriver(NICDriver):
"""Class for Intel NIC drivers.
Vendor should implement their specific drivers in this class.
"""
VENDOR = "intel"
def __init__(self, *args, **kwargs):
pass
def discover(self):
return sysinfo.nic_tree()

View File

@ -0,0 +1,253 @@
# Copyright 2020 Intel, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Cyborg Intel NIC driver implementation.
"""
import glob
import os
import socket
from oslo_config import cfg
from oslo_log import log as logging
from oslo_serialization import jsonutils
from cyborg.accelerator.common import utils
import cyborg.conf
from cyborg.objects.driver_objects import driver_attach_handle
from cyborg.objects.driver_objects import driver_attribute
from cyborg.objects.driver_objects import driver_controlpath_id
from cyborg.objects.driver_objects import driver_deployable
from cyborg.objects.driver_objects import driver_device
LOG = logging.getLogger(__name__)
PCI_DEVICES_PATH_PATTERN = "/sys/bus/pci/devices/*"
KNOWN_NICS = [("0x8086", "0x158b"), ("0x8086", "0x1572")]
DRIVER_NAME = "intel"
VF = "virtfn*"
_SRIOV_TOTALVFS = "sriov_totalvfs"
CONF = cyborg.conf.CONF
def _parse_config():
# parse nic config.
cyborg.conf.devices.register_dynamic_opts(CONF)
try:
pdm = utils.parse_mappings(CONF.x710_static.physical_device_mappings)
fdm = utils.parse_mappings(CONF.x710_static.function_device_mappings)
except cfg.NoSuchOptError:
return None, None
else:
return pdm, fdm
def get_physical_network_and_traits(pci_info, physnet_device_mappings,
function_device_mappings, pf_nic=None):
traits = []
physnet = None
func_name = None
if pf_nic:
pf_addr = pf_nic["device"]
traits.append("CUSTOM_VF")
else:
pf_addr = pci_info["PCI_SLOT_NAME"]
traits.append("CUSTOM_PF")
if not pf_addr:
LOG.error("Incorrect report data received. Missing PF info.")
pf_ifname = utils.get_ifname_by_pci_address(pf_addr)
if physnet_device_mappings:
for key, devices in physnet_device_mappings.items():
if pf_ifname in devices:
physnet = key
break
if function_device_mappings:
for key, devices in function_device_mappings.items():
if pf_ifname in devices:
func_name = key
break
if func_name:
traits.append("CUSTOM_" + func_name.upper())
if physnet:
traits.append("CUSTOM_" + physnet.upper())
return {"traits": traits, "physical_network": physnet}
def read_line(filename):
with open(filename) as f:
return f.readline().strip()
def find_nics_by_know_list():
return set(filter(
lambda p: (
read_line(os.path.join(p, "vendor")),
read_line(os.path.join(p, "device"))
) in KNOWN_NICS,
glob.glob(PCI_DEVICES_PATH_PATTERN)))
def pci_attributes(path):
with open(os.path.join(path, "uevent")) as f:
attributes = dict(map(
lambda p: p.strip().split("="),
f.readlines()
))
with open(os.path.join(path, "vendor")) as f:
attributes["VENDOR"] = f.readline().strip()
with open(os.path.join(path, "device")) as f:
attributes["PRODUCT_ID"] = f.readline().strip()
return attributes
def nic_gen(path, physnet_device_mappings=None, function_device_mappings=None,
pf_nic=None):
pci_info = pci_attributes(path)
nic = {
"name": "_".join((socket.gethostname(), pci_info["PCI_SLOT_NAME"])),
"device": pci_info["PCI_SLOT_NAME"],
"type": "NIC",
"vendor": pci_info["VENDOR"],
"product_id": pci_info["PRODUCT_ID"],
"rc": "CUSTOM_NIC",
"stub": False,
}
# TODO(Xinran): need check device id and call get_traits differently.
updates = get_physical_network_and_traits(pci_info,
physnet_device_mappings,
function_device_mappings,
pf_nic)
nic.update(updates)
return nic
def all_pfs_with_vf():
return set(filter(
lambda p: glob.glob(os.path.join(p, VF)),
find_nics_by_know_list()))
def all_vfs_in_pf(pf_path):
return map(
lambda p:
os.path.join(
os.path.dirname(os.path.dirname(p)),
os.path.basename(os.readlink(p))),
glob.glob(os.path.join(pf_path, VF)))
def nic_tree():
physnet_device_mappings, function_device_mappings = _parse_config()
nics = []
pfs_has_vf = all_pfs_with_vf()
for n in find_nics_by_know_list():
nic = nic_gen(n, physnet_device_mappings, function_device_mappings)
if n in pfs_has_vf:
vfs = []
for vf in all_vfs_in_pf(n):
vf_nic = nic_gen(vf, physnet_device_mappings,
function_device_mappings, nic)
vfs.append(vf_nic)
nic["vfs"] = vfs
nics.append(_generate_driver_device(nic))
return nics
def _generate_driver_device(nic):
driver_device_obj = driver_device.DriverDevice()
driver_device_obj.vendor = nic["vendor"]
driver_device_obj.stub = nic["stub"]
driver_device_obj.model = nic.get("model", "miss_model_info")
driver_device_obj.vendor_board_info = nic.get(
"vendor_board_info",
"miss_vb_info")
std_board_info = {"product_id": nic.get("product_id", None)}
driver_device_obj.std_board_info = jsonutils.dumps(std_board_info)
driver_device_obj.type = nic["type"]
driver_device_obj.controlpath_id = _generate_controlpath_id(nic)
driver_device_obj.deployable_list = _generate_dep_list(nic)
return driver_device_obj
def _generate_controlpath_id(nic):
driver_cpid = driver_controlpath_id.DriverControlPathID()
driver_cpid.cpid_type = "PCI"
driver_cpid.cpid_info = utils.pci_str_to_json(nic["device"])
return driver_cpid
def _generate_dep_list(nic):
dep_list = []
# pf without sriov enabled.
if "vfs" not in nic:
driver_dep = driver_deployable.DriverDeployable()
driver_dep.num_accelerators = 1
driver_dep.attach_handle_list = [
_generate_attach_handle(nic)]
driver_dep.name = nic["name"]
driver_dep.driver_name = DRIVER_NAME
driver_dep.attribute_list = _generate_attribute_list(nic)
dep_list = [driver_dep]
# pf with sriov enabled, may have several vfs.
else:
for vf in nic["vfs"]:
driver_dep = driver_deployable.DriverDeployable()
driver_dep.num_accelerators = 1
driver_dep.attach_handle_list = [
_generate_attach_handle(vf)]
driver_dep.name = vf["name"]
driver_dep.driver_name = DRIVER_NAME
driver_dep.attribute_list = _generate_attribute_list(vf)
dep_list.append(driver_dep)
return dep_list
def _generate_attach_handle(nic):
driver_ah = driver_attach_handle.DriverAttachHandle()
driver_ah.attach_type = "PCI"
driver_ah.attach_info = utils.pci_str_to_json(nic["device"],
nic["physical_network"])
driver_ah.in_use = False
return driver_ah
def _generate_attribute_list(nic):
attr_list = []
for k, v in nic.items():
if k == "rc":
driver_attr = driver_attribute.DriverAttribute()
driver_attr.key, driver_attr.value = k, v
attr_list.append(driver_attr)
if k == "traits":
values = nic.get(k, [])
for idx, val in enumerate(values):
driver_attr = driver_attribute.DriverAttribute()
driver_attr.key = "trait" + str(idx)
driver_attr.value = val
attr_list.append(driver_attr)
return attr_list

View File

@ -20,6 +20,7 @@ DEVICE_GPU = 'GPU'
DEVICE_FPGA = 'FPGA'
DEVICE_AICHIP = 'AICHIP'
DEVICE_QAT = 'QAT'
DEVICE_NIC = 'NIC'
ARQ_STATES = (ARQ_INITIAL, ARQ_BIND_STARTED, ARQ_BOUND, ARQ_UNBOUND,
@ -58,7 +59,7 @@ ARQ_STATES_TRANSFORM_MATRIX = {
# Device type
DEVICE_TYPE = (DEVICE_GPU, DEVICE_FPGA, DEVICE_AICHIP, DEVICE_QAT)
DEVICE_TYPE = (DEVICE_GPU, DEVICE_FPGA, DEVICE_AICHIP, DEVICE_QAT, DEVICE_NIC)
# Attach handle type
@ -76,7 +77,8 @@ RESOURCES = {
"FPGA": orc.FPGA,
"PGPU": orc.PGPU,
"VGPU": orc.VGPU,
"QAT": "CUSTOM_QAT"
"QAT": "CUSTOM_QAT",
"NIC": "CUSTOM_NIC"
}
@ -90,8 +92,8 @@ ACCEL_SPECS = (
SUPPORT_RESOURCES = (
FPGA, GPU, VGPU, PGPU, QAT) = (
"FPGA", "GPU", "VGPU", "PGPU", "CUSTOM_QAT"
FPGA, GPU, VGPU, PGPU, QAT, NIC) = (
"FPGA", "GPU", "VGPU", "PGPU", "CUSTOM_QAT", "CUSTOM_NIC"
)

View File

@ -415,3 +415,7 @@ class NotAcceptable(CyborgException):
class FPGAProgramError(CyborgException):
_msg_fmt = _("FPGA programming failed with return %(ret)s.")
class PciDeviceNotFoundById(NotFound):
_msg_fmt = _("PCI device %(id)s not found")

View File

@ -19,6 +19,7 @@ from cyborg.conf import agent
from cyborg.conf import api
from cyborg.conf import database
from cyborg.conf import default
from cyborg.conf import devices
from cyborg.conf import glance
from cyborg.conf import keystone
from cyborg.conf import nova
@ -31,6 +32,7 @@ api.register_opts(CONF)
agent.register_opts(CONF)
database.register_opts(CONF)
default.register_opts(CONF)
devices.register_opts(CONF)
service_token.register_opts(CONF)
glance.register_opts(CONF)
keystone.register_opts(CONF)

57
cyborg/conf/devices.py Normal file
View File

@ -0,0 +1,57 @@
# Copyright 2020 Intel, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_config import cfg
nic_group = cfg.OptGroup(
name='nic_devices',
title='nic device ID options',
help="""This is used to config specific nic devices.
""")
nic_opts = [
cfg.ListOpt('enabled_nic_types',
default=[],
help=" ")
]
def register_opts(conf):
conf.register_group(nic_group)
conf.register_opts(nic_opts, group=nic_group)
def register_dynamic_opts(conf):
"""Register dynamically-generated options and groups.
This must be called by the service that wishes to use the options **after**
the initial configuration has been loaded.
"""
opts = [
cfg.ListOpt('physical_device_mappings', default=[],
item_type=cfg.types.String()),
cfg.ListOpt('function_device_mappings', default=[],
item_type=cfg.types.String()),
]
# Register the '[nic_type]/physical_device_mappings' and
# '[nic_type]/function_device_mappings' opts, implicitly
# registering the '[nic_type]' groups in the process
for nic_type in conf.nic_devices.enabled_nic_types:
conf.register_opts(opts, group=nic_type)
def list_opts():
return {nic_group: nic_opts}

View File

@ -0,0 +1,22 @@
"""add_nic_type
Revision ID: 899cead40bc9
Revises: 7e6f1f107f2b
Create Date: 2020-09-18 02:33:42.640673
"""
# revision identifiers, used by Alembic.
revision = '899cead40bc9'
down_revision = '7e6f1f107f2b'
from alembic import op
import sqlalchemy as sa
def upgrade():
new_device_type = sa.Enum('GPU', 'FPGA', 'AICHIP', 'QAT', 'NIC',
name='device_type')
op.alter_column('devices', 'type',
existing_type=new_device_type,
nullable=False)

View File

@ -81,7 +81,7 @@ class Device(Base):
id = Column(Integer, primary_key=True)
uuid = Column(String(36), nullable=False, unique=True)
type = Column(Enum('GPU', 'FPGA', 'AICHIP', 'QAT',
type = Column(Enum('GPU', 'FPGA', 'AICHIP', 'QAT', 'NIC',
name='device_type'), nullable=False)
vendor = Column(String(255), nullable=False)
model = Column(String(255), nullable=False)

View File

@ -13,7 +13,6 @@
# License for the specific language governing permissions and limitations
# under the License.
import argparse
import copy
import glob
import os
@ -282,22 +281,3 @@ def create_fake_sysfs(prefix=""):
os.makedirs(sys_class_fpga)
create_devices_path_and_files(FPGA_TREE, sys_device, sys_class_fpga)
create_devices_soft_link(sys_class_fpga)
def main():
create_fake_sysfs()
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Generate a fake sysfs for intel FPGA.")
group = parser.add_mutually_exclusive_group()
group.add_argument("-v", "--verbose", action="store_true")
group.add_argument("-q", "--quiet", action="store_true")
parser.add_argument("-p", "--prefix", type=str,
default="/tmp", dest="p",
help='Set the prefix path of the fake sysfs. '
'default "/tmp"')
args = parser.parse_args()
create_fake_sysfs(args.p)

View File

@ -0,0 +1,245 @@
#!/usr/bin/python
# Copyright 2021 Intel, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import copy
import os
import shutil
PF0_ADDR = "0000:05:00.0"
PF1_ADDR = "0000:06:00.0"
VF0_ADDR = "0000:05:01.0"
NIC_TREE = {
"dev.0": {"bdf": PF0_ADDR,
"vfs": {"dev.2": {"bdf": VF0_ADDR}}},
"dev.1": {"bdf": PF1_ADDR}}
SYS_DEVICES = "sys/devices"
PCI_DEVICES_PATH = "sys/bus/pci/devices"
DEV_PREFIX = "intel-nic"
NIC_DEVICE_COMMON_SUB_DIR = ["power", "msi_irqs"]
NIC_DEVICE_COMMON_CONTENT = {
"broken_parity_status": "0",
"class": "0x0b4000",
"config": "",
"consistent_dma_mask_bits": "64",
"d3cold_allowed": "1",
"device": "0x158b",
"dma_mask_bits": "64",
"driver_override": "(null)",
"enable": "1",
"irq": "33",
"local_cpulist": "0-7,16-23",
"local_cpus": "000000,00000000,00000000,00000000,00ff00ff",
"max_link_speed": "5 GT/s",
"max_link_width": "16",
"modalias": "pci:v00008086d000037C8sv00008086sd00000002bc0Bsc40i00",
"msi_bus": "1",
"numa_node": "0",
"resource0": "",
"subsystem_device": "0x0002",
"subsystem_vendor": "0x8086",
"vendor": "0x8086"}
NIC_DEVICES_SPECIAL_COMMON_CONTENT = {
"dev.0": {
"resource": [
"0x0000000000000000 0x0000000000000000 0x0000000000000000",
"0x0000000000000000 0x0000000000000000 0x0000000000000000",
"0x00000000d1340000 0x00000000d137ffff 0x0000000000140204",
"0x0000000000000000 0x0000000000000000 0x0000000000000000",
"0x00000000d1300000 0x00000000d133ffff 0x0000000000140204",
"0x0000000000000000 0x0000000000000000 0x0000000000000000",
"0x0000000000000000 0x0000000000000000 0x0000000000000000",
"0x00000000d1390000 0x00000000d139ffff 0x0000000000140204",
"0x0000000000000000 0x0000000000000000 0x0000000000000000",
"0x00000000d1390000 0x00000000d139ffff 0x0000000000140204",
"0x0000000000000000 0x0000000000000000 0x0000000000000000",
"0x0000000000000000 0x0000000000000000 0x0000000000000000",
"0x0000000000000000 0x0000000000000000 0x0000000000000000"],
"resource2": "",
"resource4": "",
"sriov_numvfs": "1",
"sriov_totalvfs": "1",
"uevent": [
"DRIVER=c6xx",
"PCI_CLASS=B4000",
"PCI_ID=8086:37C8",
"PCI_SUBSYS_ID=8086:0002",
"PCI_SLOT_NAME=0000:05:00.0",
"MODALIAS=pci:v00008086d000037C8sv00008086sd00000002bc0Bsc40i00"],
},
"dev.1": {
"resource": [
"0x00000000fbc00000 0x00000000fbc7ffff 0x000000000014220c",
"0x0000000000000000 0x0000000000000000 0x0000000000000000",
"0x00000000fbc80000 0x00000000fbcfffff 0x000000000014220c",
"0x0000000000000000 0x0000000000000000 0x0000000000000000",
"0x0000000000000000 0x0000000000000000 0x0000000000000000",
"0x0000000000000000 0x0000000000000000 0x0000000000000000",
"0x0000000000000000 0x0000000000000000 0x0000000000000000",
"0x00000000fbd00000 0x00000000fbd7ffff 0x000000000014220c",
"0x0000000000000000 0x0000000000000000 0x0000000000000000",
"0x0000000000000000 0x0000000000000000 0x0000000000000000",
"0x0000000000000000 0x0000000000000000 0x0000000000000000",
"0x0000000000000000 0x0000000000000000 0x0000000000000000",
"0x0000000000000000 0x0000000000000000 0x0000000000000000"],
"resource2": "",
"sriov_numvfs": "0",
"sriov_totalvfs": "0",
"uevent": [
"DRIVER=c6xx",
"PCI_CLASS=B4000",
"PCI_ID=8086:37C8",
"PCI_SUBSYS_ID=8086:0002",
"PCI_SLOT_NAME=0000:06:00.0",
"MODALIAS=pci:v00008086d000037C8sv00008086sd00000002bc0Bsc40i00"],
},
"dev.2": {
"d3cold_allowed": "0",
"device": "0x37c9",
"modalias": "pci:v00008086d000037C9sv00008086sd00000000bc0Bsc40i00",
"irq": "0",
"resource": [
"0x00000000c6100000 0x00000000c617ffff 0x000000000014220c",
"0x0000000000000000 0x0000000000000000 0x0000000000000000",
"0x0000000000000000 0x0000000000000000 0x0000000000000000",
"0x0000000000000000 0x0000000000000000 0x0000000000000000",
"0x0000000000000000 0x0000000000000000 0x0000000000000000",
"0x0000000000000000 0x0000000000000000 0x0000000000000000",
"0x0000000000000000 0x0000000000000000 0x0000000000000000",
"0x0000000000000000 0x0000000000000000 0x0000000000000000",
"0x0000000000000000 0x0000000000000000 0x0000000000000000",
"0x0000000000000000 0x0000000000000000 0x0000000000000000",
"0x0000000000000000 0x0000000000000000 0x0000000000000000",
"0x0000000000000000 0x0000000000000000 0x0000000000000000",
"0x0000000000000000 0x0000000000000000 0x0000000000000000"],
"uevent": [
"DRIVER=c6xx",
"PCI_CLASS=B4000",
"PCI_ID=8086:37C8",
"PCI_SUBSYS_ID=8086:0002",
"PCI_SLOT_NAME=0000:05:01.0",
"MODALIAS=pci:v00008086d000037C8sv00008086sd00000002bc0Bsc40i00"],
}
}
NIC_DEVICE_COMMON_SOFT_LINK = {
"driver": "../../../../../../bus/pci/drivers/c6xx",
"iommu": "../../../../../virtual/iommu/dmar1",
"subsystem": "../../../../../../bus/pci"
}
NIC_DEVICES_SPECIAL_SOFT_LINK = {
"dev.0": {
"driver": "../../../../../..//bus/pci/drivers/c6xx",
"iommu_group": "../../../../../../kernel/iommu_groups/20",
# "virtfn0": "../0000:05:01.0/",
},
"dev.1": {
"driver": "../../../../../..//bus/pci/drivers/c6xx",
"iommu_group": "../../../../../../kernel/iommu_groups/21",
},
"dev.2": {
"iommu_group": "../../../../../../kernel/iommu_groups/67",
# "physfn": "../0000:05:00.0/",
}
}
NIC_DEVICE_PF_SOFT_LINK = {
"virtfn": lambda k, v: (k + str(int(v.rsplit(".", 1)[-1])),
"/".join(["..", v]))
}
NIC_DEVICE_VF_SOFT_LINK = {
"physfn": lambda k, v: (k, "/".join(["..", v]))
}
def gen_nic_content(path, dev):
content = copy.copy(NIC_DEVICE_COMMON_CONTENT)
content.update(NIC_DEVICES_SPECIAL_COMMON_CONTENT[dev])
for k, v in content.items():
p = os.path.join(path, k)
if not v:
os.mknod(p)
elif type(v) is str:
with open(p, 'a') as f:
f.write(v + "\n")
elif type(v) is list:
with open(p, 'a') as f:
f.writelines([l + "\n" for l in v])
def gen_nic_sub_dir(path):
for d in NIC_DEVICE_COMMON_SUB_DIR:
p = os.path.join(path, d)
os.makedirs(p)
def gen_nic_pf_soft_link(path, bdf):
for k, v in NIC_DEVICE_PF_SOFT_LINK.items():
if callable(v):
k, v = v(k, bdf)
os.symlink(v, os.path.join(path, k))
def gen_nic_common_soft_link(path, bdf):
for k, v in NIC_DEVICE_COMMON_SOFT_LINK.items():
os.symlink(v, os.path.join(path, k))
def gen_nic_vf_soft_link(path, bdf):
for k, v in NIC_DEVICE_VF_SOFT_LINK.items():
if callable(v):
k, v = v(k, bdf)
os.symlink(v, os.path.join(path, k))
def create_devices_path_and_files(tree, device_path, vf=False, pfinfo=None):
for k, v in tree.items():
bdf = v["bdf"]
pci_path = "pci" + bdf.rsplit(":", 1)[0]
bdf_path = os.path.join(device_path, pci_path, bdf)
ln = "-".join([DEV_PREFIX, k])
dev_path = os.path.join(bdf_path, "nic", ln)
os.makedirs(dev_path)
gen_nic_content(bdf_path, k)
gen_nic_sub_dir(bdf_path)
if vf:
gen_nic_pf_soft_link(pfinfo["path"], bdf)
gen_nic_vf_soft_link(bdf_path, pfinfo["bdf"])
pfinfo = {"path": bdf_path, "bdf": bdf}
if "vfs" in v:
create_devices_path_and_files(
v["vfs"], device_path, True, pfinfo)
os.symlink("../../../" + bdf, os.path.join(dev_path, "device"))
pci_dev = os.path.join(device_path.split(SYS_DEVICES)[0],
PCI_DEVICES_PATH)
if not os.path.exists(pci_dev):
os.makedirs(pci_dev)
os.symlink("../../.." + bdf_path.split("sys")[-1],
os.path.join(pci_dev, bdf))
def create_fake_sysfs(prefix=""):
sys_device = os.path.join(prefix, SYS_DEVICES)
basedir = os.path.dirname(sys_device)
if os.path.exists(basedir):
shutil.rmtree(basedir, ignore_errors=False, onerror=None)
create_devices_path_and_files(NIC_TREE, sys_device)

View File

@ -0,0 +1,140 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
from unittest import mock
from cyborg.accelerator.drivers.nic.intel import sysinfo
from cyborg.accelerator.drivers.nic.intel.driver import IntelNICDriver
from cyborg.tests import base
from cyborg.tests.unit.accelerator.drivers.nic.intel import prepare_test_data
import fixtures
class TestIntelNICDriver(base.TestCase):
def setUp(self):
super(TestIntelNICDriver, self).setUp()
self.pcipath = sysinfo.PCI_DEVICES_PATH_PATTERN
tmp_sys_dir = self.useFixture(fixtures.TempDir())
prepare_test_data.create_fake_sysfs(tmp_sys_dir.path)
tmp_path = tmp_sys_dir.path
sysinfo.PCI_DEVICES_PATH_PATTERN = os.path.join(
tmp_path, sysinfo.PCI_DEVICES_PATH_PATTERN.split("/", 1)[-1])
def tearDown(self):
super(TestIntelNICDriver, self).tearDown()
sysinfo.PCI_DEVICES_PATH = self.pcipath
@mock.patch("cyborg.accelerator.common.utils.get_ifname_by_pci_address")
def test_discover(self, mock_device_ifname):
mock_device_ifname.return_value = "ethx"
attach_handle_list = [
[
{'attach_type': 'PCI',
'attach_info': '{"bus": "05", '
'"device": "01", '
'"domain": "0000", '
'"function": "0"}',
'in_use': False}
],
[
{'attach_type': 'PCI',
'attach_info': '{"bus": "06", '
'"device": "00", '
'"domain": "0000", '
'"function": "0"}',
'in_use': False}
]
]
attribute_list = [
[
{
"key": "rc",
"value": "CUSTOM_NIC"
},
{
"key": "trait0",
"value": "CUSTOM_VF"
}
],
[
{
"key": "rc",
"value": "CUSTOM_NIC"
},
{
"key": "trait0",
"value": "CUSTOM_PF"
}
],
]
expected = [{'vendor': '0x8086',
'type': 'NIC',
'deployable_list':
[
{'num_accelerators': 1,
'name': '0000:05:01.0',
'attach_handle_list': attach_handle_list[0],
'attribute_list':attribute_list[0]
},
],
'controlpath_id':
{
'cpid_info': '{"bus": "05", '
'"device": "00", '
'"domain": "0000", '
'"function": "0"}',
'cpid_type': 'PCI'}
},
{'vendor': '0x8086',
'type': 'NIC',
'deployable_list':
[
{'num_accelerators': 1,
'name': '0000:06:00.0',
'attach_handle_list': attach_handle_list[1],
'attribute_list':attribute_list[1]
},
],
'controlpath_id':
{
'cpid_info': '{"bus": "06", '
'"device": "00", '
'"domain": "0000", '
'"function": "0"}',
'cpid_type': 'PCI'}
}
]
intel = IntelNICDriver()
nics = intel.discover()
list.sort(nics, key=lambda x: x._obj_deployable_list[0].name)
self.assertEqual(2, len(nics))
for i in range(len(nics)):
nic_dict = nics[i].as_dict()
nic_dep_list = nic_dict['deployable_list']
nic_attach_handle_list = \
nic_dep_list[0].as_dict()['attach_handle_list']
nic_attribute_list = \
nic_dep_list[0].as_dict()['attribute_list']
self.assertEqual(expected[i]['vendor'], nic_dict['vendor'])
self.assertEqual(expected[i]['controlpath_id'],
nic_dict['controlpath_id'])
self.assertEqual(expected[i]['deployable_list'][0]
['num_accelerators'],
nic_dep_list[0].as_dict()['num_accelerators'])
self.assertEqual(1, len(nic_attach_handle_list))
self.assertEqual(attach_handle_list[i][0],
nic_attach_handle_list[0].as_dict())
self.assertEqual(attribute_list[i][0],
nic_attribute_list[0].as_dict())

View File

@ -0,0 +1,31 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from cyborg.accelerator.drivers.nic.base import NICDriver
from cyborg.accelerator.drivers.nic.intel.driver import IntelNICDriver # noqa
from cyborg.tests import base
class TestNICDriver(base.TestCase):
def test_create(self):
NICDriver.create("intel")
self.assertRaises(LookupError, NICDriver.create, "other")
def test_discover(self):
d = NICDriver()
self.assertRaises(NotImplementedError, d.discover)
def test_discover_vendors(self):
d = NICDriver()
self.assertRaises(NotImplementedError, d.discover_vendors)

View File

@ -37,6 +37,11 @@
- The driver for Inspur FPGA Cards.
- None
- Test results reported at Aug 2020. Please reference: `Inspur FPGA Driver Test Report <https://wiki.openstack.org/wiki/Cyborg/TestReport/InspurFPGA>`_
* - Intel NIC Driver
- None
- The driver for Intel NIC Cards.
- None
- Test results reported at Feb 2021. Please reference: `Intel NIC Driver Test Report <https://wiki.openstack.org/wiki/Cyborg/TestReport/IntelNic>`_
.. note:: Temporary Test Report: This is a temporary test report, it is only
valid for a short time, if you encounter problems, please contact the

View File

@ -0,0 +1,10 @@
---
features:
- |
The Intel nic driver defines the Intel x710 NIC's data model in
Cyborg. It also proposes a standard configuration format to manage
networking related devices. The Intel X710 NIC supports DDP(Dynamic
Device Personalization) which provides the ability to reconfigure the
packet processing pipeline to support a broader range of traffic types.
It also supports SR-IOV technology, each physical card can be virtualized
into mulitiple VFs.

View File

@ -53,6 +53,7 @@ cyborg.accelerator.driver =
fake_driver = cyborg.accelerator.drivers.fake:FakeDriver
huawei_ascend_driver = cyborg.accelerator.drivers.aichip.huawei.ascend:AscendDriver
intel_qat_driver = cyborg.accelerator.drivers.qat.intel.driver:IntelQATDriver
intel_nic_driver = cyborg.accelerator.drivers.nic.intel.driver:IntelNICDriver
oslo.config.opts =
cyborg = cyborg.conf.opts:list_opts