openstack/zun
Code Issues Proposed changes

port pci tracker from nova to zun

Browse Source 
Change-Id: Ie9f278afb1af68379f8f641eba2e97dd582697cc
Depends-On: I93ce4fdec7e6d870ee4e57e73374089ba27becfe
Partially-Implements: blueprint support-pcipassthroughfilter
This commit is contained in:
ShunliZhou 2017-09-26 10:53:26 +08:00 committed by Hongbin Lu
parent 4d91ab84bb
commit f44af3de86
13 changed files with 1019 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
zun/compute/compute_node_tracker.py
View File

@ -22,6 +22,7 @@ from zun.common import utils
from zun.compute import claims
from zun import objects
from zun.objects import base as obj_base
from zun.pci import manager as pci_manager
from zun.scheduler import client as scheduler_client
LOG = logging.getLogger(__name__)
@ -36,6 +37,17 @@ class ComputeNodeTracker(object):
self.tracked_containers = {}
self.old_resources = collections.defaultdict(objects.ComputeNode)
self.scheduler_client = scheduler_client.SchedulerClient()
self.pci_tracker = None
def _setup_pci_tracker(self, context, compute_node):
if not self.pci_tracker:
n_id = compute_node.uuid
self.pci_tracker = pci_manager.PciDevTracker(context, node_id=n_id)
dev_json = self.container_driver.get_pci_resources()
self.pci_tracker.update_devices_from_compute_resources(dev_json)
dev_pools_obj = self.pci_tracker.stats.to_device_pools_obj()
compute_node.pci_device_pools = dev_pools_obj
def update_available_resources(self, context):
# Check if the compute_node is already registered
@ -49,6 +61,7 @@ class ComputeNodeTracker(object):
node.create(context)
LOG.info('Node created for :%(host)s', {'host': self.host})
self.container_driver.get_available_resources(node)
self._setup_pci_tracker(context, node)
self.compute_node = node
self._update_available_resource(context)
# NOTE(sbiswas7): Consider removing the return statement if not needed
@ -175,7 +188,9 @@ class ComputeNodeTracker(object):
return
# Persist the stats to the Scheduler
self.scheduler_client.update_resource(compute_node)
# Update pci tracker here
if self.pci_tracker:
self.pci_tracker.save()
def _resource_change(self, compute_node):
"""Check to see if any resources have changed."""

3
zun/container/driver.py
View File

@ -183,6 +183,9 @@ class ContainerDriver(object):
os_capability_linux.LinuxHost().get_host_numa_topology(numa_topo_obj)
return numa_topo_obj
def get_pci_resources(self):
return os_capability_linux.LinuxHost().get_pci_resources()
def get_host_mem(self):
return os_capability_linux.LinuxHost().get_host_mem()

157
zun/container/os_capability/host_capability.py
View File

@ -13,8 +13,14 @@
# License for the specific language governing permissions and limitations
# under the License.
from oslo_concurrency import processutils
from oslo_serialization import jsonutils
from zun.common import exception
from zun.common import utils
from zun import objects
from zun.objects import fields
from zun.pci import utils as pci_utils
class Host(object):
@ -67,3 +73,154 @@ class Host(object):
mem_ava = int(mem_free) + int(buffers) + int(cached)
mem_used = int(mem_total) - int(mem_ava)
return int(mem_total), int(mem_free), int(mem_ava), int(mem_used)
def get_pci_resources(self):
addresses = []
try:
output, status = processutils.execute('lspci', '-D', '-nnmm')
lines = output.split('\n')
for line in lines:
if not line:
continue
columns = line.split()
address = columns[0]
addresses.append(address)
except processutils.ProcessExecutionError:
raise exception.CommandError(cmd='lspci')
pci_info = []
for addr in addresses:
pci_info.append(self._get_pci_dev_info(addr))
return jsonutils.dumps(pci_info)
def _get_pci_dev_info(self, address):
"""Returns a dict of PCI device."""
def _get_device_type(address):
"""Get a PCI device's device type.
An assignable PCI device can be a normal PCI device,
a SR-IOV Physical Function (PF), or a SR-IOV Virtual
Function (VF). Only normal PCI devices or SR-IOV VFs
are assignable.
"""
try:
path = '/sys/bus/pci/devices/' + address + '/'
output, status = processutils.execute('ls', path)
if "physfn" in output:
phys_address = None
upath = '/sys/bus/pci/devices/%s/physfn/uevent' % address
try:
ou, st = processutils.execute('cat', upath)
lines = ou.split('\n')
for line in lines:
if 'PCI_SLOT_NAME' in line:
columns = line.split("=")
phys_address = columns[1]
except processutils.ProcessExecutionError:
raise exception.CommandError(cmd='cat')
return {'dev_type': fields.PciDeviceType.SRIOV_VF,
'parent_addr': phys_address}
if "virtfn" in output:
return {'dev_type': fields.PciDeviceType.SRIOV_PF}
except processutils.ProcessExecutionError:
raise exception.CommandError(cmd='ls')
return {'dev_type': fields.PciDeviceType.STANDARD}
def _get_device_capabilities(device, address):
"""Get PCI VF device's additional capabilities.
If a PCI device is a virtual function, this function reads the PCI
parent's network capabilities (must be always a NIC device) and
appends this information to the device's dictionary.
"""
if device.get('dev_type') == fields.PciDeviceType.SRIOV_VF:
pcinet_info = self._get_pcinet_info(address)
if pcinet_info:
return {'capabilities':
{'network': pcinet_info.get('capabilities')}}
return {}
def _get_product_and_vendor(address):
try:
output, status = processutils.execute('lspci', '-n', '-s',
address)
value = output.split()[2]
result = value.split(":")
return result[0], result[1]
except processutils.ProcessExecutionError:
raise exception.CommandError(cmd='lspci')
def _get_numa_node(address):
numa_node = None
try:
output, status = processutils.execute('lspci', '-vmm', '-s',
address)
lines = output.split('\n')
for line in lines:
if 'NUMANode' in line:
numa_node = int(line.split(":")[1])
except processutils.ProcessExecutionError:
raise exception.CommandError(cmd='lspci')
return numa_node
dev_name = 'pci_' + address.replace(":", "_").replace(".", "_")
product_id, vendor_id = _get_product_and_vendor(address)
numa_node = _get_numa_node(address)
device = {
"dev_id": dev_name,
"address": address,
"product_id": product_id,
"vendor_id": vendor_id,
"numa_node": numa_node
}
device['label'] = 'label_%(vendor_id)s_%(product_id)s' % device
device.update(_get_device_type(address))
device.update(_get_device_capabilities(device, address))
return device
def _get_pcinet_info(self, vf_address):
"""Returns a dict of NET device."""
devname = pci_utils.get_net_name_by_vf_pci_address(vf_address)
if not devname:
return
ifname = pci_utils.get_ifname_by_pci_address(vf_address)
# Features from the that libvirt supported, get them by ethtool -k
# Note: I cannot find the rdma feature returned by ethtool, correct me
# if the string is wrong.
FEATURES_LIST = ['rx-checksumming', 'tx-checksumming',
'scatter-gather', 'tcp-segmentation-offload',
'generic-segmentation-offload',
'generic-receive-offload', 'large-receive-offload',
'rx-vlan-offload', 'tx-vlan-offload',
'ntuple-filters', 'receive-hashing',
'tx-udp_tnl-segmentation', 'rdma']
FEATURES_MAP = {'rx-checksumming': 'rx',
'tx-checksumming': 'tx',
'scatter-gather': 'sg',
'tcp-segmentation-offload': 'tso',
'generic-segmentation-offload': 'gso',
'generic-receive-offload': 'gro',
'large-receive-offload': 'lro',
'rx-vlan-offload': 'rxvlan',
'tx-vlan-offload': 'txvlan',
'ntuple-filters': 'ntuple',
'receive-hashing': 'rxhash',
'tx-udp_tnl-segmentation': 'txudptnl',
'rdma': 'rdma'}
features = []
try:
output, status = processutils.execute('ethtool', '-k', ifname)
lines = output.split('\n')
for line in lines:
columns = line.split(":")
if columns[0].strip() in FEATURES_LIST:
if "on" in columns[1].strip():
features.append(FEATURES_MAP.get(columns[0].strip()))
except processutils.ProcessExecutionError:
raise exception.CommandError(cmd='ethtool -k')
return {'name': devname,
'capabilities': features}

38
zun/db/sqlalchemy/alembic/versions/f046346d1d87_add_timestamp_to_pci_device.py Normal file
View File

@ -0,0 +1,38 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""add timestamp to pci device
Revision ID: f046346d1d87
Revises: ff7b9665d504
Create Date: 2017-10-09 15:30:34.922130
"""
# revision identifiers, used by Alembic.
revision = 'f046346d1d87'
down_revision = 'ff7b9665d504'
branch_labels = None
depends_on = None
from alembic import op
import sqlalchemy as sa
def upgrade():
op.add_column('pci_device',
sa.Column('created_at', sa.DateTime(), nullable=True))
op.add_column('pci_device',
sa.Column('updated_at', sa.DateTime(), nullable=True))
op.drop_column('pci_device', 'request_id')
op.add_column('pci_device', sa.Column('request_id', sa.String(36),
nullable=True))

8
zun/objects/compute_node.py
View File

@ -30,7 +30,8 @@ class ComputeNode(base.ZunPersistentObject, base.ZunObject):
# Version 1.6: Add mem_used to compute node
# Version 1.7: Change get_by_hostname to get_by_name
# Version 1.8: Add pci_device_pools to compute node
VERSION = '1.8'
# Version 1.9: Change PciDevicePoolList to ObjectField
VERSION = '1.9'
fields = {
'uuid': fields.UUIDField(read_only=True, nullable=False),
@ -53,8 +54,8 @@ class ComputeNode(base.ZunPersistentObject, base.ZunObject):
'labels': fields.DictOfStringsField(nullable=True),
# NOTE(pmurray): the pci_device_pools field maps to the
# pci_stats field in the database
'pci_device_pools': fields.ListOfObjectsField('PciDevicePool',
nullable=True),
'pci_device_pools': fields.ObjectField('PciDevicePoolList',
nullable=True),
}
@staticmethod
@ -168,6 +169,7 @@ class ComputeNode(base.ZunPersistentObject, base.ZunObject):
numa_obj = updates.pop('numa_topology', None)
if numa_obj is not None:
updates['numa_topology'] = numa_obj._to_dict()
self._convert_pci_stats_to_db_format(updates)
dbapi.update_compute_node(context, self.uuid, updates)
self.obj_reset_changes(recursive=True)

22
zun/objects/pci_device.py
View File

@ -83,14 +83,15 @@ class PciDevice(base.ZunPersistentObject, base.ZunObject):
"""
# Version 1.0: Initial version
VERSION = '1.0'
# Version 1.1: Change compute_node_uuid to uuid type
VERSION = '1.1'
fields = {
'id': fields.IntegerField(),
'uuid': fields.UUIDField(),
# Note(yjiang5): the compute_node_uuid may be None because the pci
# device objects are created before the compute node is created in DB
'compute_node_uuid': fields.IntegerField(nullable=True),
'compute_node_uuid': fields.UUIDField(nullable=True),
'address': fields.StringField(),
'vendor_id': fields.StringField(),
'product_id': fields.StringField(),
@ -192,7 +193,7 @@ class PciDevice(base.ZunPersistentObject, base.ZunObject):
return pci_device
@base.remotable
def save(self, context):
def save(self):
if self.status == z_fields.PciDeviceStatus.REMOVED:
self.status = z_fields.PciDeviceStatus.DELETED
dbapi.destroy_pci_device(self.compute_node_uuid,
@ -203,9 +204,9 @@ class PciDevice(base.ZunPersistentObject, base.ZunObject):
updates['extra_info'] = jsonutils.dumps(updates['extra_info'])
if updates:
db_pci = dbapi.update_pci_device(self.compute_node_uuid,
self.address, updates)
self._from_db_object(context, self, db_pci)
dbapi.update_pci_device(self.compute_node_uuid,
self.address, updates)
# self._from_db_object(context, self, db_pci)
@staticmethod
def _bulk_update_status(dev_list, status):
@ -373,22 +374,21 @@ class PciDevice(base.ZunPersistentObject, base.ZunObject):
@staticmethod
def _from_db_object_list(db_objects, cls, context):
"""Converts a list of database entities to a list of formal objects."""
return [PciDevice._from_db_object(cls(context), obj)
return [PciDevice._from_db_object(context, cls(context), obj)
for obj in db_objects]
@base.remotable_classmethod
def list_by_compute_node(cls, context, node_id):
db_dev_list = dbapi.get_all_pci_device_by_node(context, node_id)
db_dev_list = dbapi.get_all_pci_device_by_node(node_id)
return PciDevice._from_db_object_list(db_dev_list, cls, context)
@base.remotable_classmethod
def list_by_container_uuid(cls, context, uuid):
db_dev_list = dbapi.get_all_pci_device_by_container_uuid(context, uuid)
db_dev_list = dbapi.get_all_pci_device_by_container_uuid(uuid)
return PciDevice._from_db_object_list(db_dev_list, cls, context)
@base.remotable_classmethod
def list_by_parent_address(cls, context, node_id, parent_addr):
db_dev_list = dbapi.get_all_pci_device_by_parent_addr(context,
node_id,
db_dev_list = dbapi.get_all_pci_device_by_parent_addr(node_id,
parent_addr)
return PciDevice._from_db_object_list(db_dev_list, cls, context)

338
zun/pci/manager.py Normal file
View File

@ -0,0 +1,338 @@
# Copyright (c) 2017 Intel, Inc.
# Copyright (c) 2017 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import collections
from oslo_config import cfg
from oslo_log import log as logging
from oslo_serialization import jsonutils
from zun.common import exception
from zun import objects
from zun.objects import fields
from zun.pci import stats
from zun.pci import whitelist
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
class PciDevTracker(object):
"""Manage pci devices in a compute node.
This class fetches pci passthrough information from compute node
and tracks the usage of these devices.
It's called by compute node resource tracker to allocate and free
devices to/from containers, and to update the available pci passthrough
devices information from compute node periodically.
`pci_devs` attribute of this class is the in-memory "master copy" of all
devices on each compute node, and all data changes that happen when
claiming/allocating/freeing
devices HAVE TO be made against container contained in `pci_devs` list,
because they are periodically flushed to the DB when the save()
method is called.
It is unsafe to fetch PciDevice objects elsewhere in the code for update
purposes as those changes will end up being overwritten when the `pci_devs`
are saved.
"""
def __init__(self, context, node_id=None):
"""Create a pci device tracker.
If a node_id is passed in, it will fetch pci devices information
from database, otherwise, it will create an empty devices list
and the resource tracker will update the node_id information later.
"""
super(PciDevTracker, self).__init__()
self.stale = {}
self.node_id = node_id
self.dev_filter = whitelist.Whitelist(CONF.pci.passthrough_whitelist)
self.stats = stats.PciDeviceStats(dev_filter=self.dev_filter)
self._context = context
if node_id:
self.pci_devs = objects.PciDevice.list_by_compute_node(
context, node_id)
else:
self.pci_devs = []
self._build_device_tree(self.pci_devs)
self._initial_instance_usage()
def _initial_instance_usage(self):
self.allocations = collections.defaultdict(list)
self.claims = collections.defaultdict(list)
for dev in self.pci_devs:
uuid = dev.container_uuid
if dev.status == fields.PciDeviceStatus.CLAIMED:
self.claims[uuid].append(dev)
elif dev.status == fields.PciDeviceStatus.ALLOCATED:
self.allocations[uuid].append(dev)
elif dev.status == fields.PciDeviceStatus.AVAILABLE:
self.stats.add_device(dev)
def save(self):
for dev in self.pci_devs:
if dev.obj_what_changed():
dev.save()
if dev.status == fields.PciDeviceStatus.DELETED:
self.pci_devs.remove(dev)
@property
def pci_stats(self):
return self.stats
def update_devices_from_compute_resources(self, devices_json):
"""Sync the pci device tracker with compute node information.
To support pci device hot plug, we sync with the compute node
periodically, fetching all devices information from compute node,
update the tracker and sync the DB information.
Devices should not be hot-plugged when assigned to a container,
but possibly the compute node has no such guarantee. The best
we can do is to give a warning if a device is changed
or removed while assigned.
:param devices_json: The JSON-ified string of device information
that is returned from the compute node.
"""
devices = []
for dev in jsonutils.loads(devices_json):
if self.dev_filter.device_assignable(dev):
devices.append(dev)
self._set_hvdevs(devices)
@staticmethod
def _build_device_tree(all_devs):
"""Build a tree of devices that represents parent-child relationships.
We need to have the relationships set up so that we can easily make
all the necessary changes to parent/child devices without having to
figure it out at each call site.
This method just adds references to relevant containers already found
in `pci_devs` to `child_devices` and `parent_device` fields of each
one.
Currently relationships are considered for SR-IOV PFs/VFs only.
"""
# Ensures that devices are ordered in ASC so VFs will come
# after their PFs.
all_devs.sort(key=lambda x: x.address)
parents = {}
for dev in all_devs:
if dev.status in (fields.PciDeviceStatus.REMOVED,
fields.PciDeviceStatus.DELETED):
# NOTE(ndipanov): Removed devs are pruned from
# self.pci_devs on save() so we need to make sure we
# are not looking at removed ones as we may build up
# the tree sooner than they are pruned.
continue
if dev.dev_type == fields.PciDeviceType.SRIOV_PF:
dev.child_devices = []
parents[dev.address] = dev
elif dev.dev_type == fields.PciDeviceType.SRIOV_VF:
dev.parent_device = parents.get(dev.parent_addr)
if dev.parent_device:
parents[dev.parent_addr].child_devices.append(dev)
def _set_hvdevs(self, devices):
exist_addrs = set([dev.address for dev in self.pci_devs])
new_addrs = set([dev['address'] for dev in devices])
for existed in self.pci_devs:
if existed.address in exist_addrs - new_addrs:
try:
existed.remove()
except exception.PciDeviceInvalidStatus as e:
LOG.warning(("Trying to remove device with %(status)s "
"ownership %(instance_uuid)s because of "
"%(pci_exception)s"),
{'status': existed.status,
'container_uuid': existed.container_uuid,
'pci_exception': e.format_message()})
# Note(yjiang5): remove the device by force so that
# db entry is cleaned in next sync.
existed.status = fields.PciDeviceStatus.REMOVED
else:
# Note(yjiang5): no need to update stats if an assigned
# device is hot removed.
self.stats.remove_device(existed)
else:
new_value = next((dev for dev in devices if
dev['address'] == existed.address))
new_value['compute_node_id'] = self.node_id
if existed.status in (fields.PciDeviceStatus.CLAIMED,
fields.PciDeviceStatus.ALLOCATED):
# Pci properties may change while assigned because of
# hotplug or config changes. Although normally this should
# not happen.
# As the devices have been assigned to a container,
# we defer the change till the container is destroyed.
# We will not sync the new properties with database
# before that.
# TODO(yjiang5): Not sure if this is a right policy, but
# at least it avoids some confusion and, if needed,
# we can add more action like killing the container
# by force in future.
self.stale[new_value['address']] = new_value
else:
existed.update_device(new_value)
for dev in [dev for dev in devices if
dev['address'] in new_addrs - exist_addrs]:
dev['compute_node_uuid'] = self.node_id
dev_obj = objects.PciDevice.create(self._context, dev)
self.pci_devs.append(dev_obj)
self.stats.add_device(dev_obj)
self._build_device_tree(self.pci_devs)
def _claim_container(self, context, pci_requests):
devs = self.stats.consume_requests(pci_requests.requests)
if not devs:
return None
container_uuid = pci_requests.container_uuid
for dev in devs:
dev.claim(container_uuid)
return devs
def _allocate_container(self, container, devs):
for dev in devs:
dev.allocate(container)
def allocate_container(self, container):
devs = self.claims.pop(container['uuid'], [])
self._allocate_container(container, devs)
if devs:
self.allocations[container['uuid']] += devs
def claim_container(self, context, pci_requests, container_numa_topology):
devs = []
if self.pci_devs and pci_requests.requests:
container_uuid = pci_requests.container_uuid
devs = self._claim_container(context, pci_requests,
container_numa_topology)
if devs:
self.claims[container_uuid] = devs
return devs
def free_device(self, dev, container):
"""Free device from pci resource tracker
:param dev: cloned pci device object that needs to be free
:param container: the container that this pci device
is allocated to
"""
for pci_dev in self.pci_devs:
# Find the matching pci device in the pci resource tracker.
# Once found, free it.
if (dev.id == pci_dev.id and
dev.container_uuid == container['uuid']):
self._remove_device_from_pci_mapping(
container['uuid'], pci_dev, self.allocations)
self._remove_device_from_pci_mapping(
container['uuid'], pci_dev, self.claims)
self._free_device(pci_dev)
break
def _remove_device_from_pci_mapping(self, container_uuid,
pci_device, pci_mapping):
"""Remove a PCI device from allocations or claims.
If there are no more PCI devices, pop the uuid.
"""
pci_devices = pci_mapping.get(container_uuid, [])
if pci_device in pci_devices:
pci_devices.remove(pci_device)
if len(pci_devices) == 0:
pci_mapping.pop(container_uuid, None)
def _free_device(self, dev, container=None):
freed_devs = dev.free(container)
stale = self.stale.pop(dev.address, None)
if stale:
dev.update_device(stale)
for dev in freed_devs:
self.stats.add_device(dev)
def _free_container(self, container):
for dev in self.pci_devs:
if dev.status in (fields.PciDeviceStatus.CLAIMED,
fields.PciDeviceStatus.ALLOCATED):
if dev.container_uuid == container['uuid']:
self._free_device(dev)
def free_container(self, context, container):
if self.allocations.pop(container['uuid'], None):
self._free_container(container)
elif self.claims.pop(container['uuid'], None):
self._free_container(container)
def update_pci_for_container(self, context, container, sign):
"""Update PCI usage information if devices are de/allocated."""
if not self.pci_devs:
return
if sign == -1:
self.free_container(context, container)
if sign == 1:
self.allocate_container(container)
def clean_usage(self, containers, orphans):
"""Remove all usages for containers not passed in the parameter.
The caller should hold the COMPUTE_RESOURCE_SEMAPHORE lock
"""
existed = set(cnt['uuid'] for cnt in containers)
existed |= set(cnt['uuid'] for cnt in orphans)
# need to copy keys, because the dict is modified in the loop body
for uuid in list(self.claims):
if uuid not in existed:
devs = self.claims.pop(uuid, [])
for dev in devs:
self._free_device(dev)
# need to copy keys, because the dict is modified in the loop body
for uuid in list(self.allocations):
if uuid not in existed:
devs = self.allocations.pop(uuid, [])
for dev in devs:
self._free_device(dev)
def get_container_pci_devs(cnt, request_id=None):
"""Get the devices allocated to one or all requests for a container.
- For generic PCI request, the request id is None.
- For sr-iov networking, the request id is a valid uuid
- There are a couple of cases where all the PCI devices allocated to a
container need to be returned.
"""
pci_devices = cnt.pci_devices
if pci_devices is None:
return []
return [device for device in pci_devices
if device.request_id == request_id or request_id == 'all']

13
zun/pci/utils.py
View File

@ -181,3 +181,16 @@ def get_vf_num_by_pci_address(pci_addr):
if vf_num is None:
raise exception.PciDeviceNotFoundById(id=pci_addr)
return vf_num
def get_net_name_by_vf_pci_address(vfaddress):
"""Given the VF PCI address, returns the net device name and ifname."""
try:
mac = get_mac_by_pci_address(vfaddress).split(':')
ifname = get_ifname_by_pci_address(vfaddress)
return ("net_%(ifname)s_%(mac)s" %
{'ifname': ifname, 'mac': '_'.join(mac)}), ifname
except Exception:
LOG.warning("No net device was found for VF %(vfaddress)s",
{'vfaddress': vfaddress})
return

13
zun/tests/base.py
View File

@ -16,6 +16,7 @@
import copy
import os
import fixtures
import mock
from oslo_config import cfg
from oslo_log import log
@ -128,3 +129,15 @@ class TestCase(base.BaseTestCase):
return os.path.join(root, project_file)
else:
return root
def stub_out(self, old, new):
"""Replace a function for the duration of the test.
Use the monkey patch fixture to replace a function for the
duration of a test. Useful when you want to provide fake
methods instead of mocks during testing.
This should be used instead of self.stubs.Set (which is based
on mox) going forward.
"""
self.useFixture(fixtures.MonkeyPatch(old, new))

81
zun/tests/unit/container/os_capability/linux/os_capability_linux/test_os_capability_linux.py
View File

@ -18,6 +18,8 @@ import six
from mock import mock_open
from oslo_concurrency import processutils
from oslo_serialization import jsonutils
from zun.common import exception
from zun.container.os_capability.linux import os_capability_linux
from zun.tests import base
@ -73,3 +75,82 @@ class TestOSCapability(base.BaseTestCase):
output = os_capability_linux.LinuxHost().get_host_mem()
used = (3882464 - 3556372)
self.assertEqual((3882464, 3514608, 3556372, used), output)
@mock.patch('zun.pci.utils.get_ifname_by_pci_address')
@mock.patch('zun.pci.utils.get_net_name_by_vf_pci_address')
@mock.patch('oslo_concurrency.processutils.execute')
def test_get_pci_resource(self, mock_output, mock_netname,
mock_ifname):
mock_netname.return_value = 'net_enp2s0f3_ec_38_8f_79_11_2b'
mock_ifname.return_value = 'enp2s0f3'
value1 = '''0000:02:10.7 "Ethernet controller...." ""'''
value2 = '02:10.7 0200: 8086:1520 (rev 01)'
value3 = '''Slot: 02:10.7
Class: Ethernet controller
Vendor: Intel Corporation
Device: I350 Ethernet Controller Virtual Function
Rev: 01
NUMANode: 0'''
value4 = 'class physfn'
value5 = '''DRIVER=igbvf
PCI_CLASS=20000
PCI_ID=8086:1520
PCI_SUBSYS_ID=FFFF:0000
PCI_SLOT_NAME=0000:02:10.7
MODALIAS=pci:v00008086d00001520sv0000FFFFsd00000000bc02sc00i00'''
value6 = '''Features for enp2s0f3:
rx-checksumming: on
tx-checksumming: on
scatter-gather: on
tcp-segmentation-offload: on
generic-receive-offload: on
large-receive-offload: off [fixed]
rx-vlan-offload: on
tx-vlan-offload: on
ntuple-filters: off [fixed]
receive-hashing: on
highdma: on [fixed]
rx-vlan-filter: on [fixed]
vlan-challenged: off [fixed]
tx-lockless: off [fixed]
netns-local: off [fixed]
tx-gso-robust: off [fixed]
tx-fcoe-segmentation: off [fixed]
tx-gre-segmentation: off [fixed]
tx-ipip-segmentation: off [fixed]
tx-sit-segmentation: off [fixed]
tx-udp_tnl-segmentation: off [fixed]
tx-mpls-segmentation: off [fixed]
rx-fcs: off [fixed]
tx-vlan-stag-hw-insert: off [fixed]
rx-vlan-stag-hw-parse: off [fixed]
rx-vlan-stag-filter: off [fixed]'''
values = [(value1, 0),
(value2, 0),
(value3, 0),
(value4, 0),
(value5, 0),
(value6, 0)]
mock_output.side_effect = values
expected = {"dev_id": "pci_0000_02_10_7",
"address": "0000:02:10.7",
"product_id": "8086",
"vendor_id": "1520",
"numa_node": 0,
"label": "label_1520_8086",
"dev_type": "VF",
"parent_addr": "0000:02:10.7"}
output = os_capability_linux.LinuxHost().get_pci_resources()
pci_infos = jsonutils.loads(output)
for pci_info in pci_infos:
self.assertEqual(expected['dev_id'], str(pci_info['dev_id']))
self.assertEqual(expected['address'], str(pci_info['address']))
self.assertEqual(expected['product_id'],
str(pci_info['product_id']))
self.assertEqual(expected['vendor_id'], str(pci_info['vendor_id']))
self.assertEqual(expected['numa_node'], pci_info['numa_node'])
self.assertEqual(expected['label'], str(pci_info['label']))
self.assertEqual(expected['dev_type'], str(pci_info['dev_type']))
self.assertEqual(expected['parent_addr'],
str(pci_info['parent_addr']))

4
zun/tests/unit/objects/test_objects.py
View File

@ -354,8 +354,8 @@ object_data = {
'ResourceProvider': '1.0-92b427359d5a4cf9ec6c72cbe630ee24',
'ZunService': '1.1-b1549134bfd5271daec417ca8cabc77e',
'Capsule': '1.3-f4c6b8fede0fa9488fc4f77b97601654',
'PciDevice': '1.0-19fdd11935cda5e92947913f081d9edd',
'ComputeNode': '1.8-5f5f893df0b514c88a3abaec1dfbb89e',
'PciDevice': '1.1-6e3f0851ad1cf12583e6af4df1883979',
'ComputeNode': '1.9-e8536102d3b28cb3378e9e26f508cd72',
'PciDevicePool': '1.0-3f5ddc3ff7bfa14da7f6c7e9904cc000',
'PciDevicePoolList': '1.0-15ecf022a68ddbb8c2a6739cfc9f8f5e'
}

299
zun/tests/unit/pci/test_manager.py Normal file
View File

@ -0,0 +1,299 @@
# Copyright (c) 2017 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import copy
import mock
import zun
from zun.common import context
from zun.objects import fields
from zun.pci import manager
from zun.tests.unit.db import base
from zun.tests.unit.pci import fakes as pci_fakes
from zun.tests import uuidsentinel
fake_pci = {
'compute_node_uuid': 1,
'address': '0000:00:00.1',
'product_id': 'p',
'vendor_id': 'v',
'request_id': None,
'status': fields.PciDeviceStatus.AVAILABLE,
'dev_type': fields.PciDeviceType.STANDARD,
'parent_addr': None,
'numa_node': 0}
fake_pci_1 = dict(fake_pci, address='0000:00:00.2',
product_id='p1', vendor_id='v1')
fake_pci_2 = dict(fake_pci, address='0000:00:00.3')
fake_pci_3 = dict(fake_pci, address='0000:00:01.1',
dev_type=fields.PciDeviceType.SRIOV_PF,
vendor_id='v2', product_id='p2', numa_node=None)
fake_pci_4 = dict(fake_pci, address='0000:00:02.1',
dev_type=fields.PciDeviceType.SRIOV_VF,
parent_addr='0000:00:01.1',
vendor_id='v2', product_id='p2', numa_node=None)
fake_pci_5 = dict(fake_pci, address='0000:00:02.2',
dev_type=fields.PciDeviceType.SRIOV_VF,
parent_addr='0000:00:01.1',
vendor_id='v2', product_id='p2', numa_node=None)
fake_db_dev = {
'created_at': None,
'updated_at': None,
'deleted_at': None,
'deleted': None,
'id': 1,
'uuid': uuidsentinel.pci_device1,
'compute_node_uuid': 1,
'address': '0000:00:00.1',
'vendor_id': 'v',
'product_id': 'p',
'numa_node': 1,
'dev_type': fields.PciDeviceType.STANDARD,
'status': fields.PciDeviceStatus.AVAILABLE,
'dev_id': 'i',
'label': 'l',
'container_uuid': None,
'extra_info': '{}',
'request_id': None,
'parent_addr': None,
}
fake_db_dev_1 = dict(fake_db_dev, vendor_id='v1',
uuid=uuidsentinel.pci_device1,
product_id='p1', id=2,
address='0000:00:00.2',
numa_node=0)
fake_db_dev_2 = dict(fake_db_dev, id=3, address='0000:00:00.3',
uuid=uuidsentinel.pci_device2,
numa_node=None, parent_addr='0000:00:00.1')
fake_db_devs = [fake_db_dev, fake_db_dev_1, fake_db_dev_2]
fake_db_dev_3 = dict(fake_db_dev, id=4, address='0000:00:01.1',
uuid=uuidsentinel.pci_device3,
vendor_id='v2', product_id='p2',
numa_node=None, dev_type=fields.PciDeviceType.SRIOV_PF)
fake_db_dev_4 = dict(fake_db_dev, id=5, address='0000:00:02.1',
uuid=uuidsentinel.pci_device4,
numa_node=None, dev_type=fields.PciDeviceType.SRIOV_VF,
vendor_id='v2', product_id='p2',
parent_addr='0000:00:01.1')
fake_db_dev_5 = dict(fake_db_dev, id=6, address='0000:00:02.2',
uuid=uuidsentinel.pci_device5,
numa_node=None, dev_type=fields.PciDeviceType.SRIOV_VF,
vendor_id='v2', product_id='p2',
parent_addr='0000:00:01.1')
fake_db_devs_tree = [fake_db_dev_3, fake_db_dev_4, fake_db_dev_5]
class PciDevTrackerTestCase(base.DbTestCase):
def _fake_get_pci_devices(self, node_id):
return self.fake_devs
def _fake_pci_device_update(self, node_id, address, value):
self.update_called += 1
self.called_values = value
fake_return = copy.deepcopy(fake_db_dev)
return fake_return
def _fake_pci_device_destroy(self, node_id, address):
self.destroy_called += 1
def _create_tracker(self, fake_devs):
self.fake_devs = fake_devs
self.tracker = manager.PciDevTracker(self.fake_context, 1)
def setUp(self):
super(PciDevTrackerTestCase, self).setUp()
self.fake_context = context.get_admin_context()
self.fake_devs = fake_db_devs[:]
self.stub_out('zun.db.api.get_all_pci_device_by_node',
self._fake_get_pci_devices)
# The fake_pci_whitelist must be called before creating the fake
# devices
patcher = pci_fakes.fake_pci_whitelist()
self.addCleanup(patcher.stop)
self._create_tracker(fake_db_devs[:])
def test_pcidev_tracker_create(self):
self.assertEqual(len(self.tracker.pci_devs), 3)
free_devs = self.tracker.pci_stats.get_free_devs()
self.assertEqual(len(free_devs), 3)
self.assertEqual(list(self.tracker.stale), [])
self.assertEqual(len(self.tracker.stats.pools), 3)
self.assertEqual(self.tracker.node_id, 1)
for dev in self.tracker.pci_devs:
self.assertIsNone(dev.parent_device)
self.assertEqual(dev.child_devices, [])
def test_pcidev_tracker_create_device_tree(self):
self._create_tracker(fake_db_devs_tree)
self.assertEqual(len(self.tracker.pci_devs), 3)
free_devs = self.tracker.pci_stats.get_free_devs()
self.assertEqual(len(free_devs), 3)
self.assertEqual(list(self.tracker.stale), [])
self.assertEqual(len(self.tracker.stats.pools), 2)
self.assertEqual(self.tracker.node_id, 1)
pf = [dev for dev in self.tracker.pci_devs
if dev.dev_type == fields.PciDeviceType.SRIOV_PF].pop()
vfs = [dev for dev in self.tracker.pci_devs
if dev.dev_type == fields.PciDeviceType.SRIOV_VF]
self.assertEqual(2, len(vfs))
# Assert we build the device tree correctly
self.assertEqual(vfs, pf.child_devices)
for vf in vfs:
self.assertEqual(vf.parent_device, pf)
def test_pcidev_tracker_create_device_tree_pf_only(self):
self._create_tracker([fake_db_dev_3])
self.assertEqual(len(self.tracker.pci_devs), 1)
free_devs = self.tracker.pci_stats.get_free_devs()
self.assertEqual(len(free_devs), 1)
self.assertEqual(list(self.tracker.stale), [])
self.assertEqual(len(self.tracker.stats.pools), 1)
self.assertEqual(self.tracker.node_id, 1)
pf = self.tracker.pci_devs[0]
self.assertIsNone(pf.parent_device)
self.assertEqual([], pf.child_devices)
def test_pcidev_tracker_create_device_tree_vf_only(self):
self._create_tracker([fake_db_dev_4])
self.assertEqual(len(self.tracker.pci_devs), 1)
free_devs = self.tracker.pci_stats.get_free_devs()
self.assertEqual(len(free_devs), 1)
self.assertEqual(list(self.tracker.stale), [])
self.assertEqual(len(self.tracker.stats.pools), 1)
self.assertEqual(self.tracker.node_id, 1)
vf = self.tracker.pci_devs[0]
self.assertIsNone(vf.parent_device)
self.assertEqual([], vf.child_devices)
@mock.patch.object(zun.objects.PciDevice, 'list_by_compute_node')
def test_pcidev_tracker_create_no_nodeid(self, mock_get_cn):
self.tracker = manager.PciDevTracker(self.fake_context)
self.assertEqual(len(self.tracker.pci_devs), 0)
self.assertFalse(mock_get_cn.called)
@mock.patch.object(zun.objects.PciDevice, 'list_by_compute_node')
def test_pcidev_tracker_create_with_nodeid(self, mock_get_cn):
self.tracker = manager.PciDevTracker(self.fake_context, node_id=1)
mock_get_cn.assert_called_once_with(self.fake_context, 1)
def test_set_hvdev_new_dev(self):
fake_pci_3 = dict(fake_pci, address='0000:00:00.4', vendor_id='v2')
fake_pci_devs = [copy.deepcopy(fake_pci), copy.deepcopy(fake_pci_1),
copy.deepcopy(fake_pci_2), copy.deepcopy(fake_pci_3)]
self.tracker._set_hvdevs(fake_pci_devs)
self.assertEqual(len(self.tracker.pci_devs), 4)
self.assertEqual(set([dev.address for
dev in self.tracker.pci_devs]),
set(['0000:00:00.1', '0000:00:00.2',
'0000:00:00.3', '0000:00:00.4']))
self.assertEqual(set([dev.vendor_id for
dev in self.tracker.pci_devs]),
set(['v', 'v1', 'v2']))
def test_set_hvdev_new_dev_tree_maintained(self):
# Make sure the device tree is properly maintained when there are new
# devices reported by the driver
self._create_tracker(fake_db_devs_tree)
fake_new_device = dict(fake_pci_5, id=12, address='0000:00:02.3')
fake_pci_devs = [copy.deepcopy(fake_pci_3),
copy.deepcopy(fake_pci_4),
copy.deepcopy(fake_pci_5),
copy.deepcopy(fake_new_device)]
self.tracker._set_hvdevs(fake_pci_devs)
self.assertEqual(len(self.tracker.pci_devs), 4)
pf = [dev for dev in self.tracker.pci_devs
if dev.dev_type == fields.PciDeviceType.SRIOV_PF].pop()
vfs = [dev for dev in self.tracker.pci_devs
if dev.dev_type == fields.PciDeviceType.SRIOV_VF]
self.assertEqual(3, len(vfs))
# Assert we build the device tree correctly
self.assertEqual(vfs, pf.child_devices)
for vf in vfs:
self.assertEqual(vf.parent_device, pf)
def test_set_hvdev_changed(self):
fake_pci_v2 = dict(fake_pci, address='0000:00:00.2', vendor_id='v1')
fake_pci_devs = [copy.deepcopy(fake_pci), copy.deepcopy(fake_pci_2),
copy.deepcopy(fake_pci_v2)]
self.tracker._set_hvdevs(fake_pci_devs)
self.assertEqual(set([dev.vendor_id for
dev in self.tracker.pci_devs]),
set(['v', 'v1']))
def test_set_hvdev_remove(self):
self.tracker._set_hvdevs([fake_pci])
self.assertEqual(
len([dev for dev in self.tracker.pci_devs
if dev.status == fields.PciDeviceStatus.REMOVED]),
2)
def test_set_hvdev_remove_tree_maintained(self):
# Make sure the device tree is properly maintained when there are
# devices removed from the system (not reported by the driver but known
# from previous scans)
self._create_tracker(fake_db_devs_tree)
fake_pci_devs = [copy.deepcopy(fake_pci_3), copy.deepcopy(fake_pci_4)]
self.tracker._set_hvdevs(fake_pci_devs)
self.assertEqual(
2,
len([dev for dev in self.tracker.pci_devs
if dev.status != fields.PciDeviceStatus.REMOVED]))
pf = [dev for dev in self.tracker.pci_devs
if dev.dev_type == fields.PciDeviceType.SRIOV_PF].pop()
vfs = [dev for dev in self.tracker.pci_devs
if (dev.dev_type == fields.PciDeviceType.SRIOV_VF and
dev.status != fields.PciDeviceStatus.REMOVED)]
self.assertEqual(1, len(vfs))
self.assertEqual(vfs, pf.child_devices)
self.assertEqual(vfs[0].parent_device, pf)
def test_save(self):
self.stub_out('zun.db.api.update_pci_device',
self._fake_pci_device_update)
fake_pci_v3 = dict(fake_pci, address='0000:00:00.2', vendor_id='v3')
fake_pci_devs = [copy.deepcopy(fake_pci), copy.deepcopy(fake_pci_2),
copy.deepcopy(fake_pci_v3)]
self.tracker._set_hvdevs(fake_pci_devs)
self.update_called = 0
self.tracker.save()
self.assertEqual(self.update_called, 3)
def test_save_removed(self):
self.stub_out('zun.db.api.update_pci_device',
self._fake_pci_device_update)
self.stub_out('zun.db.api.destroy_pci_device',
self._fake_pci_device_destroy)
self.assertEqual(len(self.tracker.pci_devs), 3)
dev = self.tracker.pci_devs[0]
self.destroy_called = 0
self.update_called = 0
dev.remove()
self.tracker.save()
self.assertEqual(len(self.tracker.pci_devs), 2)
self.assertEqual(self.destroy_called, 1)

43
zun/tests/unit/pci/test_utils.py
View File

@ -263,3 +263,46 @@ class GetVfNumByPciAddressTestCase(base.TestCase):
utils.get_vf_num_by_pci_address,
self.pci_address
)
class GetNetNameByVfPciAddressTestCase(base.TestCase):
def setUp(self):
super(GetNetNameByVfPciAddressTestCase, self).setUp()
self._get_mac = mock.patch.object(utils, 'get_mac_by_pci_address')
self.mock_get_mac = self._get_mac.start()
self._get_ifname = mock.patch.object(
utils, 'get_ifname_by_pci_address')
self.mock_get_ifname = self._get_ifname.start()
self.addCleanup(self._get_mac.stop)
self.addCleanup(self._get_ifname.stop)
self.mac = 'ca:fe:ca:fe:ca:fe'
self.if_name = 'enp7s0f0'
self.pci_address = '0000:07:02.1'
def test_correct_behaviour(self):
ref_net_name = ('net_enp7s0f0_ca_fe_ca_fe_ca_fe', 'enp7s0f0')
self.mock_get_mac.return_value = self.mac
self.mock_get_ifname.return_value = self.if_name
net_name = utils.get_net_name_by_vf_pci_address(self.pci_address)
self.assertEqual(ref_net_name, net_name)
self.mock_get_mac.called_once_with(self.pci_address)
self.mock_get_ifname.called_once_with(self.pci_address)
def test_wrong_mac(self):
self.mock_get_mac.side_effect = (
exception.PciDeviceNotFoundById(self.pci_address))
net_name = utils.get_net_name_by_vf_pci_address(self.pci_address)
self.assertIsNone(net_name)
self.mock_get_mac.called_once_with(self.pci_address)
self.mock_get_ifname.assert_not_called()
def test_wrong_ifname(self):
self.mock_get_mac.return_value = self.mac
self.mock_get_ifname.side_effect = (
exception.PciDeviceNotFoundById(self.pci_address))
net_name = utils.get_net_name_by_vf_pci_address(self.pci_address)
self.assertIsNone(net_name)
self.mock_get_mac.called_once_with(self.pci_address)
self.mock_get_ifname.called_once_with(self.pci_address)