diff --git a/zun/compute/compute_node_tracker.py b/zun/compute/compute_node_tracker.py index 923867d9a..fb7ad4949 100644 --- a/zun/compute/compute_node_tracker.py +++ b/zun/compute/compute_node_tracker.py @@ -22,6 +22,7 @@ from zun.common import utils from zun.compute import claims from zun import objects from zun.objects import base as obj_base +from zun.pci import manager as pci_manager from zun.scheduler import client as scheduler_client LOG = logging.getLogger(__name__) @@ -36,6 +37,17 @@ class ComputeNodeTracker(object): self.tracked_containers = {} self.old_resources = collections.defaultdict(objects.ComputeNode) self.scheduler_client = scheduler_client.SchedulerClient() + self.pci_tracker = None + + def _setup_pci_tracker(self, context, compute_node): + if not self.pci_tracker: + n_id = compute_node.uuid + self.pci_tracker = pci_manager.PciDevTracker(context, node_id=n_id) + dev_json = self.container_driver.get_pci_resources() + self.pci_tracker.update_devices_from_compute_resources(dev_json) + + dev_pools_obj = self.pci_tracker.stats.to_device_pools_obj() + compute_node.pci_device_pools = dev_pools_obj def update_available_resources(self, context): # Check if the compute_node is already registered @@ -49,6 +61,7 @@ class ComputeNodeTracker(object): node.create(context) LOG.info('Node created for :%(host)s', {'host': self.host}) self.container_driver.get_available_resources(node) + self._setup_pci_tracker(context, node) self.compute_node = node self._update_available_resource(context) # NOTE(sbiswas7): Consider removing the return statement if not needed @@ -175,7 +188,9 @@ class ComputeNodeTracker(object): return # Persist the stats to the Scheduler self.scheduler_client.update_resource(compute_node) - # Update pci tracker here + + if self.pci_tracker: + self.pci_tracker.save() def _resource_change(self, compute_node): """Check to see if any resources have changed.""" diff --git a/zun/container/driver.py b/zun/container/driver.py index fa935842d..c61fe3774 100644 --- a/zun/container/driver.py +++ b/zun/container/driver.py @@ -183,6 +183,9 @@ class ContainerDriver(object): os_capability_linux.LinuxHost().get_host_numa_topology(numa_topo_obj) return numa_topo_obj + def get_pci_resources(self): + return os_capability_linux.LinuxHost().get_pci_resources() + def get_host_mem(self): return os_capability_linux.LinuxHost().get_host_mem() diff --git a/zun/container/os_capability/host_capability.py b/zun/container/os_capability/host_capability.py index cb7647738..82c8abadc 100644 --- a/zun/container/os_capability/host_capability.py +++ b/zun/container/os_capability/host_capability.py @@ -13,8 +13,14 @@ # License for the specific language governing permissions and limitations # under the License. +from oslo_concurrency import processutils +from oslo_serialization import jsonutils + +from zun.common import exception from zun.common import utils from zun import objects +from zun.objects import fields +from zun.pci import utils as pci_utils class Host(object): @@ -67,3 +73,154 @@ class Host(object): mem_ava = int(mem_free) + int(buffers) + int(cached) mem_used = int(mem_total) - int(mem_ava) return int(mem_total), int(mem_free), int(mem_ava), int(mem_used) + + def get_pci_resources(self): + addresses = [] + try: + output, status = processutils.execute('lspci', '-D', '-nnmm') + lines = output.split('\n') + for line in lines: + if not line: + continue + columns = line.split() + address = columns[0] + addresses.append(address) + except processutils.ProcessExecutionError: + raise exception.CommandError(cmd='lspci') + + pci_info = [] + for addr in addresses: + pci_info.append(self._get_pci_dev_info(addr)) + + return jsonutils.dumps(pci_info) + + def _get_pci_dev_info(self, address): + """Returns a dict of PCI device.""" + + def _get_device_type(address): + """Get a PCI device's device type. + + An assignable PCI device can be a normal PCI device, + a SR-IOV Physical Function (PF), or a SR-IOV Virtual + Function (VF). Only normal PCI devices or SR-IOV VFs + are assignable. + """ + try: + path = '/sys/bus/pci/devices/' + address + '/' + output, status = processutils.execute('ls', path) + if "physfn" in output: + phys_address = None + upath = '/sys/bus/pci/devices/%s/physfn/uevent' % address + try: + ou, st = processutils.execute('cat', upath) + lines = ou.split('\n') + for line in lines: + if 'PCI_SLOT_NAME' in line: + columns = line.split("=") + phys_address = columns[1] + except processutils.ProcessExecutionError: + raise exception.CommandError(cmd='cat') + return {'dev_type': fields.PciDeviceType.SRIOV_VF, + 'parent_addr': phys_address} + if "virtfn" in output: + return {'dev_type': fields.PciDeviceType.SRIOV_PF} + except processutils.ProcessExecutionError: + raise exception.CommandError(cmd='ls') + return {'dev_type': fields.PciDeviceType.STANDARD} + + def _get_device_capabilities(device, address): + """Get PCI VF device's additional capabilities. + + If a PCI device is a virtual function, this function reads the PCI + parent's network capabilities (must be always a NIC device) and + appends this information to the device's dictionary. + """ + if device.get('dev_type') == fields.PciDeviceType.SRIOV_VF: + pcinet_info = self._get_pcinet_info(address) + if pcinet_info: + return {'capabilities': + {'network': pcinet_info.get('capabilities')}} + return {} + + def _get_product_and_vendor(address): + try: + output, status = processutils.execute('lspci', '-n', '-s', + address) + value = output.split()[2] + result = value.split(":") + return result[0], result[1] + except processutils.ProcessExecutionError: + raise exception.CommandError(cmd='lspci') + + def _get_numa_node(address): + numa_node = None + try: + output, status = processutils.execute('lspci', '-vmm', '-s', + address) + lines = output.split('\n') + for line in lines: + if 'NUMANode' in line: + numa_node = int(line.split(":")[1]) + except processutils.ProcessExecutionError: + raise exception.CommandError(cmd='lspci') + return numa_node + + dev_name = 'pci_' + address.replace(":", "_").replace(".", "_") + product_id, vendor_id = _get_product_and_vendor(address) + numa_node = _get_numa_node(address) + device = { + "dev_id": dev_name, + "address": address, + "product_id": product_id, + "vendor_id": vendor_id, + "numa_node": numa_node + } + device['label'] = 'label_%(vendor_id)s_%(product_id)s' % device + device.update(_get_device_type(address)) + device.update(_get_device_capabilities(device, address)) + return device + + def _get_pcinet_info(self, vf_address): + """Returns a dict of NET device.""" + devname = pci_utils.get_net_name_by_vf_pci_address(vf_address) + if not devname: + return + + ifname = pci_utils.get_ifname_by_pci_address(vf_address) + # Features from the that libvirt supported, get them by ethtool -k + # Note: I cannot find the rdma feature returned by ethtool, correct me + # if the string is wrong. + FEATURES_LIST = ['rx-checksumming', 'tx-checksumming', + 'scatter-gather', 'tcp-segmentation-offload', + 'generic-segmentation-offload', + 'generic-receive-offload', 'large-receive-offload', + 'rx-vlan-offload', 'tx-vlan-offload', + 'ntuple-filters', 'receive-hashing', + 'tx-udp_tnl-segmentation', 'rdma'] + FEATURES_MAP = {'rx-checksumming': 'rx', + 'tx-checksumming': 'tx', + 'scatter-gather': 'sg', + 'tcp-segmentation-offload': 'tso', + 'generic-segmentation-offload': 'gso', + 'generic-receive-offload': 'gro', + 'large-receive-offload': 'lro', + 'rx-vlan-offload': 'rxvlan', + 'tx-vlan-offload': 'txvlan', + 'ntuple-filters': 'ntuple', + 'receive-hashing': 'rxhash', + 'tx-udp_tnl-segmentation': 'txudptnl', + 'rdma': 'rdma'} + + features = [] + try: + output, status = processutils.execute('ethtool', '-k', ifname) + lines = output.split('\n') + for line in lines: + columns = line.split(":") + if columns[0].strip() in FEATURES_LIST: + if "on" in columns[1].strip(): + features.append(FEATURES_MAP.get(columns[0].strip())) + except processutils.ProcessExecutionError: + raise exception.CommandError(cmd='ethtool -k') + return {'name': devname, + 'capabilities': features} diff --git a/zun/db/sqlalchemy/alembic/versions/f046346d1d87_add_timestamp_to_pci_device.py b/zun/db/sqlalchemy/alembic/versions/f046346d1d87_add_timestamp_to_pci_device.py new file mode 100644 index 000000000..61e49794a --- /dev/null +++ b/zun/db/sqlalchemy/alembic/versions/f046346d1d87_add_timestamp_to_pci_device.py @@ -0,0 +1,38 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""add timestamp to pci device + +Revision ID: f046346d1d87 +Revises: ff7b9665d504 +Create Date: 2017-10-09 15:30:34.922130 + +""" + +# revision identifiers, used by Alembic. +revision = 'f046346d1d87' +down_revision = 'ff7b9665d504' +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.add_column('pci_device', + sa.Column('created_at', sa.DateTime(), nullable=True)) + op.add_column('pci_device', + sa.Column('updated_at', sa.DateTime(), nullable=True)) + op.drop_column('pci_device', 'request_id') + op.add_column('pci_device', sa.Column('request_id', sa.String(36), + nullable=True)) diff --git a/zun/objects/compute_node.py b/zun/objects/compute_node.py index 657a64c61..99f9129f0 100644 --- a/zun/objects/compute_node.py +++ b/zun/objects/compute_node.py @@ -30,7 +30,8 @@ class ComputeNode(base.ZunPersistentObject, base.ZunObject): # Version 1.6: Add mem_used to compute node # Version 1.7: Change get_by_hostname to get_by_name # Version 1.8: Add pci_device_pools to compute node - VERSION = '1.8' + # Version 1.9: Change PciDevicePoolList to ObjectField + VERSION = '1.9' fields = { 'uuid': fields.UUIDField(read_only=True, nullable=False), @@ -53,8 +54,8 @@ class ComputeNode(base.ZunPersistentObject, base.ZunObject): 'labels': fields.DictOfStringsField(nullable=True), # NOTE(pmurray): the pci_device_pools field maps to the # pci_stats field in the database - 'pci_device_pools': fields.ListOfObjectsField('PciDevicePool', - nullable=True), + 'pci_device_pools': fields.ObjectField('PciDevicePoolList', + nullable=True), } @staticmethod @@ -168,6 +169,7 @@ class ComputeNode(base.ZunPersistentObject, base.ZunObject): numa_obj = updates.pop('numa_topology', None) if numa_obj is not None: updates['numa_topology'] = numa_obj._to_dict() + self._convert_pci_stats_to_db_format(updates) dbapi.update_compute_node(context, self.uuid, updates) self.obj_reset_changes(recursive=True) diff --git a/zun/objects/pci_device.py b/zun/objects/pci_device.py index c838068f1..a310b1a94 100644 --- a/zun/objects/pci_device.py +++ b/zun/objects/pci_device.py @@ -83,14 +83,15 @@ class PciDevice(base.ZunPersistentObject, base.ZunObject): """ # Version 1.0: Initial version - VERSION = '1.0' + # Version 1.1: Change compute_node_uuid to uuid type + VERSION = '1.1' fields = { 'id': fields.IntegerField(), 'uuid': fields.UUIDField(), # Note(yjiang5): the compute_node_uuid may be None because the pci # device objects are created before the compute node is created in DB - 'compute_node_uuid': fields.IntegerField(nullable=True), + 'compute_node_uuid': fields.UUIDField(nullable=True), 'address': fields.StringField(), 'vendor_id': fields.StringField(), 'product_id': fields.StringField(), @@ -192,7 +193,7 @@ class PciDevice(base.ZunPersistentObject, base.ZunObject): return pci_device @base.remotable - def save(self, context): + def save(self): if self.status == z_fields.PciDeviceStatus.REMOVED: self.status = z_fields.PciDeviceStatus.DELETED dbapi.destroy_pci_device(self.compute_node_uuid, @@ -203,9 +204,9 @@ class PciDevice(base.ZunPersistentObject, base.ZunObject): updates['extra_info'] = jsonutils.dumps(updates['extra_info']) if updates: - db_pci = dbapi.update_pci_device(self.compute_node_uuid, - self.address, updates) - self._from_db_object(context, self, db_pci) + dbapi.update_pci_device(self.compute_node_uuid, + self.address, updates) + # self._from_db_object(context, self, db_pci) @staticmethod def _bulk_update_status(dev_list, status): @@ -373,22 +374,21 @@ class PciDevice(base.ZunPersistentObject, base.ZunObject): @staticmethod def _from_db_object_list(db_objects, cls, context): """Converts a list of database entities to a list of formal objects.""" - return [PciDevice._from_db_object(cls(context), obj) + return [PciDevice._from_db_object(context, cls(context), obj) for obj in db_objects] @base.remotable_classmethod def list_by_compute_node(cls, context, node_id): - db_dev_list = dbapi.get_all_pci_device_by_node(context, node_id) + db_dev_list = dbapi.get_all_pci_device_by_node(node_id) return PciDevice._from_db_object_list(db_dev_list, cls, context) @base.remotable_classmethod def list_by_container_uuid(cls, context, uuid): - db_dev_list = dbapi.get_all_pci_device_by_container_uuid(context, uuid) + db_dev_list = dbapi.get_all_pci_device_by_container_uuid(uuid) return PciDevice._from_db_object_list(db_dev_list, cls, context) @base.remotable_classmethod def list_by_parent_address(cls, context, node_id, parent_addr): - db_dev_list = dbapi.get_all_pci_device_by_parent_addr(context, - node_id, + db_dev_list = dbapi.get_all_pci_device_by_parent_addr(node_id, parent_addr) return PciDevice._from_db_object_list(db_dev_list, cls, context) diff --git a/zun/pci/manager.py b/zun/pci/manager.py new file mode 100644 index 000000000..6ee6185d3 --- /dev/null +++ b/zun/pci/manager.py @@ -0,0 +1,338 @@ +# Copyright (c) 2017 Intel, Inc. +# Copyright (c) 2017 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import collections + +from oslo_config import cfg +from oslo_log import log as logging +from oslo_serialization import jsonutils + +from zun.common import exception +from zun import objects +from zun.objects import fields +from zun.pci import stats +from zun.pci import whitelist + +CONF = cfg.CONF +LOG = logging.getLogger(__name__) + + +class PciDevTracker(object): + """Manage pci devices in a compute node. + + This class fetches pci passthrough information from compute node + and tracks the usage of these devices. + + It's called by compute node resource tracker to allocate and free + devices to/from containers, and to update the available pci passthrough + devices information from compute node periodically. + + `pci_devs` attribute of this class is the in-memory "master copy" of all + devices on each compute node, and all data changes that happen when + claiming/allocating/freeing + devices HAVE TO be made against container contained in `pci_devs` list, + because they are periodically flushed to the DB when the save() + method is called. + + It is unsafe to fetch PciDevice objects elsewhere in the code for update + purposes as those changes will end up being overwritten when the `pci_devs` + are saved. + """ + + def __init__(self, context, node_id=None): + """Create a pci device tracker. + + If a node_id is passed in, it will fetch pci devices information + from database, otherwise, it will create an empty devices list + and the resource tracker will update the node_id information later. + """ + + super(PciDevTracker, self).__init__() + self.stale = {} + self.node_id = node_id + self.dev_filter = whitelist.Whitelist(CONF.pci.passthrough_whitelist) + self.stats = stats.PciDeviceStats(dev_filter=self.dev_filter) + self._context = context + if node_id: + self.pci_devs = objects.PciDevice.list_by_compute_node( + context, node_id) + else: + self.pci_devs = [] + self._build_device_tree(self.pci_devs) + self._initial_instance_usage() + + def _initial_instance_usage(self): + self.allocations = collections.defaultdict(list) + self.claims = collections.defaultdict(list) + for dev in self.pci_devs: + uuid = dev.container_uuid + if dev.status == fields.PciDeviceStatus.CLAIMED: + self.claims[uuid].append(dev) + elif dev.status == fields.PciDeviceStatus.ALLOCATED: + self.allocations[uuid].append(dev) + elif dev.status == fields.PciDeviceStatus.AVAILABLE: + self.stats.add_device(dev) + + def save(self): + for dev in self.pci_devs: + if dev.obj_what_changed(): + dev.save() + if dev.status == fields.PciDeviceStatus.DELETED: + self.pci_devs.remove(dev) + + @property + def pci_stats(self): + return self.stats + + def update_devices_from_compute_resources(self, devices_json): + """Sync the pci device tracker with compute node information. + + To support pci device hot plug, we sync with the compute node + periodically, fetching all devices information from compute node, + update the tracker and sync the DB information. + + Devices should not be hot-plugged when assigned to a container, + but possibly the compute node has no such guarantee. The best + we can do is to give a warning if a device is changed + or removed while assigned. + + :param devices_json: The JSON-ified string of device information + that is returned from the compute node. + """ + + devices = [] + for dev in jsonutils.loads(devices_json): + if self.dev_filter.device_assignable(dev): + devices.append(dev) + self._set_hvdevs(devices) + + @staticmethod + def _build_device_tree(all_devs): + """Build a tree of devices that represents parent-child relationships. + + We need to have the relationships set up so that we can easily make + all the necessary changes to parent/child devices without having to + figure it out at each call site. + + This method just adds references to relevant containers already found + in `pci_devs` to `child_devices` and `parent_device` fields of each + one. + + Currently relationships are considered for SR-IOV PFs/VFs only. + """ + + # Ensures that devices are ordered in ASC so VFs will come + # after their PFs. + all_devs.sort(key=lambda x: x.address) + + parents = {} + for dev in all_devs: + if dev.status in (fields.PciDeviceStatus.REMOVED, + fields.PciDeviceStatus.DELETED): + # NOTE(ndipanov): Removed devs are pruned from + # self.pci_devs on save() so we need to make sure we + # are not looking at removed ones as we may build up + # the tree sooner than they are pruned. + continue + if dev.dev_type == fields.PciDeviceType.SRIOV_PF: + dev.child_devices = [] + parents[dev.address] = dev + elif dev.dev_type == fields.PciDeviceType.SRIOV_VF: + dev.parent_device = parents.get(dev.parent_addr) + if dev.parent_device: + parents[dev.parent_addr].child_devices.append(dev) + + def _set_hvdevs(self, devices): + exist_addrs = set([dev.address for dev in self.pci_devs]) + new_addrs = set([dev['address'] for dev in devices]) + + for existed in self.pci_devs: + if existed.address in exist_addrs - new_addrs: + try: + existed.remove() + except exception.PciDeviceInvalidStatus as e: + LOG.warning(("Trying to remove device with %(status)s " + "ownership %(instance_uuid)s because of " + "%(pci_exception)s"), + {'status': existed.status, + 'container_uuid': existed.container_uuid, + 'pci_exception': e.format_message()}) + # Note(yjiang5): remove the device by force so that + # db entry is cleaned in next sync. + existed.status = fields.PciDeviceStatus.REMOVED + else: + # Note(yjiang5): no need to update stats if an assigned + # device is hot removed. + self.stats.remove_device(existed) + else: + new_value = next((dev for dev in devices if + dev['address'] == existed.address)) + new_value['compute_node_id'] = self.node_id + if existed.status in (fields.PciDeviceStatus.CLAIMED, + fields.PciDeviceStatus.ALLOCATED): + # Pci properties may change while assigned because of + # hotplug or config changes. Although normally this should + # not happen. + + # As the devices have been assigned to a container, + # we defer the change till the container is destroyed. + # We will not sync the new properties with database + # before that. + + # TODO(yjiang5): Not sure if this is a right policy, but + # at least it avoids some confusion and, if needed, + # we can add more action like killing the container + # by force in future. + self.stale[new_value['address']] = new_value + else: + existed.update_device(new_value) + + for dev in [dev for dev in devices if + dev['address'] in new_addrs - exist_addrs]: + dev['compute_node_uuid'] = self.node_id + dev_obj = objects.PciDevice.create(self._context, dev) + self.pci_devs.append(dev_obj) + self.stats.add_device(dev_obj) + + self._build_device_tree(self.pci_devs) + + def _claim_container(self, context, pci_requests): + devs = self.stats.consume_requests(pci_requests.requests) + if not devs: + return None + + container_uuid = pci_requests.container_uuid + for dev in devs: + dev.claim(container_uuid) + return devs + + def _allocate_container(self, container, devs): + for dev in devs: + dev.allocate(container) + + def allocate_container(self, container): + devs = self.claims.pop(container['uuid'], []) + self._allocate_container(container, devs) + if devs: + self.allocations[container['uuid']] += devs + + def claim_container(self, context, pci_requests, container_numa_topology): + devs = [] + if self.pci_devs and pci_requests.requests: + container_uuid = pci_requests.container_uuid + devs = self._claim_container(context, pci_requests, + container_numa_topology) + if devs: + self.claims[container_uuid] = devs + return devs + + def free_device(self, dev, container): + """Free device from pci resource tracker + + :param dev: cloned pci device object that needs to be free + :param container: the container that this pci device + is allocated to + """ + for pci_dev in self.pci_devs: + # Find the matching pci device in the pci resource tracker. + # Once found, free it. + if (dev.id == pci_dev.id and + dev.container_uuid == container['uuid']): + self._remove_device_from_pci_mapping( + container['uuid'], pci_dev, self.allocations) + self._remove_device_from_pci_mapping( + container['uuid'], pci_dev, self.claims) + self._free_device(pci_dev) + break + + def _remove_device_from_pci_mapping(self, container_uuid, + pci_device, pci_mapping): + """Remove a PCI device from allocations or claims. + + If there are no more PCI devices, pop the uuid. + """ + pci_devices = pci_mapping.get(container_uuid, []) + if pci_device in pci_devices: + pci_devices.remove(pci_device) + if len(pci_devices) == 0: + pci_mapping.pop(container_uuid, None) + + def _free_device(self, dev, container=None): + freed_devs = dev.free(container) + stale = self.stale.pop(dev.address, None) + if stale: + dev.update_device(stale) + for dev in freed_devs: + self.stats.add_device(dev) + + def _free_container(self, container): + for dev in self.pci_devs: + if dev.status in (fields.PciDeviceStatus.CLAIMED, + fields.PciDeviceStatus.ALLOCATED): + if dev.container_uuid == container['uuid']: + self._free_device(dev) + + def free_container(self, context, container): + if self.allocations.pop(container['uuid'], None): + self._free_container(container) + elif self.claims.pop(container['uuid'], None): + self._free_container(container) + + def update_pci_for_container(self, context, container, sign): + """Update PCI usage information if devices are de/allocated.""" + if not self.pci_devs: + return + + if sign == -1: + self.free_container(context, container) + if sign == 1: + self.allocate_container(container) + + def clean_usage(self, containers, orphans): + """Remove all usages for containers not passed in the parameter. + + The caller should hold the COMPUTE_RESOURCE_SEMAPHORE lock + """ + existed = set(cnt['uuid'] for cnt in containers) + existed |= set(cnt['uuid'] for cnt in orphans) + + # need to copy keys, because the dict is modified in the loop body + for uuid in list(self.claims): + if uuid not in existed: + devs = self.claims.pop(uuid, []) + for dev in devs: + self._free_device(dev) + # need to copy keys, because the dict is modified in the loop body + for uuid in list(self.allocations): + if uuid not in existed: + devs = self.allocations.pop(uuid, []) + for dev in devs: + self._free_device(dev) + + +def get_container_pci_devs(cnt, request_id=None): + """Get the devices allocated to one or all requests for a container. + + - For generic PCI request, the request id is None. + - For sr-iov networking, the request id is a valid uuid + - There are a couple of cases where all the PCI devices allocated to a + container need to be returned. + """ + pci_devices = cnt.pci_devices + if pci_devices is None: + return [] + return [device for device in pci_devices + if device.request_id == request_id or request_id == 'all'] diff --git a/zun/pci/utils.py b/zun/pci/utils.py index bcc9fcaaf..e29c93233 100644 --- a/zun/pci/utils.py +++ b/zun/pci/utils.py @@ -181,3 +181,16 @@ def get_vf_num_by_pci_address(pci_addr): if vf_num is None: raise exception.PciDeviceNotFoundById(id=pci_addr) return vf_num + + +def get_net_name_by_vf_pci_address(vfaddress): + """Given the VF PCI address, returns the net device name and ifname.""" + try: + mac = get_mac_by_pci_address(vfaddress).split(':') + ifname = get_ifname_by_pci_address(vfaddress) + return ("net_%(ifname)s_%(mac)s" % + {'ifname': ifname, 'mac': '_'.join(mac)}), ifname + except Exception: + LOG.warning("No net device was found for VF %(vfaddress)s", + {'vfaddress': vfaddress}) + return diff --git a/zun/tests/base.py b/zun/tests/base.py index 16a7bdf44..3ab6ac281 100644 --- a/zun/tests/base.py +++ b/zun/tests/base.py @@ -16,6 +16,7 @@ import copy import os +import fixtures import mock from oslo_config import cfg from oslo_log import log @@ -128,3 +129,15 @@ class TestCase(base.BaseTestCase): return os.path.join(root, project_file) else: return root + + def stub_out(self, old, new): + """Replace a function for the duration of the test. + + Use the monkey patch fixture to replace a function for the + duration of a test. Useful when you want to provide fake + methods instead of mocks during testing. + + This should be used instead of self.stubs.Set (which is based + on mox) going forward. + """ + self.useFixture(fixtures.MonkeyPatch(old, new)) diff --git a/zun/tests/unit/container/os_capability/linux/os_capability_linux/test_os_capability_linux.py b/zun/tests/unit/container/os_capability/linux/os_capability_linux/test_os_capability_linux.py index 8912a2d80..b6927b701 100644 --- a/zun/tests/unit/container/os_capability/linux/os_capability_linux/test_os_capability_linux.py +++ b/zun/tests/unit/container/os_capability/linux/os_capability_linux/test_os_capability_linux.py @@ -18,6 +18,8 @@ import six from mock import mock_open from oslo_concurrency import processutils +from oslo_serialization import jsonutils + from zun.common import exception from zun.container.os_capability.linux import os_capability_linux from zun.tests import base @@ -73,3 +75,82 @@ class TestOSCapability(base.BaseTestCase): output = os_capability_linux.LinuxHost().get_host_mem() used = (3882464 - 3556372) self.assertEqual((3882464, 3514608, 3556372, used), output) + + @mock.patch('zun.pci.utils.get_ifname_by_pci_address') + @mock.patch('zun.pci.utils.get_net_name_by_vf_pci_address') + @mock.patch('oslo_concurrency.processutils.execute') + def test_get_pci_resource(self, mock_output, mock_netname, + mock_ifname): + mock_netname.return_value = 'net_enp2s0f3_ec_38_8f_79_11_2b' + mock_ifname.return_value = 'enp2s0f3' + value1 = '''0000:02:10.7 "Ethernet controller...." ""''' + value2 = '02:10.7 0200: 8086:1520 (rev 01)' + value3 = '''Slot: 02:10.7 + Class: Ethernet controller + Vendor: Intel Corporation + Device: I350 Ethernet Controller Virtual Function + Rev: 01 + NUMANode: 0''' + value4 = 'class physfn' + value5 = '''DRIVER=igbvf + PCI_CLASS=20000 + PCI_ID=8086:1520 + PCI_SUBSYS_ID=FFFF:0000 + PCI_SLOT_NAME=0000:02:10.7 + MODALIAS=pci:v00008086d00001520sv0000FFFFsd00000000bc02sc00i00''' + value6 = '''Features for enp2s0f3: +rx-checksumming: on +tx-checksumming: on +scatter-gather: on +tcp-segmentation-offload: on +generic-receive-offload: on +large-receive-offload: off [fixed] +rx-vlan-offload: on +tx-vlan-offload: on +ntuple-filters: off [fixed] +receive-hashing: on +highdma: on [fixed] +rx-vlan-filter: on [fixed] +vlan-challenged: off [fixed] +tx-lockless: off [fixed] +netns-local: off [fixed] +tx-gso-robust: off [fixed] +tx-fcoe-segmentation: off [fixed] +tx-gre-segmentation: off [fixed] +tx-ipip-segmentation: off [fixed] +tx-sit-segmentation: off [fixed] +tx-udp_tnl-segmentation: off [fixed] +tx-mpls-segmentation: off [fixed] +rx-fcs: off [fixed] +tx-vlan-stag-hw-insert: off [fixed] +rx-vlan-stag-hw-parse: off [fixed] +rx-vlan-stag-filter: off [fixed]''' + values = [(value1, 0), + (value2, 0), + (value3, 0), + (value4, 0), + (value5, 0), + (value6, 0)] + mock_output.side_effect = values + expected = {"dev_id": "pci_0000_02_10_7", + "address": "0000:02:10.7", + "product_id": "8086", + "vendor_id": "1520", + "numa_node": 0, + "label": "label_1520_8086", + "dev_type": "VF", + "parent_addr": "0000:02:10.7"} + output = os_capability_linux.LinuxHost().get_pci_resources() + + pci_infos = jsonutils.loads(output) + for pci_info in pci_infos: + self.assertEqual(expected['dev_id'], str(pci_info['dev_id'])) + self.assertEqual(expected['address'], str(pci_info['address'])) + self.assertEqual(expected['product_id'], + str(pci_info['product_id'])) + self.assertEqual(expected['vendor_id'], str(pci_info['vendor_id'])) + self.assertEqual(expected['numa_node'], pci_info['numa_node']) + self.assertEqual(expected['label'], str(pci_info['label'])) + self.assertEqual(expected['dev_type'], str(pci_info['dev_type'])) + self.assertEqual(expected['parent_addr'], + str(pci_info['parent_addr'])) diff --git a/zun/tests/unit/objects/test_objects.py b/zun/tests/unit/objects/test_objects.py index bfc041797..423c2d93d 100644 --- a/zun/tests/unit/objects/test_objects.py +++ b/zun/tests/unit/objects/test_objects.py @@ -354,8 +354,8 @@ object_data = { 'ResourceProvider': '1.0-92b427359d5a4cf9ec6c72cbe630ee24', 'ZunService': '1.1-b1549134bfd5271daec417ca8cabc77e', 'Capsule': '1.3-f4c6b8fede0fa9488fc4f77b97601654', - 'PciDevice': '1.0-19fdd11935cda5e92947913f081d9edd', - 'ComputeNode': '1.8-5f5f893df0b514c88a3abaec1dfbb89e', + 'PciDevice': '1.1-6e3f0851ad1cf12583e6af4df1883979', + 'ComputeNode': '1.9-e8536102d3b28cb3378e9e26f508cd72', 'PciDevicePool': '1.0-3f5ddc3ff7bfa14da7f6c7e9904cc000', 'PciDevicePoolList': '1.0-15ecf022a68ddbb8c2a6739cfc9f8f5e' } diff --git a/zun/tests/unit/pci/test_manager.py b/zun/tests/unit/pci/test_manager.py new file mode 100644 index 000000000..a6ad63625 --- /dev/null +++ b/zun/tests/unit/pci/test_manager.py @@ -0,0 +1,299 @@ +# Copyright (c) 2017 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy + +import mock + +import zun +from zun.common import context +from zun.objects import fields +from zun.pci import manager +from zun.tests.unit.db import base +from zun.tests.unit.pci import fakes as pci_fakes +from zun.tests import uuidsentinel + + +fake_pci = { + 'compute_node_uuid': 1, + 'address': '0000:00:00.1', + 'product_id': 'p', + 'vendor_id': 'v', + 'request_id': None, + 'status': fields.PciDeviceStatus.AVAILABLE, + 'dev_type': fields.PciDeviceType.STANDARD, + 'parent_addr': None, + 'numa_node': 0} +fake_pci_1 = dict(fake_pci, address='0000:00:00.2', + product_id='p1', vendor_id='v1') +fake_pci_2 = dict(fake_pci, address='0000:00:00.3') + +fake_pci_3 = dict(fake_pci, address='0000:00:01.1', + dev_type=fields.PciDeviceType.SRIOV_PF, + vendor_id='v2', product_id='p2', numa_node=None) +fake_pci_4 = dict(fake_pci, address='0000:00:02.1', + dev_type=fields.PciDeviceType.SRIOV_VF, + parent_addr='0000:00:01.1', + vendor_id='v2', product_id='p2', numa_node=None) +fake_pci_5 = dict(fake_pci, address='0000:00:02.2', + dev_type=fields.PciDeviceType.SRIOV_VF, + parent_addr='0000:00:01.1', + vendor_id='v2', product_id='p2', numa_node=None) + +fake_db_dev = { + 'created_at': None, + 'updated_at': None, + 'deleted_at': None, + 'deleted': None, + 'id': 1, + 'uuid': uuidsentinel.pci_device1, + 'compute_node_uuid': 1, + 'address': '0000:00:00.1', + 'vendor_id': 'v', + 'product_id': 'p', + 'numa_node': 1, + 'dev_type': fields.PciDeviceType.STANDARD, + 'status': fields.PciDeviceStatus.AVAILABLE, + 'dev_id': 'i', + 'label': 'l', + 'container_uuid': None, + 'extra_info': '{}', + 'request_id': None, + 'parent_addr': None, + } +fake_db_dev_1 = dict(fake_db_dev, vendor_id='v1', + uuid=uuidsentinel.pci_device1, + product_id='p1', id=2, + address='0000:00:00.2', + numa_node=0) +fake_db_dev_2 = dict(fake_db_dev, id=3, address='0000:00:00.3', + uuid=uuidsentinel.pci_device2, + numa_node=None, parent_addr='0000:00:00.1') +fake_db_devs = [fake_db_dev, fake_db_dev_1, fake_db_dev_2] + +fake_db_dev_3 = dict(fake_db_dev, id=4, address='0000:00:01.1', + uuid=uuidsentinel.pci_device3, + vendor_id='v2', product_id='p2', + numa_node=None, dev_type=fields.PciDeviceType.SRIOV_PF) +fake_db_dev_4 = dict(fake_db_dev, id=5, address='0000:00:02.1', + uuid=uuidsentinel.pci_device4, + numa_node=None, dev_type=fields.PciDeviceType.SRIOV_VF, + vendor_id='v2', product_id='p2', + parent_addr='0000:00:01.1') +fake_db_dev_5 = dict(fake_db_dev, id=6, address='0000:00:02.2', + uuid=uuidsentinel.pci_device5, + numa_node=None, dev_type=fields.PciDeviceType.SRIOV_VF, + vendor_id='v2', product_id='p2', + parent_addr='0000:00:01.1') +fake_db_devs_tree = [fake_db_dev_3, fake_db_dev_4, fake_db_dev_5] + + +class PciDevTrackerTestCase(base.DbTestCase): + def _fake_get_pci_devices(self, node_id): + return self.fake_devs + + def _fake_pci_device_update(self, node_id, address, value): + self.update_called += 1 + self.called_values = value + fake_return = copy.deepcopy(fake_db_dev) + return fake_return + + def _fake_pci_device_destroy(self, node_id, address): + self.destroy_called += 1 + + def _create_tracker(self, fake_devs): + self.fake_devs = fake_devs + self.tracker = manager.PciDevTracker(self.fake_context, 1) + + def setUp(self): + super(PciDevTrackerTestCase, self).setUp() + self.fake_context = context.get_admin_context() + self.fake_devs = fake_db_devs[:] + self.stub_out('zun.db.api.get_all_pci_device_by_node', + self._fake_get_pci_devices) + # The fake_pci_whitelist must be called before creating the fake + # devices + patcher = pci_fakes.fake_pci_whitelist() + self.addCleanup(patcher.stop) + self._create_tracker(fake_db_devs[:]) + + def test_pcidev_tracker_create(self): + self.assertEqual(len(self.tracker.pci_devs), 3) + free_devs = self.tracker.pci_stats.get_free_devs() + self.assertEqual(len(free_devs), 3) + self.assertEqual(list(self.tracker.stale), []) + self.assertEqual(len(self.tracker.stats.pools), 3) + self.assertEqual(self.tracker.node_id, 1) + for dev in self.tracker.pci_devs: + self.assertIsNone(dev.parent_device) + self.assertEqual(dev.child_devices, []) + + def test_pcidev_tracker_create_device_tree(self): + self._create_tracker(fake_db_devs_tree) + + self.assertEqual(len(self.tracker.pci_devs), 3) + free_devs = self.tracker.pci_stats.get_free_devs() + self.assertEqual(len(free_devs), 3) + self.assertEqual(list(self.tracker.stale), []) + self.assertEqual(len(self.tracker.stats.pools), 2) + self.assertEqual(self.tracker.node_id, 1) + pf = [dev for dev in self.tracker.pci_devs + if dev.dev_type == fields.PciDeviceType.SRIOV_PF].pop() + vfs = [dev for dev in self.tracker.pci_devs + if dev.dev_type == fields.PciDeviceType.SRIOV_VF] + self.assertEqual(2, len(vfs)) + + # Assert we build the device tree correctly + self.assertEqual(vfs, pf.child_devices) + for vf in vfs: + self.assertEqual(vf.parent_device, pf) + + def test_pcidev_tracker_create_device_tree_pf_only(self): + self._create_tracker([fake_db_dev_3]) + + self.assertEqual(len(self.tracker.pci_devs), 1) + free_devs = self.tracker.pci_stats.get_free_devs() + self.assertEqual(len(free_devs), 1) + self.assertEqual(list(self.tracker.stale), []) + self.assertEqual(len(self.tracker.stats.pools), 1) + self.assertEqual(self.tracker.node_id, 1) + pf = self.tracker.pci_devs[0] + self.assertIsNone(pf.parent_device) + self.assertEqual([], pf.child_devices) + + def test_pcidev_tracker_create_device_tree_vf_only(self): + self._create_tracker([fake_db_dev_4]) + + self.assertEqual(len(self.tracker.pci_devs), 1) + free_devs = self.tracker.pci_stats.get_free_devs() + self.assertEqual(len(free_devs), 1) + self.assertEqual(list(self.tracker.stale), []) + self.assertEqual(len(self.tracker.stats.pools), 1) + self.assertEqual(self.tracker.node_id, 1) + vf = self.tracker.pci_devs[0] + self.assertIsNone(vf.parent_device) + self.assertEqual([], vf.child_devices) + + @mock.patch.object(zun.objects.PciDevice, 'list_by_compute_node') + def test_pcidev_tracker_create_no_nodeid(self, mock_get_cn): + self.tracker = manager.PciDevTracker(self.fake_context) + self.assertEqual(len(self.tracker.pci_devs), 0) + self.assertFalse(mock_get_cn.called) + + @mock.patch.object(zun.objects.PciDevice, 'list_by_compute_node') + def test_pcidev_tracker_create_with_nodeid(self, mock_get_cn): + self.tracker = manager.PciDevTracker(self.fake_context, node_id=1) + mock_get_cn.assert_called_once_with(self.fake_context, 1) + + def test_set_hvdev_new_dev(self): + fake_pci_3 = dict(fake_pci, address='0000:00:00.4', vendor_id='v2') + fake_pci_devs = [copy.deepcopy(fake_pci), copy.deepcopy(fake_pci_1), + copy.deepcopy(fake_pci_2), copy.deepcopy(fake_pci_3)] + self.tracker._set_hvdevs(fake_pci_devs) + self.assertEqual(len(self.tracker.pci_devs), 4) + self.assertEqual(set([dev.address for + dev in self.tracker.pci_devs]), + set(['0000:00:00.1', '0000:00:00.2', + '0000:00:00.3', '0000:00:00.4'])) + self.assertEqual(set([dev.vendor_id for + dev in self.tracker.pci_devs]), + set(['v', 'v1', 'v2'])) + + def test_set_hvdev_new_dev_tree_maintained(self): + # Make sure the device tree is properly maintained when there are new + # devices reported by the driver + self._create_tracker(fake_db_devs_tree) + + fake_new_device = dict(fake_pci_5, id=12, address='0000:00:02.3') + fake_pci_devs = [copy.deepcopy(fake_pci_3), + copy.deepcopy(fake_pci_4), + copy.deepcopy(fake_pci_5), + copy.deepcopy(fake_new_device)] + self.tracker._set_hvdevs(fake_pci_devs) + self.assertEqual(len(self.tracker.pci_devs), 4) + + pf = [dev for dev in self.tracker.pci_devs + if dev.dev_type == fields.PciDeviceType.SRIOV_PF].pop() + vfs = [dev for dev in self.tracker.pci_devs + if dev.dev_type == fields.PciDeviceType.SRIOV_VF] + self.assertEqual(3, len(vfs)) + + # Assert we build the device tree correctly + self.assertEqual(vfs, pf.child_devices) + for vf in vfs: + self.assertEqual(vf.parent_device, pf) + + def test_set_hvdev_changed(self): + fake_pci_v2 = dict(fake_pci, address='0000:00:00.2', vendor_id='v1') + fake_pci_devs = [copy.deepcopy(fake_pci), copy.deepcopy(fake_pci_2), + copy.deepcopy(fake_pci_v2)] + self.tracker._set_hvdevs(fake_pci_devs) + self.assertEqual(set([dev.vendor_id for + dev in self.tracker.pci_devs]), + set(['v', 'v1'])) + + def test_set_hvdev_remove(self): + self.tracker._set_hvdevs([fake_pci]) + self.assertEqual( + len([dev for dev in self.tracker.pci_devs + if dev.status == fields.PciDeviceStatus.REMOVED]), + 2) + + def test_set_hvdev_remove_tree_maintained(self): + # Make sure the device tree is properly maintained when there are + # devices removed from the system (not reported by the driver but known + # from previous scans) + self._create_tracker(fake_db_devs_tree) + + fake_pci_devs = [copy.deepcopy(fake_pci_3), copy.deepcopy(fake_pci_4)] + self.tracker._set_hvdevs(fake_pci_devs) + self.assertEqual( + 2, + len([dev for dev in self.tracker.pci_devs + if dev.status != fields.PciDeviceStatus.REMOVED])) + pf = [dev for dev in self.tracker.pci_devs + if dev.dev_type == fields.PciDeviceType.SRIOV_PF].pop() + vfs = [dev for dev in self.tracker.pci_devs + if (dev.dev_type == fields.PciDeviceType.SRIOV_VF and + dev.status != fields.PciDeviceStatus.REMOVED)] + self.assertEqual(1, len(vfs)) + + self.assertEqual(vfs, pf.child_devices) + self.assertEqual(vfs[0].parent_device, pf) + + def test_save(self): + self.stub_out('zun.db.api.update_pci_device', + self._fake_pci_device_update) + fake_pci_v3 = dict(fake_pci, address='0000:00:00.2', vendor_id='v3') + fake_pci_devs = [copy.deepcopy(fake_pci), copy.deepcopy(fake_pci_2), + copy.deepcopy(fake_pci_v3)] + self.tracker._set_hvdevs(fake_pci_devs) + self.update_called = 0 + self.tracker.save() + self.assertEqual(self.update_called, 3) + + def test_save_removed(self): + self.stub_out('zun.db.api.update_pci_device', + self._fake_pci_device_update) + self.stub_out('zun.db.api.destroy_pci_device', + self._fake_pci_device_destroy) + self.assertEqual(len(self.tracker.pci_devs), 3) + dev = self.tracker.pci_devs[0] + self.destroy_called = 0 + self.update_called = 0 + dev.remove() + self.tracker.save() + self.assertEqual(len(self.tracker.pci_devs), 2) + self.assertEqual(self.destroy_called, 1) diff --git a/zun/tests/unit/pci/test_utils.py b/zun/tests/unit/pci/test_utils.py index eb60071b6..5199be1b9 100644 --- a/zun/tests/unit/pci/test_utils.py +++ b/zun/tests/unit/pci/test_utils.py @@ -263,3 +263,46 @@ class GetVfNumByPciAddressTestCase(base.TestCase): utils.get_vf_num_by_pci_address, self.pci_address ) + + +class GetNetNameByVfPciAddressTestCase(base.TestCase): + + def setUp(self): + super(GetNetNameByVfPciAddressTestCase, self).setUp() + self._get_mac = mock.patch.object(utils, 'get_mac_by_pci_address') + self.mock_get_mac = self._get_mac.start() + self._get_ifname = mock.patch.object( + utils, 'get_ifname_by_pci_address') + self.mock_get_ifname = self._get_ifname.start() + self.addCleanup(self._get_mac.stop) + self.addCleanup(self._get_ifname.stop) + + self.mac = 'ca:fe:ca:fe:ca:fe' + self.if_name = 'enp7s0f0' + self.pci_address = '0000:07:02.1' + + def test_correct_behaviour(self): + ref_net_name = ('net_enp7s0f0_ca_fe_ca_fe_ca_fe', 'enp7s0f0') + self.mock_get_mac.return_value = self.mac + self.mock_get_ifname.return_value = self.if_name + net_name = utils.get_net_name_by_vf_pci_address(self.pci_address) + self.assertEqual(ref_net_name, net_name) + self.mock_get_mac.called_once_with(self.pci_address) + self.mock_get_ifname.called_once_with(self.pci_address) + + def test_wrong_mac(self): + self.mock_get_mac.side_effect = ( + exception.PciDeviceNotFoundById(self.pci_address)) + net_name = utils.get_net_name_by_vf_pci_address(self.pci_address) + self.assertIsNone(net_name) + self.mock_get_mac.called_once_with(self.pci_address) + self.mock_get_ifname.assert_not_called() + + def test_wrong_ifname(self): + self.mock_get_mac.return_value = self.mac + self.mock_get_ifname.side_effect = ( + exception.PciDeviceNotFoundById(self.pci_address)) + net_name = utils.get_net_name_by_vf_pci_address(self.pci_address) + self.assertIsNone(net_name) + self.mock_get_mac.called_once_with(self.pci_address) + self.mock_get_ifname.called_once_with(self.pci_address)