Add PCI devices plugin to inspector

Adds a new plugin to distinguish PCI devices returned by Ironic
Python Agent. Recognized PCI devices are then registered in node
capabilities and later can be used by nova flavors.

Change-Id: I6565b8c4aa76de240a6c4d795635300ff2d0c30b
Partial-Bug: #1580893

config-generator.conf

@@ -5,6 +5,7 @@ namespace = ironic_inspector.common.ironic
 namespace = ironic_inspector.common.swift
 namespace = ironic_inspector.plugins.capabilities
 namespace = ironic_inspector.plugins.discovery
+namespace = ironic_inspector.plugins.pci_devices
 namespace = keystonemiddleware.auth_token
 namespace = oslo.db
 namespace = oslo.log

devstack/plugin.sh

@@ -18,7 +18,7 @@ IRONIC_INSPECTOR_URI="http://$IRONIC_INSPECTOR_HOST:$IRONIC_INSPECTOR_PORT"
 IRONIC_INSPECTOR_BUILD_RAMDISK=$(trueorfalse False IRONIC_INSPECTOR_BUILD_RAMDISK)
 IRONIC_AGENT_KERNEL_URL=${IRONIC_AGENT_KERNEL_URL:-http://tarballs.openstack.org/ironic-python-agent/coreos/files/coreos_production_pxe.vmlinuz}
 IRONIC_AGENT_RAMDISK_URL=${IRONIC_AGENT_RAMDISK_URL:-http://tarballs.openstack.org/ironic-python-agent/coreos/files/coreos_production_pxe_image-oem.cpio.gz}
-IRONIC_INSPECTOR_COLLECTORS=${IRONIC_INSPECTOR_COLLECTORS:-default,logs}
+IRONIC_INSPECTOR_COLLECTORS=${IRONIC_INSPECTOR_COLLECTORS:-default,logs,pci-devices}
 IRONIC_INSPECTOR_RAMDISK_LOGDIR=${IRONIC_INSPECTOR_RAMDISK_LOGDIR:-$IRONIC_INSPECTOR_DATA_DIR/ramdisk-logs}
 IRONIC_INSPECTOR_ALWAYS_STORE_RAMDISK_LOGS=${IRONIC_INSPECTOR_ALWAYS_STORE_RAMDISK_LOGS:-True}
 IRONIC_INSPECTOR_TIMEOUT=${IRONIC_INSPECTOR_TIMEOUT:-600}

doc/source/usage.rst

@@ -199,6 +199,12 @@ needed:
 ``capabilities``
     detect node capabilities: CPU, boot mode, etc. See `Capabilities
     Detection`_ for more details.
+``pci_devices``
+    gathers the list of all PCI devices returned by the ramdisk and compares to
+    those defined in ``alias`` field(s) from ``pci_devices`` section of
+    configuration file. The recognized PCI devices and their count are then
+    stored in node properties. This information can be later used in nova
+    flavors for node scheduling.
 
 Here are some plugins that can be additionally enabled:
 

ironic_inspector/conf.py

@@ -79,7 +79,7 @@ PROCESSING_OPTS = [
                 deprecated_group='discoverd'),
     cfg.StrOpt('default_processing_hooks',
                default='ramdisk_error,root_disk_selection,scheduler,'
-                       'validate_interfaces,capabilities',
+                       'validate_interfaces,capabilities,pci_devices',
                help='Comma-separated list of default hooks for processing '
                     'pipeline. Hook \'scheduler\' updates the node with the '
                     'minimum properties required by the Nova scheduler. '

ironic_inspector/plugins/pci_devices.py

@@ -0,0 +1,87 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Gather and distinguish PCI devices from inventory."""
+
+import collections
+import json
+
+from oslo_config import cfg
+
+from ironic_inspector.common.i18n import _LI, _LW, _LE
+from ironic_inspector.plugins import base
+from ironic_inspector import utils
+
+PCI_DEVICES_OPTS = [
+    cfg.MultiStrOpt('alias',
+                    default=[],
+                    help='An alias for PCI device identified by \'vendor_id\' '
+                         'and \'product_id\' fields. Format: '
+                         '{"vendor_id": "1234", "product_id": "5678", "name": '
+                         '"pci_dev1"}'),
+]
+
+
+def list_opts():
+    return [
+        ('pci_devices', PCI_DEVICES_OPTS)
+    ]
+
+CONF = cfg.CONF
+CONF.register_opts(PCI_DEVICES_OPTS, group='pci_devices')
+
+LOG = utils.getProcessingLogger(__name__)
+
+
+def _parse_pci_alias_entry():
+    parsed_pci_devices = []
+    for pci_alias_entry in CONF.pci_devices.alias:
+        try:
+            parsed_entry = json.loads(pci_alias_entry)
+            if set(parsed_entry) != {'vendor_id', 'product_id', 'name'}:
+                raise KeyError(_LE("The 'alias' entry should contain "
+                                   "exactly 'vendor_id', 'product_id' and "
+                                   "'name' keys"))
+            parsed_pci_devices.append(parsed_entry)
+        except (ValueError, KeyError) as ex:
+            LOG.error(_LE("Error parsing 'alias' option: %s"), ex)
+    return {(dev['vendor_id'], dev['product_id']): dev['name']
+            for dev in parsed_pci_devices}
+
+
+class PciDevicesHook(base.ProcessingHook):
+    """Processing hook for counting and distinguishing various PCI devices.
+
+        That information can be later used by nova for node scheduling.
+    """
+    aliases = _parse_pci_alias_entry()
+
+    def _found_pci_devices_count(self, found_pci_devices):
+        return collections.Counter([(dev['vendor_id'], dev['product_id'])
+                                    for dev in found_pci_devices
+                                    if (dev['vendor_id'], dev['product_id'])
+                                    in self.aliases])
+
+    def before_update(self, introspection_data, node_info, **kwargs):
+        if 'pci_devices' not in introspection_data:
+            if CONF.pci_devices.alias:
+                LOG.warning(_LW('No PCI devices information was received from '
+                            'the ramdisk.'))
+            return
+        alias_count = {self.aliases[id_pair]: count for id_pair, count in
+                       self._found_pci_devices_count(
+                           introspection_data['pci_devices']).items()}
+        if alias_count:
+            node_info.update_capabilities(**alias_count)
+            LOG.info(_LI('Found the following PCI devices: %s'),
+                     alias_count)

ironic_inspector/test/unit/test_plugins_pci_devices.py

@@ -0,0 +1,102 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+
+from ironic_inspector import node_cache
+from ironic_inspector.plugins import base
+from ironic_inspector.plugins import pci_devices
+from ironic_inspector.test import base as test_base
+
+
+class TestPciDevicesHook(test_base.NodeTest):
+    hook = pci_devices.PciDevicesHook()
+
+    def test_parse_pci_alias_entry(self):
+        pci_alias = ['{"vendor_id": "foo1", "product_id": "bar1",'
+                     ' "name": "baz1"}',
+                     '{"vendor_id": "foo2", "product_id": "bar2",'
+                     ' "name": "baz2"}']
+        valid_pci_entry = {("foo1", "bar1"): "baz1", ("foo2", "bar2"): "baz2"}
+        base.CONF.set_override('alias', pci_alias, 'pci_devices')
+        parsed_pci_entry = pci_devices._parse_pci_alias_entry()
+        self.assertDictEqual(valid_pci_entry, parsed_pci_entry)
+
+    def test_parse_pci_alias_entry_no_entries(self):
+        pci_alias = []
+        base.CONF.set_override('alias', pci_alias, 'pci_devices')
+        parsed_pci_alias = pci_devices._parse_pci_alias_entry()
+        self.assertFalse(parsed_pci_alias)
+
+    @mock.patch('ironic_inspector.plugins.pci_devices.LOG')
+    def test_parse_pci_alias_entry_invalid_json(self, mock_oslo_log):
+        pci_alias = ['{"vendor_id": "foo1", "product_id": "bar1",'
+                     ' "name": "baz1"}', '{"invalid" = "entry"}']
+        base.CONF.set_override('alias', pci_alias, 'pci_devices')
+        valid_pci_alias = {("foo1", "bar1"): "baz1"}
+        parsed_pci_alias = pci_devices._parse_pci_alias_entry()
+        self.assertDictEqual(valid_pci_alias, parsed_pci_alias)
+        mock_oslo_log.error.assert_called_once()
+
+    @mock.patch('ironic_inspector.plugins.pci_devices.LOG')
+    def test_parse_pci_alias_entry_invalid_keys(self, mock_oslo_log):
+        pci_alias = ['{"vendor_id": "foo1", "product_id": "bar1",'
+                     ' "name": "baz1"}', '{"invalid": "keys"}']
+        base.CONF.set_override('alias', pci_alias, 'pci_devices')
+        valid_pci_alias = {("foo1", "bar1"): "baz1"}
+        parsed_pci_alias = pci_devices._parse_pci_alias_entry()
+        self.assertDictEqual(valid_pci_alias, parsed_pci_alias)
+        mock_oslo_log.error.assert_called_once()
+
+    @mock.patch.object(hook, 'aliases', {("1234", "5678"): "pci_dev1",
+                                         ("9876", "5432"): "pci_dev2"})
+    @mock.patch.object(node_cache.NodeInfo, 'update_capabilities',
+                       autospec=True)
+    def test_before_update(self, mock_update_props):
+        self.data['pci_devices'] = [
+            {"vendor_id": "1234", "product_id": "5678"},
+            {"vendor_id": "1234", "product_id": "5678"},
+            {"vendor_id": "1234", "product_id": "7890"},
+            {"vendor_id": "9876", "product_id": "5432"}
+        ]
+        expected_pci_devices_count = {"pci_dev1": 2, "pci_dev2": 1}
+        self.hook.before_update(self.data, self.node_info)
+        mock_update_props.assert_called_once_with(self.node_info,
+                                                  **expected_pci_devices_count)
+
+    @mock.patch('ironic_inspector.plugins.pci_devices.LOG')
+    @mock.patch.object(node_cache.NodeInfo, 'update_capabilities',
+                       autospec=True)
+    def test_before_update_no_pci_info_from_ipa(self, mock_update_props,
+                                                mock_oslo_log):
+        pci_alias = ['{"vendor_id": "foo1", "product_id": "bar1",'
+                     ' "name": "baz1"}']
+        base.CONF.set_override('alias', pci_alias, 'pci_devices')
+        self.hook.before_update(self.data, self.node_info)
+        mock_oslo_log.warning.assert_called_once()
+        self.assertFalse(mock_update_props.called)
+
+    @mock.patch.object(pci_devices, '_parse_pci_alias_entry')
+    @mock.patch('ironic_inspector.plugins.pci_devices.LOG')
+    @mock.patch.object(node_cache.NodeInfo, 'update_capabilities',
+                       autospec=True)
+    def test_before_update_no_match(self, mock_update_props, mock_oslo_log,
+                                    mock_parse_pci_alias):
+        self.data['pci_devices'] = [
+            {"vendor_id": "1234", "product_id": "5678"},
+            {"vendor_id": "1234", "product_id": "7890"},
+        ]
+        mock_parse_pci_alias.return_value = {("9876", "5432"): "pci_dev"}
+        self.hook.before_update(self.data, self.node_info)
+        self.assertFalse(mock_update_props.called)
+        self.assertFalse(mock_oslo_log.info.called)

releasenotes/notes/pci_devices-plugin-5b93196e0e973155.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - Adds new processing hook pci_devices for setting node
+    capabilities based on PCI devices present on a node
+    and rules in the [pci_devices] aliases configuration
+    option. Requires "pci-devices" collector to be enabled
+    in IPA.

setup.cfg

@@ -33,6 +33,7 @@ ironic_inspector.hooks.processing =
     raid_device = ironic_inspector.plugins.raid_device:RaidDeviceDetection
     capabilities = ironic_inspector.plugins.capabilities:CapabilitiesHook
     local_link_connection = ironic_inspector.plugins.local_link_connection:GenericLocalLinkConnectionHook
+    pci_devices = ironic_inspector.plugins.pci_devices:PciDevicesHook
 ironic_inspector.hooks.node_not_found =
     example = ironic_inspector.plugins.example:example_not_found_hook
     enroll = ironic_inspector.plugins.discovery:enroll_node_not_found_hook
@@ -59,6 +60,7 @@ oslo.config.opts =
     ironic_inspector.common.swift = ironic_inspector.common.swift:list_opts
     ironic_inspector.plugins.discovery = ironic_inspector.plugins.discovery:list_opts
     ironic_inspector.plugins.capabilities = ironic_inspector.plugins.capabilities:list_opts
+    ironic_inspector.plugins.pci_devices = ironic_inspector.plugins.pci_devices:list_opts
 oslo.config.opts.defaults =
     ironic_inspector = ironic_inspector.conf:set_config_defaults