# Copyright 2020 Red Hat Inc. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import testtools import time from tempest.common import compute from tempest.common.utils.linux import remote_client from tempest import config from tempest import exceptions as tempest_exc from tempest.lib import exceptions as lib_exc from whitebox_tempest_plugin.api.compute import base from whitebox_tempest_plugin.api.compute import numa_helper from whitebox_tempest_plugin import hardware from whitebox_tempest_plugin.services import clients from oslo_log import log as logging CONF = config.CONF LOG = logging.getLogger(__name__) class SRIOVBase(base.BaseWhiteboxComputeTest): @classmethod def skip_checks(cls): super(SRIOVBase, cls).skip_checks() if getattr(CONF.whitebox_hardware, 'sriov_physnet', None) is None: raise cls.skipException('Requires sriov_physnet parameter ' 'to be set in order to execute test ' 'cases.') if getattr(CONF.whitebox_hardware, 'sriov_vlan_id', None) is None: raise cls.skipException('Requires ' 'sriov_vlan_id parameter ' 'to be set in order to execute ' 'test cases.') def _validate_pf_pci_address_in_xml(self, port_id, host_dev_xml): """Validates pci address matches between port info and guest XML :param server_id: str, id of the instance to analyze :param host_dev_xml: eTree XML, host dev xml element """ binding_profile = self._get_port_attribute(port_id, 'binding:profile') pci_addr_element = host_dev_xml.find("./source/address") pci_address = hardware.get_pci_address_from_xml_device( pci_addr_element) self.assertEqual( pci_address, binding_profile['pci_slot'], 'PCI device found in XML %s' 'does not match what is tracked in binding profile for port %s' % (pci_address, binding_profile)) def _get_xml_pf_device(self, server_id): """Returns xml hostdev element from the provided server id :param server_id: str, id of the instance to analyze :return xml_network_deivce: The xml hostdev device element that matches the device search criteria """ root = self.get_server_xml(server_id) hostdev_list = root.findall( "./devices/hostdev[@type='pci']" ) self.assertEqual(len(hostdev_list), 1, 'Expect to find one ' 'and only one instance of hostdev device but ' 'instead found %d instances' % len(hostdev_list)) return hostdev_list[0] def _validate_port_xml_vlan_tag(self, port_xml_element, expected_vlan): """Validates port count and vlan are accurate in server's XML :param server_id: str, id of the instance to analyze :param port: dictionary describing port to find """ interface_vlan = port_xml_element.find("./vlan/tag").get('id', None) found_vlan = int(interface_vlan) if interface_vlan else None self.assertEqual( expected_vlan, found_vlan, 'Interface should have have vlan ' 'tag %s but instead it is tagged with %s' % (expected_vlan, found_vlan)) class SRIOVNumaAffinity(SRIOVBase, numa_helper.NUMAHelperMixin): # Test utilizes the optional host parameter for server creation introduced # in 2.74. It allows the guest to be scheduled to a specific compute host. # This allows the test to fill NUMA nodes on the same host. min_microversion = '2.74' required = {'hw:cpu_policy': 'dedicated', 'hw:pci_numa_affinity_policy': 'required'} preferred = {'hw:cpu_policy': 'dedicated', 'hw:pci_numa_affinity_policy': 'preferred'} @classmethod def skip_checks(cls): super(SRIOVNumaAffinity, cls).skip_checks() if ( CONF.whitebox_hardware.sriov_vnic_type not in ['direct', 'macvtap'] ): raise cls.skipException('Tests are designed for vnic types ' 'direct or macvtap') if getattr(CONF.whitebox_hardware, 'physnet_numa_affinity', None) is None: raise cls.skipException('Requires physnet_numa_affinity parameter ' 'to be set in order to execute test ' 'cases.') if getattr(CONF.whitebox_hardware, 'dedicated_cpus_per_numa', None) is None: raise cls.skipException('Requires dedicated_cpus_per_numa ' 'parameter to be set in order to execute ' 'test cases.') if len(CONF.whitebox_hardware.cpu_topology) < 2: raise cls.skipException('Requires 2 or more NUMA nodes to ' 'execute test.') if not compute.is_scheduler_filter_enabled('SameHostFilter'): raise cls.skipException('SameHostFilter required.') def setUp(self): super(SRIOVNumaAffinity, self).setUp() self.vlan_id = \ CONF.whitebox_hardware.sriov_vlan_id self.dedicated_cpus_per_numa = \ CONF.whitebox_hardware.dedicated_cpus_per_numa self.affinity_node = str(CONF.whitebox_hardware.physnet_numa_affinity) self.physical_net = CONF.whitebox_hardware.sriov_physnet self.vnic_type = CONF.whitebox_hardware.sriov_vnic_type self.network = self._create_net_from_physical_network( self.vlan_id, self.physical_net) self._create_subnet(self.network['network']['id']) self.flavor = self.create_flavor( vcpus=self.dedicated_cpus_per_numa, extra_specs={'hw:cpu_policy': 'dedicated'} ) def _get_dedicated_cpus_from_numa_node(self, numa_node, cpu_dedicated_set): cpu_ids = set(CONF.whitebox_hardware.cpu_topology.get(numa_node)) dedicated_cpus = cpu_dedicated_set.intersection(cpu_ids) return dedicated_cpus def _preferred_test_procedure(self, flavor, port_a, port_b, image_id): server_a = self.create_test_server( flavor=flavor['id'], networks=[{'port': port_a['port']['id']}], image_id=image_id, wait_until='ACTIVE' ) # Determine the host that guest A lands on and use that information # to force guest B to land on the same host host = self.get_host_for_server(server_a['id']) server_b = self.create_test_server( flavor=flavor['id'], networks=[{'port': port_b['port']['id']}], scheduler_hints={'same_host': server_a['id']}, image_id=image_id, wait_until='ACTIVE' ) # Determine the pCPUs that have affinity with the host's SR-IOV port. # Then confirm the first instance's pCPUs match the pCPUs from the # NUMA node with affinity to the SR-IOV port. host_sm = clients.NovaServiceManager(host, 'nova-compute', self.os_admin.services_client) cpu_dedicated_set = host_sm.get_cpu_dedicated_set() cpu_pins_a = self.get_pinning_as_set(server_a['id']) pcpus_with_affinity = self._get_dedicated_cpus_from_numa_node( self.affinity_node, cpu_dedicated_set) self.assertEqual( cpu_pins_a, pcpus_with_affinity, 'Expected pCPUs for server A, ' 'id: %s to be equal to %s but instead are %s' % (server_a['id'], pcpus_with_affinity, cpu_pins_a)) # Find the pinned pCPUs used by server B. They are not expected to have # affinity so just confirm they are a subset of the host's # cpu_dedicated_set. Also confirm pCPUs are not rescued between guest A # and B cpu_pins_b = self.get_pinning_as_set(server_b['id']) self.assertTrue( cpu_pins_b.issubset(set(cpu_dedicated_set)), 'Expected pCPUs for server B id: %s to be subset of %s but ' 'instead are %s' % (server_b['id'], cpu_dedicated_set, cpu_pins_b)) self.assertTrue( cpu_pins_a.isdisjoint(cpu_pins_b), 'Cpus %s for server A %s are not disjointed with Cpus %s of ' 'server B %s' % (cpu_pins_a, server_a['id'], cpu_pins_b, server_b['id'])) # Validate servers A and B have correct sr-iov interface # information in the xml. Its type and vlan should be accurate. for server, port in zip([server_a, server_b], [port_a, port_b]): interface_xml_element = self._get_xml_interface_device( server['id'], port['port']['id'] ) self._validate_port_xml_vlan_tag( interface_xml_element, self.vlan_id) def _required_test_procedure(self, flavor, port_a, port_b, image_id): server_a = self.create_test_server( flavor=flavor['id'], networks=[{'port': port_a['port']['id']}], image_id=image_id, wait_until='ACTIVE' ) # Determine the host that guest A lands on and use that information # to force guest B to land on the same host. With server A 'filling' # pCPUs from the NUMA Node with SR-IOV NIC affinity, and with NUMA # policy set to required, creation of server B should fail host = self.get_host_for_server(server_a['id']) self.assertRaises(tempest_exc.BuildErrorException, self.create_test_server, flavor=flavor['id'], networks=[{'port': port_b['port']['id']}], scheduler_hints={'same_host': server_a['id']}, image_id=image_id, wait_until='ACTIVE') # Determine the pCPUs that have affinity with the host's SR-IOV port. # Then confirm the first instance's pCPUs match the pCPUs from the # NUMA node with affinity to the SR-IOV port. host_sm = clients.NovaServiceManager(host, 'nova-compute', self.os_admin.services_client) cpu_dedicated_set = host_sm.get_cpu_dedicated_set() pcpus_with_affinity = self._get_dedicated_cpus_from_numa_node( self.affinity_node, cpu_dedicated_set) cpu_pins_a = self.get_pinning_as_set(server_a['id']) # Compare the cpu pin set from server A with the expected PCPU's # from the NUMA Node with affinity to SR-IOV NIC that was gathered # earlier from from cpu_topology self.assertEqual( cpu_pins_a, pcpus_with_affinity, 'Expected pCPUs for server %s ' 'to be equal to %s but instead are %s' % (server_a['id'], pcpus_with_affinity, cpu_pins_a)) # Validate server A has correct sr-iov interface information # in the xml. Its type and vlan should be accurate. interface_xml_element = self._get_xml_interface_device( server_a['id'], port_a['port']['id'] ) self._validate_port_xml_vlan_tag(interface_xml_element, self.vlan_id) class SRIOVNumaAffinityWithFlavor(SRIOVNumaAffinity): def test_sriov_affinity_preferred_with_flavor(self): """Validate preferred NUMA affinity with flavor level configuration 1. Create a flavor with preferred NUMA policy and hw:cpu_policy=dedicated. The flavor vcpu size will be equal to the number of dedicated PCPUs of the NUMA Node with affinity to the physnet. This should result in any deployed instance using this flavor 'filling' the NUMA Node completely. 2. Launch two instances with the flavor and an SR-IOV port. The second server should be 'forced' to schedule on the same host as the first instance. 3. Validate both instances are deployed 4. Validate the first instance has CPU affinity with the same NUMA node as the attached SR-IOV interface 5. Validate xml description of SR-IOV interface is correct for both servers """ flavor = self.create_flavor( vcpus=self.dedicated_cpus_per_numa, extra_specs=self.preferred ) port_a = self._create_port_from_vnic_type( net=self.network, vnic_type=self.vnic_type) port_b = self._create_port_from_vnic_type( net=self.network, vnic_type=self.vnic_type) self._preferred_test_procedure(flavor, port_a, port_b, self.image_ref) def test_sriov_affinity_required_with_flavor(self): """Validate required NUMA affinity with flavor level configuration 1. Pick a single compute host and gather its cpu_dedicated_set configuration. Determine which of these dedicated PCPU's have affinity and do not have affinity with the SRIOV physnet. 2. Create flavor with required NUMA policy and hw:cpu_policy=dedicated. The vcpu size of the flavor will be equal to the number of dedicated PCPUs of the NUMA Node with affinity to the physnet. This should result in any deployed instance using this flavor 'filling' the NUMA Node completely. 3. Launch two instances with the flavor and an SR-IOV port. The second server should be 'forced' to schedule on the same host as the first instance. 4. Validate only the first instance is created successfully and the second should fail to deploy 5. Validate the first instance has CPU affinity with the same NUMA node as the attached SR-IOV interface 6. Validate xml description of sr-iov interface is correct for first server 7. Based on the VF pci address provided to the first instance, validate it's NUMA affinity and assert the instance's dedicated pCPU's are all from the same NUMA. """ # Create a cpu_dedicated_set comprised of the PCPU's of just this NUMA # Node flavor = self.create_flavor( vcpus=self.dedicated_cpus_per_numa, extra_specs=self.required ) port_a = self._create_port_from_vnic_type( net=self.network, vnic_type=self.vnic_type) port_b = self._create_port_from_vnic_type( net=self.network, vnic_type=self.vnic_type) self._required_test_procedure(flavor, port_a, port_b, self.image_ref) class SRIOVNumaAffinityWithImagePolicy(SRIOVNumaAffinity): @classmethod def skip_checks(cls): super(SRIOVNumaAffinityWithImagePolicy, cls).skip_checks() if not CONF.compute_feature_enabled.supports_image_level_numa_affinity: raise cls.skipException('Deployment requires support for image ' 'level configuration of NUMA affinity ' 'policy.') def test_sriov_affinity_preferred_with_image(self): """Validate preferred NUMA affinity with image level configuration 1. Pick a single compute host and gather its cpu_dedicated_set configuration. Determine which of these dedicated PCPU's have affinity and do not have affinity with the SRIOV physnet. 2. Create an image with preferred NUMA affinity policy metadata. Also use a flavor with hw:cpu_policy=dedicated and a vCPU size equal to number of pCPUs per NUMA. 3. Launch two instances with the flavor, image, and an SR-IOV port. The second guest should be 'forced' to schedule on the same host as the first instance. 4. Validate both instances are created successfully with the first having NUMA affinity with the SR-IOV port 5. Validate xml description of SR-IOV interface is correct for both guests """ image_id = self.copy_default_image( hw_pci_numa_affinity_policy='preferred') port_a = self._create_port_from_vnic_type( net=self.network, vnic_type=self.vnic_type) port_b = self._create_port_from_vnic_type( net=self.network, vnic_type=self.vnic_type) self._preferred_test_procedure( self.flavor, port_a, port_b, image_id) def test_sriov_affinity_required_with_image(self): """Validate required NUMA affinity with image level configuration 1. Pick a single compute host and gather its cpu_dedicated_set configuration. Determine which of these dedicated PCPU's have affinity and do not have affinity with the SRIOV physnet. 2. Create an image with required NUMA affinity policy metadata. Also use a flavor with hw:cpu_policy=dedicated and a vCPU size equal to number of pCPUs per NUMA. 3. Launch two instances with the flavor, image, and an SR-IOV port. The second guest should be 'forced' to schedule on the same host as the first instance. 4. Validate only the first instance is created successfully and the second should fail to deploy 5. Validate the first instance has CPU affinity with the same NUMA node as the attached SR-IOV interface 6. Validate xml description of sr-iov interface is correct for first guest """ image_id = self.copy_default_image( hw_pci_numa_affinity_policy='required') port_a = self._create_port_from_vnic_type( net=self.network, vnic_type=self.vnic_type) port_b = self._create_port_from_vnic_type( net=self.network, vnic_type=self.vnic_type) self._required_test_procedure( self.flavor, port_a, port_b, image_id) class SRIOVNumaAffinityWithPortPolicy(SRIOVNumaAffinity): @classmethod def skip_checks(cls): super(SRIOVNumaAffinityWithPortPolicy, cls).skip_checks() if not CONF.compute_feature_enabled.supports_port_level_numa_affinity: raise cls.skipException('Deployment requires support for per port ' 'level configuration of NUMA affinity ' 'policy.') def test_sriov_affinity_preferred_with_port_policy(self): """Validate preferred NUMA affinity with port level configuration 1. Create a flavor with hw:cpu_policy=dedicated. The flavor vcpu size will be equal to the number of dedicated PCPUs of the NUMA Node with affinity to the physnet. This should result in any deployed instance using this flavor 'filling' the NUMA Node completely. 2. Create two ports that have the preferred numa affinity policy. 3. Launch two instances using the flavor and ports, with the second instance being 'forced' to schedule to the same host as the first 4. Validate both instances are created successfully with the first having NUMA affinity with the SR-IOV port 5. Validate xml description of SR-IOV interface is correct for both guests """ port_a = self._create_port_from_vnic_type( net=self.network, vnic_type=self.vnic_type, numa_affinity_policy='preferred') port_b = self._create_port_from_vnic_type( net=self.network, vnic_type=self.vnic_type, numa_affinity_policy='preferred') self._preferred_test_procedure( self.flavor, port_a, port_b, self.image_ref) def test_sriov_mixed_affinity_port_policies(self): """Validate mixed NUMA affinity policy with port level configuration 1. Create a flavor with hw:cpu_policy=dedicated. The flavor vcpu size will be equal to the number of dedicated PCPUs of the NUMA Node with affinity to the physnet. This should result in any deployed instance using this flavor 'filling' the NUMA Node completely. 2. Create two ports one with the required numa affinity policy and one with the preferred numa policy 3. Launch an instance with the port using the required policy 3. Launch a second instance and target it to the same host as the first instance with the port using the preferred policy. 4. Validate both instances are created successfully with the first having NUMA affinity with the SR-IOV port 5. Validate xml description of SR-IOV interface is correct for both guests """ port_a = self._create_port_from_vnic_type( net=self.network, vnic_type=self.vnic_type, numa_affinity_policy='required') port_b = self._create_port_from_vnic_type( net=self.network, vnic_type=self.vnic_type, numa_affinity_policy='preferred') self._preferred_test_procedure( self.flavor, port_a, port_b, self.image_ref) def test_sriov_affinity_required_with_port_policy(self): """Validate required NUMA affinity with port level configuration 1. Create a flavor with hw:cpu_policy=dedicated. The flavor vcpu size will be equal to the number of dedicated PCPUs of the NUMA Node with affinity to the physnet. This should result in any deployed instance using this flavor 'filling' the NUMA Node completely. 2. Create two ports that have the required numa affinity policy. 3. Launch two instances using the flavor, the 'required' policy ports and target the same host. 4. Validate only the first instance is created successfully and the second should fail to deploy 5. Confirm the first instance has NUMA affinity with its SR-IOV port 6. Validate xml description of sr-iov interface is correct for first guest """ port_a = self._create_port_from_vnic_type( net=self.network, vnic_type=self.vnic_type, numa_affinity_policy='required') port_b = self._create_port_from_vnic_type( net=self.network, vnic_type=self.vnic_type, numa_affinity_policy='required') self._required_test_procedure( self.flavor, port_a, port_b, self.image_ref) def test_sriov_affinity_port_policy_precedence_flavor(self): """Validate port policy precedence over flavor NUMA affinity policy 1. Create a flavor with required NUMA policy and hw:cpu_policy=dedicated. The first flavor vcpu size will be equal to the number of dedicated PCPUs of the NUMA Node with affinity to the physnet. This should result in any deployed instance using this flavor 'filling' the NUMA Node completely. 2. Create two ports that have the preferred numa affinity policy. 3. Launch an instance using the flavor and the first port. Determine the host it lands on. 4. Launch a second instance with the same flavor and the second port and target it to the same host as the first instance. 4. Validate both instances are deployed 5. Confirm the first instance has NUMA affinity with its SR-IOV port 6. Validate xml description of SR-IOV interface is correct for both instances """ required_flavor = self.create_flavor( vcpus=self.dedicated_cpus_per_numa, extra_specs=self.required) port_a = self._create_port_from_vnic_type( net=self.network, vnic_type=self.vnic_type, numa_affinity_policy='preferred') port_b = self._create_port_from_vnic_type( net=self.network, vnic_type=self.vnic_type, numa_affinity_policy='preferred') self._preferred_test_procedure( required_flavor, port_a, port_b, self.image_ref) def test_sriov_affinity_port_policy_precedence_image(self): """Validate port policy precedence over image NUMA affinity policy 1. Create a flavor with hw:cpu_policy=dedicated and a vCPU size will be equal to the number of dedicated PCPUs of the NUMA Node with affinity to the physnet. This should result in any deployed instance using this flavor 'filling' the NUMA Node completely. 2. Create an image with required numa affinity policy 3. Create two ports that have the preferred numa affinity policy. 4. Launch an instance using the flavor, image, and the first port. Determine the host it lands on. 5. Launch a second instance with the same flavor and the second port and target it to the same host as the first instance. 6. Validate both instances are deployed and first guest has affinity with attach SR-IOV port. 7. Validate xml description of SR-IOV interface is correct for both guests """ image_id = self.copy_default_image( hw_pci_numa_affinity_policy='required') port_a = self._create_port_from_vnic_type( net=self.network, vnic_type=self.vnic_type, numa_affinity_policy='preferred') port_b = self._create_port_from_vnic_type( net=self.network, vnic_type=self.vnic_type, numa_affinity_policy='preferred') self._preferred_test_procedure( self.flavor, port_a, port_b, image_id) class SRIOVNumaAffinityWithSocketPolicy(SRIOVNumaAffinity): socket_specs = {'hw:cpu_policy': 'dedicated', 'hw:pci_numa_affinity_policy': 'socket'} @classmethod def skip_checks(cls): super(SRIOVNumaAffinity, cls).skip_checks() if getattr(CONF.whitebox_hardware, 'socket_topology', None) is None: raise cls.skipException('Requires socket_topology parameter ' 'to be set in order to execute test ' 'cases.') if getattr(CONF.whitebox_hardware, 'socket_affinity', None) is None: raise cls.skipException('Requires socket_affinity parameter ' 'to be set in order to execute test ' 'cases.') def _get_cpu_ids_with_socket_affinity(self, host_dedicated_set): pcpu_ids_with_socket_affinity = [] socket_affinity = str(CONF.whitebox_hardware.socket_affinity) numa_nodes = \ CONF.whitebox_hardware.socket_topology[socket_affinity] for numa in numa_nodes: pcpu_ids_with_socket_affinity += \ self._get_dedicated_cpus_from_numa_node( str(numa), host_dedicated_set) return pcpu_ids_with_socket_affinity def _socket_test_procedure(self, flavor, port_a, port_b, image_id): server_a = self.create_test_server( flavor=flavor['id'], networks=[{'port': port_a['port']['id']}], image_id=image_id, wait_until='ACTIVE' ) # Determine the host that guest A lands on and use that information # to force guest B to land on the same host host = self.get_host_for_server(server_a['id']) server_b = self.create_test_server( flavor=flavor['id'], networks=[{'port': port_b['port']['id']}], scheduler_hints={'same_host': server_a['id']}, image_id=image_id, wait_until='ACTIVE' ) # Determine the pCPUs that have affinity with the host's SR-IOV port. # Then confirm the first instance's pCPUs match the pCPUs from the # NUMA node with affinity to the SR-IOV port. host_sm = clients.NovaServiceManager(host, 'nova-compute', self.os_admin.services_client) cpu_dedicated_set = host_sm.get_cpu_dedicated_set() cpu_pins_a = self.get_pinning_as_set(server_a['id']) pcpus_with_affinity = self._get_dedicated_cpus_from_numa_node( self.affinity_node, cpu_dedicated_set) self.assertEqual( cpu_pins_a, pcpus_with_affinity, 'Expected pCPUs for server A, ' 'id: %s to be equal to %s but instead are %s' % (server_a['id'], pcpus_with_affinity, cpu_pins_a)) # Find the pinned pCPUs used by server B. Confirm that while they will # not be comprised of pCPUs from the NUMA with affinity to the SR-IOV # port, it still has pCPUs from the same socket. cpu_pins_b = self.get_pinning_as_set(server_b['id']) pcpus_on_socket = self._get_cpu_ids_with_socket_affinity( cpu_dedicated_set) self.assertTrue( cpu_pins_b.issubset(set(pcpus_on_socket)), 'Expected pCPUs for server B id: %s to be subset of %s but ' 'instead are %s' % (server_b['id'], pcpus_on_socket, cpu_pins_b)) self.assertTrue( cpu_pins_a.isdisjoint(cpu_pins_b), 'Cpus %s for server A %s are not disjointed with Cpus %s of ' 'server B %s' % (cpu_pins_a, server_a['id'], cpu_pins_b, server_b['id'])) # Validate servers A and B have correct sr-iov interface # information in the xml. Its type and vlan should be accurate. net_vlan = CONF.whitebox_hardware.sriov_vlan_id for server, port in zip([server_a, server_b], [port_a, port_b]): interface_xml_element = self._get_xml_interface_device( server['id'], port['port']['id'] ) self._validate_port_xml_vlan_tag( interface_xml_element, net_vlan) def test_sriov_affinity_socket_policy(self): socket_flavor = self.create_flavor( vcpus=self.dedicated_cpus_per_numa, extra_specs=self.socket_specs) port_a = self._create_port_from_vnic_type( net=self.network, vnic_type=self.vnic_type) port_b = self._create_port_from_vnic_type( net=self.network, vnic_type=self.vnic_type) self._socket_test_procedure( socket_flavor, port_a, port_b, self.image_ref) class SRIOVMigration(SRIOVBase): # Test utilizes the optional host parameter for server creation introduced # in 2.74 to schedule the guest to a specific compute host. This allows the # test to dictate specific target hosts as the test progresses. min_microversion = '2.74' def setUp(self): super(SRIOVMigration, self).setUp() self.vlan_id = \ CONF.whitebox_hardware.sriov_vlan_id self.physical_net = CONF.whitebox_hardware.sriov_physnet self.network = self._create_net_from_physical_network( self.vlan_id, self.physical_net) self._create_subnet(self.network['network']['id']) def _validate_pci_allocation(self, pci_device_status_regex): """Check PCI allocation count and confirm it updates to 1""" start_time = int(time.time()) timeout = self.os_admin.services_client.build_timeout while int(time.time()) - start_time <= timeout: pci_allocated_count = self._get_pci_status_count( pci_device_status_regex) if pci_allocated_count == 1: return time.sleep(self.os_admin.services_client.build_interval + 1) raise lib_exc.TimeoutException( pci_allocated_count, 1, 'Total allocated pci devices should be 1 ' 'but instead is %s' % pci_allocated_count) @classmethod def skip_checks(cls): super(SRIOVMigration, cls).skip_checks() if (CONF.compute.min_compute_nodes < 2): raise cls.skipException('Need 2 or more compute nodes.') def _base_test_live_migration(self, vnic_type): """Parent test class that perform sr-iov live migration :param vnic_type: str, vnic_type to use when creating sr-iov port """ if CONF.compute_feature_enabled.sriov_hotplug: pci_device_status_regex = 'allocated' else: pci_device_status_regex = 'allocated|claimed' flavor = self.create_flavor() port = self._create_port_from_vnic_type( net=self.network, vnic_type=vnic_type ) server = self.create_test_server( flavor=flavor['id'], networks=[{'port': port['port']['id']}], wait_until='ACTIVE') host = self.get_host_for_server(server['id']) # Live migrate the server self.live_migrate(self.os_admin, server['id'], 'ACTIVE') # Search the instace's XML for the SR-IOV network device element based # on the mac address and binding:vnic_type from port info interface_xml_element = self._get_xml_interface_device( server['id'], port['port']['id'], ) # Validate the vlan tag persisted in instance's XML after migration self._validate_port_xml_vlan_tag(interface_xml_element, self.vlan_id) # Confirm dev_type, allocation status, and pci address information are # correct in pci_devices table of openstack DB self._verify_neutron_port_binding( server['id'], port['port']['id'] ) # Validate the total allocation of pci devices is one and only one # after instance migration self._validate_pci_allocation(pci_device_status_regex) if CONF.compute_feature_enabled.live_migrate_back_and_forth: # Migrate server back to the original host self.live_migrate(self.os_admin, server['id'], 'ACTIVE', target_host=host) # Again find the instance's network device element based on the # mac address and binding:vnic_type from the port info provided by # ports client interface_xml_element = self._get_xml_interface_device( server['id'], port['port']['id'], ) # Confirm vlan tag in interface XML, dev_type, allocation status, # and pci address information are correct in pci_devices table of # openstack DB after second migration self._validate_port_xml_vlan_tag( interface_xml_element, self.vlan_id ) self._verify_neutron_port_binding( server['id'], port['port']['id'] ) # Confirm total port allocations still remains one after final # migration self._validate_pci_allocation(pci_device_status_regex) def test_sriov_direct_live_migration(self): """Verify sriov live migration using direct type ports """ self._base_test_live_migration(vnic_type='direct') def test_sriov_macvtap_live_migration(self): """Verify sriov live migration using macvtap type ports """ self._base_test_live_migration(vnic_type='macvtap') class SRIOVAttachAndDetach(SRIOVBase): def setUp(self): super(SRIOVAttachAndDetach, self).setUp() self.vlan_id = \ CONF.whitebox_hardware.sriov_vlan_id self.physical_net = CONF.whitebox_hardware.sriov_physnet self.network = self._create_net_from_physical_network( self.vlan_id, self.physical_net) self._create_subnet(self.network['network']['id']) @classmethod def skip_checks(cls): super(SRIOVAttachAndDetach, cls).skip_checks() if not CONF.compute_feature_enabled.sriov_hotplug: raise cls.skipException('Deployment requires support for SR-IOV ' 'NIC hot-plugging') if (CONF.whitebox_hardware.sriov_nic_vendor_id is None): msg = "CONF.whitebox_hardware.sriov_nic_vendor_id needs to be set." raise cls.skipException(msg) @classmethod def setup_credentials(cls): cls.prepare_instance_network() super(SRIOVAttachAndDetach, cls).setup_credentials() def wait_for_port_detach(self, port_id): """Waits for the port's device_id to be unset. :param port_id: The id of the port being detached. :returns: The final port dict from the show_port response. """ port = self.os_primary.ports_client.show_port(port_id)['port'] device_id = port['device_id'] start = int(time.time()) # NOTE(mriedem): Nova updates the port's device_id to '' rather than # None, but it's not contractual so handle Falsey either way. while device_id: time.sleep(self.build_interval) port = self.os_primary.ports_client.show_port(port_id)['port'] device_id = port['device_id'] timed_out = int(time.time()) - start >= self.build_timeout if device_id and timed_out: message = ('Port %s failed to detach (device_id %s) within ' 'the required time (%s s).' % (port_id, device_id, self.build_timeout)) raise lib_exc.TimeoutException(message) return port def _check_device_in_guest(self, linux_client, vendor_id, product_id): """Check attached SR-IOV NIC is present in guest """ cmd = "lspci -nn | grep {0}:{1} | wc -l".format( vendor_id, product_id) sys_out = linux_client.exec_command(cmd) self.assertIsNotNone( sys_out, 'Unable to find vendor id %s when checking the guest' % 'sriov vendor id') self.assertEqual( 1, int(sys_out), 'Should only find 1 pci device ' 'device in guest but instead found %s' % int(sys_out)) def _create_ssh_client(self, server, validation_resources): """Create an ssh client to execute commands on the guest instance :param server: the ssh client will be setup to interface with the provided server instance :param valdiation_resources: necessary validation information to setup an ssh session :return linux_client: the ssh client that allows for guest command execution """ linux_client = remote_client.RemoteClient( self.get_server_ip(server, validation_resources), self.image_ssh_user, self.image_ssh_password, validation_resources['keypair']['private_key'], server=server, servers_client=self.servers_client) linux_client.validate_authentication() return linux_client def create_server_and_ssh(self): """Create a validateable instance based on provided flavor :param flavor: dict, attributes describing flavor :param validation_resources: dict, parameters necessary to setup ssh client and validate the guest """ validation_resources = self.get_test_validation_resources( self.os_primary) server = self.create_test_server( validatable=True, validation_resources=validation_resources, wait_until='ACTIVE') linux_client = self._create_ssh_client(server, validation_resources) return (server, linux_client) def _validate_port_data_after_attach(self, pre_attached_port, after_attached): """Compare the port data before and after being attached to a guest :param pre_attached_port: dict, the current interface data for attached port :param after_attached: dict, original port data when first created """ net_id = self.network.get('network').get('id') port_id = pre_attached_port['port']['id'] port_ip_addr = pre_attached_port['port']['fixed_ips'][0]['ip_address'] port_mac_addr = pre_attached_port['port']['mac_address'] self.assertEqual(after_attached['port_id'], port_id) self.assertEqual(after_attached['net_id'], net_id) self.assertEqual( after_attached['fixed_ips'][0]['ip_address'], port_ip_addr) # When using a physical SR-IOV port the originally created port's # mac address will be updated to the physical device's mac address # on the host. Original port mac should no longer match updated # host mac if pre_attached_port['port']['binding:vnic_type'] == 'direct-physical': self.assertNotEqual(after_attached['mac_addr'], port_mac_addr) else: # When not using physical, the port's mac should remain # consistent self.assertEqual(after_attached['mac_addr'], port_mac_addr) def _base_test_attach_and_detach_sriov_port(self, vnic_type): """Validate sr-iov interface can be attached/detached with guests 1. Create and sr-iov port based on the provided vnic_type 2. Launch two guests with UC access via SSH 3. Iterate over both guests doing the following steps: 3a. Attach the interface to the guest 3b. Check the return information about the attached interface matches the expected port information 3c. Confirm port information is correct in guest XML. 3d. Verify NIC is present from within the guest by checking for a pci device with matching vendor/device id 3e. Confirm the pci address associated with the port matches what is in Nova DB. 3f. Detach the interface and wait for it to be available """ # Gather SR-IOV network vlan, create two guests, and create an SR-IOV # port based on the provided vnic_type servers = [self.create_server_and_ssh(), self.create_server_and_ssh()] port = self._create_port_from_vnic_type( net=self.network, vnic_type=vnic_type ) if vnic_type == 'macvtap': vendor_id = CONF.whitebox_hardware.macvtap_virtio_vendor_id product_id = CONF.whitebox_hardware.macvtap_virtio_product_id else: vendor_id = CONF.whitebox_hardware.sriov_nic_vendor_id product_id = CONF.whitebox_hardware.sriov_vf_product_id # Iterate over both servers, attaching the sr-iov port, checking the # the attach was successful from an API, XML, and guest level and # then detach the interface from the guest for server, linux_client in servers: iface = self.interfaces_client.create_interface( server['id'], port_id=port['port']['id'])['interfaceAttachment'] # Validate the original port information with what is currently # report after the attach self._validate_port_data_after_attach(port, iface) interface_xml_element = self._get_xml_interface_device( server['id'], port['port']['id'] ) # Confirm mac address for the port in the domain XML match the # mac address reported for the port self.assertEqual( iface['mac_addr'], interface_xml_element.find('mac').attrib.get('address')) # Verify the port's VLAN tag is present in the XML self._validate_port_xml_vlan_tag(interface_xml_element, self.vlan_id) # Confirm the vendor and vf product id are present in the guest self._check_device_in_guest(linux_client, vendor_id, product_id) # Validate the port mappings are correct in the nova DB self._verify_neutron_port_binding( server['id'], port['port']['id'] ) self.interfaces_client.delete_interface( server['id'], port['port']['id']) self.wait_for_port_detach(port['port']['id']) @testtools.skipUnless(CONF.whitebox_hardware.sriov_vf_product_id, "Requires sriov NIC's VF Prodcut ID") @testtools.skipUnless(CONF.whitebox_hardware.sriov_nic_vendor_id, "Requires sriov NIC's VF ID") def test_sriov_direct_attach_detach_port(self): """Verify sriov direct port can be attached/detached from live guest """ self._base_test_attach_and_detach_sriov_port(vnic_type='direct') @testtools.skipUnless(CONF.whitebox_hardware.macvtap_virtio_product_id, "Requires sriov NIC's virtio product ID") @testtools.skipUnless(CONF.whitebox_hardware.macvtap_virtio_vendor_id, "Requires sriov NIC's virtio vendor ID") def test_sriov_macvtap_attach_detach_port(self): """Verify sriov macvtap port can be attached/detached from live guest """ self._base_test_attach_and_detach_sriov_port(vnic_type='macvtap') @testtools.skipUnless(CONF.whitebox_hardware.sriov_pf_product_id, "Requires sriov NIC's PF ID") def test_sriov_direct_physical_attach_detach_port(self): """Verify sriov direct-physical port attached/detached from guest 1. Create and sr-iov port based on the provided vnic_type 2. Launch two guests accessible by the UC via SSH. Test creates two guests to validate the same port can be attached/removed from multiple guests 3. Iterate over both guests doing the following steps: 3a. Attach the interface to the guest 3b. Check the return information about the attached interface matches the expected port information 3c. Verify NIC is present from within the guest by checking for a pci device with matching vendor/device id 3d. Confirm the pci address associated with the port matches what is in Nova DB. 3e. Detach the interface and wait for it to be available """ # Create two guests and create an SR-IOV port with vnic_type # direct-physical servers = [self.create_server_and_ssh(), self.create_server_and_ssh()] port = self._create_port_from_vnic_type( net=self.network, vnic_type='direct-physical' ) # Iterate over both servers, attaching the sr-iov port, checking the # the attach was successful from an API, XML, and guest level and # then detach the interface from the guest for server, linux_client in servers: iface = self.interfaces_client.create_interface( server['id'], port_id=port['port']['id'])['interfaceAttachment'] # Confirm the port information currently reported after the attach # match the original information for the port self._validate_port_data_after_attach(port, iface) # Validate the PCI address of the physical interface is present # for the host dev XML element in the guest host_dev_xml = self._get_xml_pf_device(server['id']) self._validate_pf_pci_address_in_xml( port['port']['id'], host_dev_xml) # Verify the the interface's vendor ID and the physical device ID # are present in the guest self._check_device_in_guest( linux_client, CONF.whitebox_hardware.sriov_nic_vendor_id, CONF.whitebox_hardware.sriov_pf_product_id) # Confirm the nova db mappings for the port are correct self._verify_neutron_port_binding( server['id'], port['port']['id'] ) self.interfaces_client.delete_interface( server['id'], port['port']['id']) self.wait_for_port_detach(port['port']['id'])