Fix dvr vip failover tests

On podified environments we can't use local IP for establishing
TCP connection via virtual IP (VIP) since tempest is running on
a node that can be a gateway node and logic of the test will be
affected. Therefore a separate VM is spawned on the external
network and it is used as a proxy host for establishing TCP
connection to the FIP.
Also introduced a function ensure_external_network_is_shared()
and simplified some tests from other files that had the same
code.

Change-Id: I099ec34299debcd43b8ce485656e3ea6d7a95f51

whitebox_neutron_tempest_plugin/config.py

@@ -101,7 +101,8 @@ WhiteboxNeutronPluginOptions = [
                 help='Specifies whether the OSP setup under test has been '
                      'configured with BGP functionality or not'),
     cfg.StrOpt('bgp_agent_config',
-               default='/etc/ovn-bgp-agent/bgp-agent.conf',
+               default='/var/lib/config-data/ansible-generated/ovn-bgp-agent'
+                       '/etc/ovn-bgp-agent/bgp-agent.conf',
                help='Path to ovn-bgp-agent config file'),
     cfg.IntOpt('sriov_pfs_per_host',
                default=1,

whitebox_neutron_tempest_plugin/tests/scenario/base.py

@@ -75,6 +75,8 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase):
         if WB_CONF.openstack_type == 'podified':
             cls.neutron_api_prefix = '{} rsh {} '.format(
                 cls.OC, cls.get_pods_of_service()[0])
+        cls.external_network = cls.os_admin.network_client.show_network(
+            CONF.network.public_network_id)['network']
 
     @classmethod
     def setup_proxy_host(cls):
@@ -205,6 +207,14 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase):
                     fip['floating_ip_address']):
                 return fp
 
+    def ensure_external_network_is_shared(self):
+        if not self.external_network['shared']:
+            self.addClassResourceCleanup(
+                self.os_admin.network_client.update_network,
+                self.external_network['id'], shared=False)
+            self.os_admin.network_client.update_network(
+                self.external_network['id'], shared=True)
+
     @classmethod
     def get_podified_nodes_data(cls):
 
@@ -738,14 +748,8 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase):
                 router = self.create_router_by_client()
 
         if topology == 'external' or topology == 'north-south':
-            external_network = self.client.show_network(
+            self.ensure_external_network_is_shared()
-                CONF.network.public_network_id)['network']
+            src_network = self.external_network
-            if not external_network['shared']:
-                self.addCleanup(self.os_admin.network_client.update_network,
-                                external_network['id'], shared=False)
-                self.os_admin.network_client.update_network(
-                    external_network['id'], shared=True)
-            src_network = external_network
         else:
             src_network = _create_local_network()
 

whitebox_neutron_tempest_plugin/tests/scenario/test_dvr_ovn.py

@@ -32,6 +32,7 @@ from tempest.lib.common.utils import test_utils
 from tempest.lib import decorators
 from tempest.lib import exceptions as lib_exceptions
 
+from whitebox_neutron_tempest_plugin.common import constants as local_constants
 from whitebox_neutron_tempest_plugin.common import utils as local_utils
 from whitebox_neutron_tempest_plugin.tests.scenario import base
 
@@ -51,9 +52,10 @@ class OvnDvrBase(base.TrafficFlowTest, base.BaseTempestTestCaseOvn):
         if len(cls.nodes) < 2:
             raise cls.skipException(
                 "The tests require environment with at least 2 nodes")
+        cls.bgp_expose_tenant_networks = False
         for node in cls.nodes:
             if WB_CONF.openstack_type == 'devstack':
-                if node['is_controller'] is not True:
+                if not node['is_controller']:
                     continue
                 cls.check_service_setting(
                     host=node, service='',
@@ -67,19 +69,28 @@ class OvnDvrBase(base.TrafficFlowTest, base.BaseTempestTestCaseOvn):
                     host=node, service='',
                     config_files=[WB_CONF.ml2_plugin_config],
                     section='ovn', param='enable_distributed_floating_ip')
-                # (rsafrono) checks for bgp are probably suitable for tripleo
+                # TODO(rsafrono) add code that defines
-                # and should be adjusted for devstack and podified cases
+                # cls.bgp_expose_tenant_networks on devstack
-                cls.bgp_expose_tenant_networks = (
+                # in case such bgp environment will be ever created
-                    WB_CONF.bgp and cls.check_service_setting(
-                        host=node, service='ovn_bgp_agent',
-                        config_files=[WB_CONF.bgp_agent_config],
-                        param='expose_tenant_networks', skip_if_fails=False))
         if WB_CONF.openstack_type == 'podified':
             config_files = cls.get_configs_of_service()
             cls.check_service_setting(
                 {'client': cls.proxy_host_client},
                 config_files=config_files, section='ovn',
                 param='enable_distributed_floating_ip', msg=msg)
+            if WB_CONF.bgp:
+                for node in cls.nodes:
+                    if node['is_networker'] and not node['is_controller']:
+                        output = node['client'].exec_command(
+                            "crudini --get {} DEFAULT "
+                            "expose_tenant_networks || true".format(
+                                WB_CONF.bgp_agent_config)).strip()
+                        if output:
+                            cls.bgp_expose_tenant_networks = eval(
+                                output.capitalize())
+                        break
+                    else:
+                        continue
 
     def _setup(self, router=None):
         router = self.create_router_by_client()
@@ -732,8 +743,6 @@ class OvnDvrTest(OvnDvrBase):
             ssh_client=test_server_client)
 
 
-@testtools.skipIf(WB_CONF.openstack_type == 'podified',
-                  'Not yet adapted for podified environment')
 class OvnDvrAdvancedTest(base.BaseTempestTestCaseAdvanced,
                          OvnDvrBase):
 
@@ -789,8 +798,7 @@ class OvnDvrAdvancedTest(base.BaseTempestTestCaseAdvanced,
     @staticmethod
     def _start_tcp_connection(ssh_client):
         ssh_process = ssh_client.open_session()
-        ip_address = WB_CONF.global_ip_address
+        cmd = ('ping {}'.format(local_constants.GLOBAL_IP))
-        cmd = ('ping {}'.format(ip_address))
         ssh_process.exec_command(cmd)
 
     def _failover_vip(
@@ -814,7 +822,7 @@ class OvnDvrAdvancedTest(base.BaseTempestTestCaseAdvanced,
                 if attr[0] == 'RTA_PREFSRC':
                     return attr[1]
 
-    def _get_filters(self, local_ip, vip_mac, vip_ip):
+    def _get_filters(self, src_ip, vip_mac, vip_ip):
         if self.external_network['provider:network_type'] == 'vlan':
             filters = 'vlan {} and '.format(
                 self.external_network['provider:segmentation_id'])
@@ -823,10 +831,10 @@ class OvnDvrAdvancedTest(base.BaseTempestTestCaseAdvanced,
 
         if not WB_CONF.bgp and vip_mac is not None:
             filters += 'ether host {} and dst host {}'.format(
-                vip_mac, local_ip)
+                vip_mac, src_ip)
         else:
             filters += 'src host {} and dst host {}'.format(
-                vip_ip, local_ip)
+                vip_ip, src_ip)
 
         filters += ' and tcp src port 22'
         return filters
@@ -844,14 +852,40 @@ class OvnDvrAdvancedTest(base.BaseTempestTestCaseAdvanced,
             ','.join(self.expected_routing_nodes)))
         actual_routing_nodes = [node['name']
                                 for node in self.nodes if
-                                not node['capture'].is_empty()]
+                                (node.get('capture') and
+                                 not node['capture'].is_empty())]
         LOG.debug('Actual routing nodes: {}'.format(
             ','.join(actual_routing_nodes)))
         self.assertCountEqual(
             self.expected_routing_nodes, actual_routing_nodes)
 
+    def create_ext_vm(self, ip):
+        # On podified environments sending traffic from local IP address
+        # affects test results since tempest is running on a node that
+        # can be a gateway node. Therefore an additional VM on external
+        # network is used as a proxy host.
+        # Note: we can not use ansible controller as a proxy host here
+        # since on some environments the ansible controller does not have
+        # access to the external network.
+        ext_vm = self._create_server(
+            network=self.external_network,
+            create_floating_ip=False)
+        ext_vm_ip = ext_vm['port']['fixed_ips'][0]['ip_address']
+        ext_vm_ssh_client = ssh.Client(
+            ext_vm_ip,
+            self.username, pkey=self.keypair['private_key'])
+        vip_ssh_client = ssh.Client(
+            ip, self.username,
+            pkey=self.keypair['private_key'],
+            proxy_client=ext_vm_ssh_client)
+        ext_vm_host = self.get_host_for_server(
+            ext_vm['server']['id'])
+        if ext_vm_host not in self.expected_routing_nodes:
+            self.expected_routing_nodes.append(ext_vm_host)
+        return vip_ssh_client, ext_vm_ip
+
     @decorators.idempotent_id('509d1432-3879-40d4-9378-e6a0d972a292')
-    def test_dvr_vip_failover(self):
+    def test_dvr_vip_failover_basic(self):
         """Test DVR during VIP failover using a tenant network and FIPs
 
         The test checks that during VIP failover on DVR environment traffic
@@ -893,6 +927,7 @@ class OvnDvrAdvancedTest(base.BaseTempestTestCaseAdvanced,
         self.client.update_port(
             vm2_port['id'], allowed_address_pairs=[{"ip_address": vip_ip}])
         vip_fip = self.create_floatingip(port=vip_port)
+        vip_fip_ip = vip_fip['floating_ip_address']
 
         self.expected_routing_nodes = []
         for vm in [vm1, vm2]:
@@ -903,24 +938,13 @@ class OvnDvrAdvancedTest(base.BaseTempestTestCaseAdvanced,
         vip_ssh_client = ssh.Client(
             vip_fip['floating_ip_address'], self.username,
             pkey=self.keypair['private_key'])
-
         vip_fip_mac = self.get_fip_port_details(vip_fip)['mac_address']
 
         # Let's set vip first on vm1 and then will do the vip failover to vm2
         self._configure_ip_address_in_vm(vm1['ssh_client'], nic, vip_ip)
-
+        self.ensure_external_network_is_shared()
-        self.external_network = self.os_admin.network_client.show_network(
+        vip_ssh_client, src_ip = self.create_ext_vm(vip_fip_ip)
-            CONF.network.public_network_id)['network']
+        filters = self._get_filters(src_ip, vip_fip_mac, vip_fip_ip)
-
-        local_ip = self.get_local_ssh_client(self.external_network).host
-
-        if local_ip is None:
-            local_ip = self._get_src_ip_from_route(
-                vip_fip['floating_ip_address'])
-
-        filters = self._get_filters(
-            local_ip, vip_fip_mac, vip_fip['floating_ip_address'])
-
         self._capture_and_test_failover_vip(
             filters, vm1, vm2, nic, vip_ip, vip_ssh_client)
 
@@ -951,9 +975,9 @@ class OvnDvrAdvancedTest(base.BaseTempestTestCaseAdvanced,
                     'enable_snat': False})
 
             # generate updated tcpdump filters
-            local_ip = self._get_src_ip_from_route(vip_ip)
+            src_ip = self._get_src_ip_from_route(vip_ip)
             filters = self._get_filters(
-                local_ip, vip_port['mac_address'], vip_ip)
+                src_ip, vip_port['mac_address'], vip_ip)
 
             # calculate new expected_routing_nodes = chassis hosting the router
             # gateway is the only expected match (traffic from that
@@ -1005,9 +1029,6 @@ class OvnDvrAdvancedTest(base.BaseTempestTestCaseAdvanced,
         vip_ip = vip_port['fixed_ips'][0]['ip_address']
         aap = [{"ip_address": vip_ip}]
 
-        vip_ssh_client = ssh.Client(
-            vip_ip, self.username, pkey=self.keypair['private_key'])
-
         self.expected_routing_nodes = []
         for vm in [vm1, vm2]:
             nic = local_utils.get_default_interface(vm['ssh_client'])
@@ -1041,16 +1062,10 @@ class OvnDvrAdvancedTest(base.BaseTempestTestCaseAdvanced,
 
         # Let's set vip first on vm1 and then will do the vip failover to vm2
         self._configure_ip_address_in_vm(vm1['ssh_client'], nic, vip_ip)
-
+        vip_ssh_client, src_ip = self.create_ext_vm(vip_ip)
-        self.external_network = self.os_admin.network_client.show_network(
-            CONF.network.public_network_id)['network']
-
-        local_ip = self.get_local_ssh_client(self.external_network).host
-        if local_ip is None:
-            local_ip = self._get_src_ip_from_route(vip_ip)
         # vip_mac is set to None because the filter should be based on the
         # vip_ip instead in this case, where no FIPs are used
-        filters = self._get_filters(local_ip, None, vip_ip)
+        filters = self._get_filters(src_ip, None, vip_ip)
 
         self._capture_and_test_failover_vip(
             filters, vm1, vm2, nic, vip_ip, vip_ssh_client)
@@ -1062,9 +1077,9 @@ class OvnDvrAdvancedTest(base.BaseTempestTestCaseAdvanced,
         # Let's set vip first on vm2 and then will do the vip failover to vm1
         if skip_ipv6 is False:
             self._configure_ip_address_in_vm(vm2['ssh_client'], nic, vip_ipv6)
-            local_ip = self._get_src_ip_from_route(vip_ipv6)
+            src_ip = self._get_src_ip_from_route(vip_ipv6)
             # vip_mac is set to None because the filter should be based on the
             # vip_ip instead in this case, where no FIPs are used
-            filters = self._get_filters(local_ip, None, vip_ipv6)
+            filters = self._get_filters(src_ip, None, vip_ipv6)
             self._capture_and_test_failover_vip(
                 filters, vm2, vm1, nic, vip_ipv6, vip_ssh_clientv6)

whitebox_neutron_tempest_plugin/tests/scenario/test_mtu.py

@@ -141,13 +141,7 @@ class GatewayMtuTest(base.TrafficFlowTest, base.BaseTempestTestCaseOvn):
         subnet = self.create_subnet(self.network)
         self.router = self.create_router_by_client()
         self.create_router_interface(self.router['id'], subnet['id'])
-        self.external_network = self.client.show_network(
+        self.ensure_external_network_is_shared()
-            CONF.network.public_network_id)['network']
-        if not self.external_network['shared']:
-            self.addCleanup(self.os_admin.network_client.update_network,
-                            self.external_network['id'], shared=False)
-            self.os_admin.network_client.update_network(
-                self.external_network['id'], shared=True)
         self.external_mtu = self.external_network['mtu']
         self.internal_mtu = self.network['mtu']
         if int(self.external_mtu) >= int(self.internal_mtu):

whitebox_neutron_tempest_plugin/tests/scenario/test_multicast.py

@@ -312,11 +312,7 @@ class BaseMulticastTest(object):
         # Note, for now we support port_type other than 'normal' only
         # when using SR-IOV environement and therefore only external network
         if topology == 'external' or topology == 'north-south':
-            if not self.external_network['shared']:
+            self.ensure_external_network_is_shared()
-                self.addCleanup(self.os_admin.network_client.update_network,
-                                self.external_network['id'], shared=False)
-                self.os_admin.network_client.update_network(
-                    self.external_network['id'], shared=True)
             sender = self._create_server_for_topology(
                 network_id=self.external_network['id'],
                 port_type=port_type)

whitebox_neutron_tempest_plugin/tests/scenario/test_qos.py

@@ -990,13 +990,7 @@ class QosTestExternalNetwork(QosBaseTest):
     """
     def setUp(self):
         super(QosTestExternalNetwork, self).setUp()
-        self.external_network = self.client.show_network(
+        self.ensure_external_network_is_shared()
-            CONF.network.public_network_id)['network']
-        if not self.external_network['shared']:
-            self.addCleanup(self.os_admin.network_client.update_network,
-                            self.external_network['id'], shared=False)
-            self.os_admin.network_client.update_network(
-                self.external_network['id'], shared=True)
         ext_vm = self._create_server(
             network=self.external_network,
             create_floating_ip=False)