From edf3cba046fd960bfbd6a17890ad995fd2c7d955 Mon Sep 17 00:00:00 2001 From: Slawek Kaplonski Date: Wed, 21 Apr 2021 10:34:02 +0200 Subject: [PATCH] Wait for HA router to be active on at least one agent Due to related bug it may happend that HA router will become active on one of the L3 agents after few minutes since it was created. And during that time when it's in "standby" mode, VM can be spawned, it will try to get metadata but haproxy in the router namespace is still not running so there will be no metadata available. That will end up with SSH authentication error as there will be no proper SSH key configured in the VM. Originally this patch was intended just as a workaround for the related bug but I think it's worth to keep it even if that bug will be solved on the Neutron's side as it may give us more clear information about the real problem in the future if something similar will happen and HA router will not be active on any of the L3 agents. Related-Bug: #1923633 Change-Id: I8c8b7c11c63ffeee4f776695f32ae686793299b3 --- neutron_tempest_plugin/scenario/base.py | 21 +++++++++++++++++++ neutron_tempest_plugin/scenario/test_basic.py | 2 +- .../scenario/test_connectivity.py | 3 +++ .../scenario/test_internal_dns.py | 1 + .../scenario/test_multicast.py | 2 +- neutron_tempest_plugin/scenario/test_ports.py | 2 +- .../scenario/test_portsecurity.py | 2 +- 7 files changed, 29 insertions(+), 4 deletions(-) diff --git a/neutron_tempest_plugin/scenario/base.py b/neutron_tempest_plugin/scenario/base.py index 334d5438..c29585fb 100644 --- a/neutron_tempest_plugin/scenario/base.py +++ b/neutron_tempest_plugin/scenario/base.py @@ -206,8 +206,29 @@ class BaseTempestTestCase(base_api.BaseNetworkTest): else: router = cls.create_admin_router(**kwargs) LOG.debug("Created router %s", router['name']) + cls._wait_for_router_ha_active(router['id']) return router + @classmethod + def _wait_for_router_ha_active(cls, router_id): + router = cls.os_admin.network_client.show_router(router_id)['router'] + if not router.get('ha'): + return + + def _router_active_on_l3_agent(): + agents = cls.os_admin.network_client.list_l3_agents_hosting_router( + router_id)['agents'] + return "active" in [agent['ha_state'] for agent in agents] + + error_msg = ( + "Router %s is not active on any of the L3 agents" % router_id) + # NOTE(slaweq): timeout here should be lower for sure, but due to + # the bug https://launchpad.net/bugs/1923633 let's wait even 10 + # minutes until router will be active on some of the L3 agents + utils.wait_until_true(_router_active_on_l3_agent, + timeout=600, sleep=5, + exception=lib_exc.TimeoutException(error_msg)) + @classmethod def skip_if_no_extension_enabled_in_l3_agents(cls, extension): l3_agents = cls.os_admin.network_client.list_agents( diff --git a/neutron_tempest_plugin/scenario/test_basic.py b/neutron_tempest_plugin/scenario/test_basic.py index 38bc40b3..7583f88b 100644 --- a/neutron_tempest_plugin/scenario/test_basic.py +++ b/neutron_tempest_plugin/scenario/test_basic.py @@ -24,7 +24,7 @@ CONF = config.CONF class NetworkBasicTest(base.BaseTempestTestCase): - credentials = ['primary'] + credentials = ['primary', 'admin'] force_tenant_isolation = False # Default to ipv4. diff --git a/neutron_tempest_plugin/scenario/test_connectivity.py b/neutron_tempest_plugin/scenario/test_connectivity.py index 66c4789a..ca7d755f 100644 --- a/neutron_tempest_plugin/scenario/test_connectivity.py +++ b/neutron_tempest_plugin/scenario/test_connectivity.py @@ -88,6 +88,8 @@ class NetworkConnectivityTest(base.BaseTempestTestCase): ap2_rt = self.create_router( router_name=data_utils.rand_name("ap2_rt"), admin_state_up=True) + self._wait_for_router_ha_active(ap1_rt['id']) + self._wait_for_router_ha_active(ap2_rt['id']) ap1_internal_port = self.create_port( ap1_net, security_groups=[self.secgroup['id']]) @@ -140,6 +142,7 @@ class NetworkConnectivityTest(base.BaseTempestTestCase): router_name=data_utils.rand_name("east_west_traffic_router"), admin_state_up=True, external_network_id=CONF.network.public_network_id) + self._wait_for_router_ha_active(router['id']) internal_port_1 = self.create_port( net_1, security_groups=[self.secgroup['id']]) diff --git a/neutron_tempest_plugin/scenario/test_internal_dns.py b/neutron_tempest_plugin/scenario/test_internal_dns.py index 406af3d6..c0a2c04e 100644 --- a/neutron_tempest_plugin/scenario/test_internal_dns.py +++ b/neutron_tempest_plugin/scenario/test_internal_dns.py @@ -24,6 +24,7 @@ CONF = config.CONF class InternalDNSTest(base.BaseTempestTestCase): + credentials = ['primary', 'admin'] @utils.requires_ext(extension="dns-integration", service="network") @decorators.idempotent_id('988347de-07af-471a-abfa-65aea9f452a6') diff --git a/neutron_tempest_plugin/scenario/test_multicast.py b/neutron_tempest_plugin/scenario/test_multicast.py index b0da235b..7c1fd2d9 100644 --- a/neutron_tempest_plugin/scenario/test_multicast.py +++ b/neutron_tempest_plugin/scenario/test_multicast.py @@ -122,7 +122,7 @@ tcpdump -i %(interface)s host %(group)s -vvneA -s0 -l -c1 &> %(result_file)s & class BaseMulticastTest(object): - credentials = ['primary'] + credentials = ['primary', 'admin'] force_tenant_isolation = False # Import configuration options diff --git a/neutron_tempest_plugin/scenario/test_ports.py b/neutron_tempest_plugin/scenario/test_ports.py index b3aeb87d..fb2d2b08 100644 --- a/neutron_tempest_plugin/scenario/test_ports.py +++ b/neutron_tempest_plugin/scenario/test_ports.py @@ -28,7 +28,7 @@ CONF = config.CONF class PortsTest(base.BaseTempestTestCase): - credentials = ['primary'] + credentials = ['primary', 'admin'] @classmethod def resource_setup(cls): diff --git a/neutron_tempest_plugin/scenario/test_portsecurity.py b/neutron_tempest_plugin/scenario/test_portsecurity.py index 257627c6..c90db08c 100644 --- a/neutron_tempest_plugin/scenario/test_portsecurity.py +++ b/neutron_tempest_plugin/scenario/test_portsecurity.py @@ -21,7 +21,7 @@ CONF = config.CONF class PortSecurityTest(base.BaseTempestTestCase): - credentials = ['primary'] + credentials = ['primary', 'admin'] required_extensions = ['port-security'] @decorators.idempotent_id('61ab176e-d48b-42b7-b38a-1ba571ecc033')