Enable fullstack multinode tests, add L3 HA test exemplar

* Created a 'resources' subdir and moved all fixture files
  to it.
* Split ML2ConfigFixture to the server-side ml2 configuration
  fixture, and the OVS agent configuration fixture.
* Neutron process logs were using H:M:S format as their file name,
  but when starting multiple agents of the same type my machine
  was fast enough to do that in the same second so that different
  processes were outputting to the same log file. No good!
  Added ms to the log name format. I also changed the log time
  from UTC to local timezone.
* Renamed and moved 'FullstackFixture' to neutron/tests/fullstack/
  resources/environment.Environment
* Added a 'Host' abstraction that groups agents that report with
  the same 'host' value. Hosts may be interconnected by the
  environment via shared bridges.
* The 'Environment' class will accept global
  attributes (This will be later filled with stuff like tunneling,
  l2pop or other environment-level flags), and in this patch accepts
  a  list of host attributes (Configuration that may differ between
  hosts like the l3 agent mode [legacy, dvr, dvr_snat]).
* Made OVS agent and L3 agent fixtures expose their bridges
  so that I could interconnect them.
* Added a super simple L3 HA test to show that this entire thing
  works.

Change-Id: Ie64de9f35bd6ab7cbad494061613ecf5e0ccd806

doc/source/devref/fullstack_testing.rst

@@ -62,18 +62,15 @@ Short Term Goals
 ================
 
 * Multinode & Stability:
-    - Interconnect the internal and external bridges
     - Convert the L3 HA failover functional test to a full stack test
     - Write a test for DHCP HA / Multiple DHCP agents per network
 * Write DVR tests
-* Write L3 HA tests
+* Write additional L3 HA tests
 * Write a test that validates L3 HA + l2pop integration after
   https://bugs.launchpad.net/neutron/+bug/1365476 is fixed.
 * Write a test that validates DVR + L3 HA integration after
   https://bugs.launchpad.net/neutron/+bug/1365473 is fixed.
 
-None of these tasks currently have owners. Feel free to send patches!
-
 After these tests are merged, it should be fair to start asking contributors to
 add full stack tests when appropriate in the patches themselves and not after
 the fact as there will probably be something to copy/paste from.

neutron/tests/base.py

@@ -38,6 +38,7 @@ from neutron.agent.linux import external_process
 from neutron.callbacks import manager as registry_manager
 from neutron.callbacks import registry
 from neutron.common import config
+from neutron.common import constants
 from neutron.common import rpc as n_rpc
 from neutron.db import agentschedulers_db
 from neutron import manager
@@ -87,6 +88,11 @@ def get_rand_name(max_length=None, prefix='test'):
     return prefix + suffix
 
 
+def get_rand_device_name(prefix='test'):
+    return get_rand_name(
+        max_length=constants.DEVICE_NAME_MAX_LEN, prefix=prefix)
+
+
 def bool_from_env(key, strict=False, default=False):
     value = os.environ.get(key)
     return strutils.bool_from_string(value, strict=strict, default=default)

neutron/tests/common/net_helpers.py

@@ -43,6 +43,7 @@ BR_PREFIX = 'test-br'
 PORT_PREFIX = 'test-port'
 VETH0_PREFIX = 'test-veth0'
 VETH1_PREFIX = 'test-veth1'
+PATCH_PREFIX = 'patch'
 
 SS_SOURCE_PORT_PATTERN = re.compile(
     r'^.*\s+\d+\s+.*:(?P<port>\d+)\s+[0-9:].*')
@@ -55,11 +56,6 @@ CHILD_PROCESS_SLEEP = os.environ.get('OS_TEST_CHILD_PROCESS_SLEEP', 0.5)
 TRANSPORT_PROTOCOLS = (n_const.PROTO_NAME_TCP, n_const.PROTO_NAME_UDP)
 
 
-def get_rand_port_name():
-    return tests_base.get_rand_name(max_length=n_const.DEVICE_NAME_MAX_LEN,
-                                    prefix=PORT_PREFIX)
-
-
 def increment_ip_cidr(ip_cidr, offset=1):
     """Increment ip_cidr offset times.
 
@@ -165,6 +161,27 @@ def get_free_namespace_port(protocol, namespace=None):
     return get_unused_port(used_ports)
 
 
+def create_patch_ports(source, destination):
+    """Hook up two OVS bridges.
+
+    The result is two patch ports, each end connected to a bridge.
+    The two patch port names will start with 'patch-', followed by identical
+    four characters. For example patch-xyzw-fedora, and patch-xyzw-ubuntu,
+    where fedora and ubuntu are random strings.
+
+    :param source: Instance of OVSBridge
+    :param destination: Instance of OVSBridge
+    """
+    common = tests_base.get_rand_name(max_length=4, prefix='')
+    prefix = '%s-%s-' % (PATCH_PREFIX, common)
+
+    source_name = tests_base.get_rand_device_name(prefix=prefix)
+    destination_name = tests_base.get_rand_device_name(prefix=prefix)
+
+    source.add_patch_port(source_name, destination_name)
+    destination.add_patch_port(destination_name, source_name)
+
+
 class RootHelperProcess(subprocess.Popen):
     def __init__(self, cmd, *args, **kwargs):
         for arg in ('stdin', 'stdout', 'stderr'):

neutron/tests/fullstack/resources/client.py

@@ -35,11 +35,11 @@ class ClientFixture(fixtures.Fixture):
         self.addCleanup(delete, data['id'])
         return data
 
-    def create_router(self, tenant_id, name=None):
+    def create_router(self, tenant_id, name=None, ha=False):
         resource_type = 'router'
 
         name = name or base.get_rand_name(prefix=resource_type)
-        spec = {'tenant_id': tenant_id, 'name': name}
+        spec = {'tenant_id': tenant_id, 'name': name, 'ha': ha}
 
         return self._create_resource(resource_type, spec)
 

neutron/tests/fullstack/resources/config.py

@@ -150,13 +150,13 @@ class NeutronConfigFixture(ConfigFixture):
 
 class ML2ConfigFixture(ConfigFixture):
 
-    def __init__(self, temp_dir):
+    def __init__(self, temp_dir, tenant_network_types):
         super(ML2ConfigFixture, self).__init__(
             temp_dir, base_filename='ml2_conf.ini')
 
         self.config.update({
             'ml2': {
-                'tenant_network_types': 'vlan',
+                'tenant_network_types': tenant_network_types,
                 'mechanism_drivers': 'openvswitch',
             },
             'ml2_type_vlan': {
@@ -168,6 +168,16 @@ class ML2ConfigFixture(ConfigFixture):
             'ml2_type_vxlan': {
                 'vni_ranges': '1001:2000',
             },
+        })
+
+
+class OVSConfigFixture(ConfigFixture):
+
+    def __init__(self, temp_dir):
+        super(OVSConfigFixture, self).__init__(
+            temp_dir, base_filename='openvswitch_agent.ini')
+
+        self.config.update({
             'ovs': {
                 'enable_tunneling': 'False',
                 'local_ip': '127.0.0.1',
@@ -181,14 +191,16 @@ class ML2ConfigFixture(ConfigFixture):
         })
 
     def _generate_bridge_mappings(self):
-        return ('physnet1:%s' %
-                base.get_rand_name(
-                    prefix='br-eth',
-                    max_length=constants.DEVICE_NAME_MAX_LEN))
+        return 'physnet1:%s' % base.get_rand_device_name(prefix='br-eth')
 
     def _generate_integration_bridge(self):
-        return base.get_rand_name(prefix='br-int',
-                                  max_length=constants.DEVICE_NAME_MAX_LEN)
+        return base.get_rand_device_name(prefix='br-int')
+
+    def get_br_int_name(self):
+        return self.config.ovs.integration_bridge
+
+    def get_br_phys_name(self):
+        return self.config.ovs.bridge_mappings.split(':')[1]
 
 
 class L3ConfigFixture(ConfigFixture):
@@ -212,8 +224,10 @@ class L3ConfigFixture(ConfigFixture):
         })
 
     def _generate_external_bridge(self):
-        return base.get_rand_name(prefix='br-ex',
-                                  max_length=constants.DEVICE_NAME_MAX_LEN)
+        return base.get_rand_device_name(prefix='br-ex')
+
+    def get_external_bridge(self):
+        return self.config.DEFAULT.external_network_bridge
 
     def _generate_namespace_suffix(self):
         return base.get_rand_name(prefix='test')

neutron/tests/fullstack/resources/environment.py

@@ -0,0 +1,184 @@
+# Copyright 2015 Red Hat, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import fixtures
+from neutronclient.common import exceptions as nc_exc
+from oslo_config import cfg
+from oslo_log import log as logging
+
+from neutron.agent.linux import utils
+from neutron.tests.common import net_helpers
+from neutron.tests.fullstack.resources import config
+from neutron.tests.fullstack.resources import process
+
+LOG = logging.getLogger(__name__)
+
+
+class HostDescription(object):
+    """A set of characteristics of an environment Host.
+
+    What agents should the host spawn? What mode should each agent operate
+    under?
+    """
+    def __init__(self, l3_agent=True):
+        self.l3_agent = l3_agent
+
+
+class Host(fixtures.Fixture):
+    """The Host class models a physical host running agents, all reporting with
+    the same hostname.
+
+    OpenStack installers or administrators connect compute nodes to the
+    physical tenant network by connecting the provider bridges to their
+    respective physical NICs. Or, if using tunneling, by configuring an
+    IP address on the appropriate physical NIC. The Host class does the same
+    with the connect_* methods.
+
+    TODO(amuller): Add start/stop/restart methods that will start/stop/restart
+    all of the agents on this host. Add a kill method that stops all agents
+    and disconnects the host from other hosts.
+    """
+
+    def __init__(self, test_name, neutron_config, host_description,
+                 central_data_bridge, central_external_bridge):
+        self.test_name = test_name
+        self.neutron_config = neutron_config
+        self.host_description = host_description
+        self.central_data_bridge = central_data_bridge
+        self.central_external_bridge = central_external_bridge
+        self.agents = {}
+
+    def _setUp(self):
+        agent_cfg_fixture = config.OVSConfigFixture(
+            self.neutron_config.temp_dir)
+        self.useFixture(agent_cfg_fixture)
+
+        br_phys = self.useFixture(
+            net_helpers.OVSBridgeFixture(
+                agent_cfg_fixture.get_br_phys_name())).bridge
+        self.connect_to_internal_network_via_vlans(br_phys)
+
+        self.ovs_agent = self.useFixture(
+            process.OVSAgentFixture(
+                self.test_name, self.neutron_config, agent_cfg_fixture))
+
+        if self.host_description.l3_agent:
+            l3_agent_cfg_fixture = self.useFixture(
+                config.L3ConfigFixture(
+                    self.neutron_config.temp_dir,
+                    self.ovs_agent.agent_cfg_fixture.get_br_int_name()))
+            br_ex = self.useFixture(
+                net_helpers.OVSBridgeFixture(
+                    l3_agent_cfg_fixture.get_external_bridge())).bridge
+            self.connect_to_external_network(br_ex)
+            self.l3_agent = self.useFixture(
+                process.L3AgentFixture(
+                    self.test_name,
+                    self.neutron_config,
+                    l3_agent_cfg_fixture))
+
+    def connect_to_internal_network_via_vlans(self, host_data_bridge):
+        # If using VLANs as a segmentation device, it's needed to connect
+        # a provider bridge to a centralized, shared bridge.
+        net_helpers.create_patch_ports(
+            self.central_data_bridge, host_data_bridge)
+
+    def connect_to_external_network(self, host_external_bridge):
+        net_helpers.create_patch_ports(
+            self.central_external_bridge, host_external_bridge)
+
+    @property
+    def l3_agent(self):
+        return self.agents['l3']
+
+    @l3_agent.setter
+    def l3_agent(self, agent):
+        self.agents['l3'] = agent
+
+    @property
+    def ovs_agent(self):
+        return self.agents['ovs']
+
+    @ovs_agent.setter
+    def ovs_agent(self, agent):
+        self.agents['ovs'] = agent
+
+
+class Environment(fixtures.Fixture):
+    """Represents a deployment topology.
+
+    Environment is a collection of hosts. It starts a Neutron server
+    and a parametrized number of Hosts, each a collection of agents.
+    The Environment accepts a collection of HostDescription, each describing
+    the type of Host to create.
+    """
+
+    def __init__(self, hosts_descriptions):
+        """
+        :param hosts_descriptions: A list of HostDescription instances.
+        """
+
+        super(Environment, self).__init__()
+        self.hosts_descriptions = hosts_descriptions
+        self.hosts = []
+
+    def wait_until_env_is_up(self):
+        utils.wait_until_true(self._processes_are_ready)
+
+    def _processes_are_ready(self):
+        try:
+            running_agents = self.neutron_server.client.list_agents()['agents']
+            agents_count = sum(len(host.agents) for host in self.hosts)
+            return len(running_agents) == agents_count
+        except nc_exc.NeutronClientException:
+            return False
+
+    def _create_host(self, description):
+        temp_dir = self.useFixture(fixtures.TempDir()).path
+        neutron_config = config.NeutronConfigFixture(
+            temp_dir, cfg.CONF.database.connection,
+            self.rabbitmq_environment)
+        self.useFixture(neutron_config)
+
+        return self.useFixture(
+            Host(self.test_name,
+                 neutron_config,
+                 description,
+                 self.central_data_bridge,
+                 self.central_external_bridge))
+
+    def _setUp(self):
+        self.temp_dir = self.useFixture(fixtures.TempDir()).path
+        self.rabbitmq_environment = self.useFixture(
+            process.RabbitmqEnvironmentFixture())
+        plugin_cfg_fixture = self.useFixture(
+            config.ML2ConfigFixture(self.temp_dir, 'vlan'))
+        neutron_cfg_fixture = self.useFixture(
+            config.NeutronConfigFixture(
+                self.temp_dir,
+                cfg.CONF.database.connection,
+                self.rabbitmq_environment))
+        self.neutron_server = self.useFixture(
+            process.NeutronServerFixture(
+                self.test_name, neutron_cfg_fixture, plugin_cfg_fixture))
+
+        self.central_data_bridge = self.useFixture(
+            net_helpers.OVSBridgeFixture('cnt-data')).bridge
+        self.central_external_bridge = self.useFixture(
+            net_helpers.OVSBridgeFixture('cnt-ex')).bridge
+
+        self.hosts = [self._create_host(description) for description in
+                      self.hosts_descriptions]
+
+        self.wait_until_env_is_up()

neutron/tests/fullstack/resources/process.py

@@ -12,15 +12,13 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-from datetime import datetime
+import datetime
 from distutils import spawn
-import functools
 import os
 
 import fixtures
 from neutronclient.common import exceptions as nc_exc
 from neutronclient.v2_0 import client
-from oslo_config import cfg
 from oslo_log import log as logging
 
 from neutron.agent.linux import async_process
@@ -28,7 +26,6 @@ from neutron.agent.linux import utils
 from neutron.common import utils as common_utils
 from neutron.tests import base
 from neutron.tests.common import net_helpers
-from neutron.tests.fullstack import config_fixtures
 
 LOG = logging.getLogger(__name__)
 
@@ -46,17 +43,18 @@ class ProcessFixture(fixtures.Fixture):
         self.process = None
 
     def _setUp(self):
-        self.addCleanup(self.stop)
         self.start()
+        self.addCleanup(self.stop)
 
     def start(self):
-        fmt = self.process_name + "--%Y-%m-%d--%H%M%S.log"
         log_dir = os.path.join(DEFAULT_LOG_DIR, self.test_name)
         common_utils.ensure_dir(log_dir)
 
+        timestamp = datetime.datetime.now().strftime("%Y-%m-%d--%H-%M-%S-%f")
+        log_file = "%s--%s.log" % (self.process_name, timestamp)
         cmd = [spawn.find_executable(self.exec_name),
                '--log-dir', log_dir,
-               '--log-file', datetime.utcnow().strftime(fmt)]
+               '--log-file', log_file]
         for filename in self.config_filenames:
             cmd += ['--config-file', filename]
         self.process = async_process.AsyncProcess(cmd)
@@ -88,54 +86,16 @@ class RabbitmqEnvironmentFixture(fixtures.Fixture):
         utils.execute(cmd, run_as_root=True)
 
 
-class FullstackFixture(fixtures.Fixture):
-    def __init__(self):
-        super(FullstackFixture, self).__init__()
-        self.test_name = None
-
-    def _setUp(self):
-        self.temp_dir = self.useFixture(fixtures.TempDir()).path
-        rabbitmq_environment = self.useFixture(RabbitmqEnvironmentFixture())
-
-        self.neutron_server = self.useFixture(
-            NeutronServerFixture(
-                self.test_name, self.temp_dir, rabbitmq_environment))
-
-    def wait_until_env_is_up(self, agents_count):
-        utils.wait_until_true(
-            functools.partial(self._processes_are_ready, agents_count))
-
-    def _processes_are_ready(self, agents_count):
-        try:
-            running_agents = self.neutron_server.client.list_agents()['agents']
-            return len(running_agents) == agents_count
-        except nc_exc.NeutronClientException:
-            return False
-
-
 class NeutronServerFixture(fixtures.Fixture):
 
     NEUTRON_SERVER = "neutron-server"
 
-    def __init__(self, test_name, temp_dir, rabbitmq_environment):
-        super(NeutronServerFixture, self).__init__()
+    def __init__(self, test_name, neutron_cfg_fixture, plugin_cfg_fixture):
         self.test_name = test_name
-        self.temp_dir = temp_dir
-        self.rabbitmq_environment = rabbitmq_environment
+        self.neutron_cfg_fixture = neutron_cfg_fixture
+        self.plugin_cfg_fixture = plugin_cfg_fixture
 
     def _setUp(self):
-        self.neutron_cfg_fixture = config_fixtures.NeutronConfigFixture(
-            self.temp_dir, cfg.CONF.database.connection,
-            self.rabbitmq_environment)
-        self.plugin_cfg_fixture = config_fixtures.ML2ConfigFixture(
-            self.temp_dir)
-
-        self.useFixture(self.neutron_cfg_fixture)
-        self.useFixture(self.plugin_cfg_fixture)
-
-        self.neutron_config = self.neutron_cfg_fixture.config
-        self.plugin_config = self.plugin_cfg_fixture.config
-
         config_filenames = [self.neutron_cfg_fixture.filename,
                             self.plugin_cfg_fixture.filename]
 
@@ -156,7 +116,8 @@ class NeutronServerFixture(fixtures.Fixture):
 
     @property
     def client(self):
-        url = "http://127.0.0.1:%s" % self.neutron_config.DEFAULT.bind_port
+        url = ("http://127.0.0.1:%s" %
+               self.neutron_cfg_fixture.config.DEFAULT.bind_port)
         return client.Client(auth_strategy="noauth", endpoint_url=url)
 
 
@@ -164,21 +125,20 @@ class OVSAgentFixture(fixtures.Fixture):
 
     NEUTRON_OVS_AGENT = "neutron-openvswitch-agent"
 
-    def __init__(self, test_name, neutron_cfg_fixture, ml2_cfg_fixture):
-        super(OVSAgentFixture, self).__init__()
+    def __init__(self, test_name, neutron_cfg_fixture, agent_cfg_fixture):
         self.test_name = test_name
         self.neutron_cfg_fixture = neutron_cfg_fixture
-        self.plugin_cfg_fixture = ml2_cfg_fixture
-
         self.neutron_config = self.neutron_cfg_fixture.config
-        self.plugin_config = self.plugin_cfg_fixture.config
+        self.agent_cfg_fixture = agent_cfg_fixture
+        self.agent_config = agent_cfg_fixture.config
 
     def _setUp(self):
-        self.useFixture(net_helpers.OVSBridgeFixture(self._get_br_int_name()))
-        self.useFixture(net_helpers.OVSBridgeFixture(self._get_br_phys_name()))
+        self.br_int = self.useFixture(
+            net_helpers.OVSBridgeFixture(
+                self.agent_cfg_fixture.get_br_int_name())).bridge
 
         config_filenames = [self.neutron_cfg_fixture.filename,
-                            self.plugin_cfg_fixture.filename]
+                            self.agent_cfg_fixture.filename]
 
         self.process_fixture = self.useFixture(ProcessFixture(
             test_name=self.test_name,
@@ -186,36 +146,22 @@ class OVSAgentFixture(fixtures.Fixture):
             exec_name=self.NEUTRON_OVS_AGENT,
             config_filenames=config_filenames))
 
-    def _get_br_int_name(self):
-        return self.plugin_config.ovs.integration_bridge
-
-    def _get_br_phys_name(self):
-        return self.plugin_config.ovs.bridge_mappings.split(':')[1]
-
 
 class L3AgentFixture(fixtures.Fixture):
 
     NEUTRON_L3_AGENT = "neutron-l3-agent"
 
-    def __init__(self, test_name, temp_dir,
-                 neutron_cfg_fixture, integration_bridge_name):
+    def __init__(self, test_name, neutron_cfg_fixture, l3_agent_cfg_fixture):
         super(L3AgentFixture, self).__init__()
         self.test_name = test_name
-        self.temp_dir = temp_dir
         self.neutron_cfg_fixture = neutron_cfg_fixture
-        self.neutron_config = self.neutron_cfg_fixture.config
-        self.integration_bridge_name = integration_bridge_name
+        self.l3_agent_cfg_fixture = l3_agent_cfg_fixture
 
     def _setUp(self):
-        self.plugin_cfg_fixture = config_fixtures.L3ConfigFixture(
-            self.temp_dir, self.integration_bridge_name)
-        self.useFixture(self.plugin_cfg_fixture)
-        self.plugin_config = self.plugin_cfg_fixture.config
-
-        self.useFixture(net_helpers.OVSBridgeFixture(self._get_br_ex_name()))
+        self.plugin_config = self.l3_agent_cfg_fixture.config
 
         config_filenames = [self.neutron_cfg_fixture.filename,
-                            self.plugin_cfg_fixture.filename]
+                            self.l3_agent_cfg_fixture.filename]
 
         self.process_fixture = self.useFixture(ProcessFixture(
             test_name=self.test_name,
@@ -225,8 +171,5 @@ class L3AgentFixture(fixtures.Fixture):
                 path=os.path.join(base.ROOTDIR, 'common', 'agents')),
             config_filenames=config_filenames))
 
-    def _get_br_ex_name(self):
-        return self.plugin_config.DEFAULT.external_network_bridge
-
     def get_namespace_suffix(self):
         return self.plugin_config.DEFAULT.test_namespace_suffix

neutron/tests/fullstack/test_l3_agent.py

@@ -12,6 +12,8 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import functools
+
 from oslo_utils import uuidutils
 
 from neutron.agent.l3 import agent as l3_agent
@@ -19,34 +21,15 @@ from neutron.agent.l3 import namespaces
 from neutron.agent.linux import ip_lib
 from neutron.agent.linux import utils
 from neutron.tests.fullstack import base
-from neutron.tests.fullstack import fullstack_fixtures as f_fixtures
-
-
-class SingleNodeEnvironment(f_fixtures.FullstackFixture):
-    def _setUp(self):
-        super(SingleNodeEnvironment, self)._setUp()
-
-        neutron_config = self.neutron_server.neutron_cfg_fixture
-        ml2_config = self.neutron_server.plugin_cfg_fixture
-
-        self.ovs_agent = self.useFixture(
-            f_fixtures.OVSAgentFixture(
-                self.test_name, neutron_config, ml2_config))
-
-        self.l3_agent = self.useFixture(
-            f_fixtures.L3AgentFixture(
-                self.test_name,
-                self.temp_dir,
-                neutron_config,
-                self.ovs_agent._get_br_int_name()))
-
-        self.wait_until_env_is_up(agents_count=2)
+from neutron.tests.fullstack.resources import environment
 
 
 class TestLegacyL3Agent(base.BaseFullStackTestCase):
     def __init__(self, *args, **kwargs):
         super(TestLegacyL3Agent, self).__init__(
-            SingleNodeEnvironment(), *args, **kwargs)
+            environment.Environment(
+                [environment.HostDescription(l3_agent=True)]),
+            *args, **kwargs)
 
     def _get_namespace(self, router_id):
         return namespaces.build_ns_name(l3_agent.NS_PREFIX, router_id)
@@ -66,5 +49,35 @@ class TestLegacyL3Agent(base.BaseFullStackTestCase):
 
         namespace = "%s@%s" % (
             self._get_namespace(router['id']),
-            self.environment.l3_agent.get_namespace_suffix(), )
+            self.environment.hosts[0].l3_agent.get_namespace_suffix(), )
         self._assert_namespace_exists(namespace)
+
+
+class TestHAL3Agent(base.BaseFullStackTestCase):
+    def __init__(self, *args, **kwargs):
+        super(TestHAL3Agent, self).__init__(
+            environment.Environment(
+                [environment.HostDescription(l3_agent=True),
+                 environment.HostDescription(l3_agent=True)]),
+            *args, **kwargs)
+
+    def _is_ha_router_active_on_one_agent(self, router_id):
+        agents = self.client.list_l3_agent_hosting_routers(router_id)
+        return (
+            agents['agents'][0]['ha_state'] != agents['agents'][1]['ha_state'])
+
+    def test_ha_router(self):
+        # TODO(amuller): Test external connectivity before and after a
+        # failover, see: https://review.openstack.org/#/c/196393/
+
+        tenant_id = uuidutils.generate_uuid()
+        router = self.safe_client.create_router(tenant_id, ha=True)
+        agents = self.client.list_l3_agent_hosting_routers(router['id'])
+        self.assertEqual(2, len(agents['agents']),
+                         'HA router must be scheduled to both nodes')
+
+        utils.wait_until_true(
+            functools.partial(
+                self._is_ha_router_active_on_one_agent,
+                router['id']),
+            timeout=90)

neutron/tests/functional/agent/test_ovs_lib.py

@@ -19,6 +19,7 @@ import uuid
 
 from neutron.agent.common import ovs_lib
 from neutron.agent.linux import ip_lib
+from neutron.tests import base as tests_base
 from neutron.tests.common import net_helpers
 from neutron.tests.functional.agent.linux import base
 
@@ -35,7 +36,7 @@ class OVSBridgeTestBase(base.BaseOVSLinuxTestCase):
         # Convert ((a, b), (c, d)) to {a: b, c: d} and add 'type' by default
         attrs = collections.OrderedDict(interface_attrs)
         attrs.setdefault('type', 'internal')
-        port_name = net_helpers.get_rand_port_name()
+        port_name = tests_base.get_rand_device_name(net_helpers.PORT_PREFIX)
         return (port_name, self.br.add_port(port_name, *attrs.items()))
 
     def create_ovs_vif_port(self, iface_id=None, mac=None,
@@ -73,7 +74,7 @@ class OVSBridgeTestCase(OVSBridgeTestBase):
         self.assertRaises(RuntimeError, cmd.execute, check_error=True)
 
     def test_replace_port(self):
-        port_name = net_helpers.get_rand_port_name()
+        port_name = tests_base.get_rand_device_name(net_helpers.PORT_PREFIX)
         self.br.replace_port(port_name, ('type', 'internal'))
         self.assertTrue(self.br.port_exists(port_name))
         self.assertEqual('internal',
@@ -150,7 +151,7 @@ class OVSBridgeTestCase(OVSBridgeTestBase):
             'remote_ip': '192.0.2.1',  # RFC 5737 TEST-NET-1
             'local_ip': '198.51.100.1',  # RFC 5737 TEST-NET-2
         }
-        port_name = net_helpers.get_rand_port_name()
+        port_name = tests_base.get_rand_device_name(net_helpers.PORT_PREFIX)
         self.br.add_tunnel_port(port_name, attrs['remote_ip'],
                                 attrs['local_ip'])
         self.assertEqual(self.ovs.db_get_val('Interface', port_name, 'type'),
@@ -160,7 +161,7 @@ class OVSBridgeTestCase(OVSBridgeTestBase):
             self.assertEqual(val, options[attr])
 
     def test_add_patch_port(self):
-        local = net_helpers.get_rand_port_name()
+        local = tests_base.get_rand_device_name(net_helpers.PORT_PREFIX)
         peer = 'remotepeer'
         self.br.add_patch_port(local, peer)
         self.assertEqual(self.ovs.db_get_val('Interface', local, 'type'),