Add full-stack tests framework

This patch introduces the full-stack tests framework, as specified in
the blueprint. In short, this adds the neutron.tests.fullstack module,
which supports test-managed neutron daemons. Currently only
neutron-server is supported and follow-up patches will support for
multiple agents.

Implements: blueprint integration-tests
Co-Authored-By: Maru Newby <marun@redhat.com>
Change-Id: Iff24fc7cd428488e918c5f06bc7f923095760b07

TESTING.rst

@@ -138,6 +138,20 @@ to install and configure all of Neutron's package dependencies.  It is
 not necessary to provide this option if devstack has already been used
 to deploy Neutron to the target host.
 
+To run all the full-stack tests, you may use: ::
+
+    tox -e dsvm-fullstack
+
+Since full-stack tests often require the same resources and
+dependencies as the functional tests, using the configuration script
+tools/configure_for_func_testing.sh is advised (as described above).
+When running full-stack tests on a clean VM for the first time, we
+advise to run ./stack.sh successfully to make sure all Neutron's
+dependencies are met.  Also note that in order to preserve resources
+on the gate, running the dsvm-functional suite will also run all
+full-stack tests (and a new worker won't be assigned specifically for
+dsvm-fullstack).
+
 To run the api tests against a live Neutron daemon, deploy tempest and
 neutron with devstack and then run the following commands: ::
 

neutron/agent/linux/async_process.py

@@ -172,6 +172,9 @@ class AsyncProcess(object):
                 LOG.exception(_LE('An error occurred while killing [%s].'),
                               self.cmd)
                 return False
+
+        if self._process:
+            self._process.wait()
         return True
 
     def _handle_process_error(self):
@@ -188,7 +191,8 @@ class AsyncProcess(object):
     def _watch_process(self, callback, kill_event):
         while not kill_event.ready():
             try:
-                if not callback():
+                output = callback()
+                if not output and output != "":
                     break
             except Exception:
                 LOG.exception(_LE('An error occurred while communicating '

neutron/agent/linux/utils.py

@@ -286,23 +286,23 @@ def get_cmdline_from_pid(pid):
         return f.readline().split('\0')[:-1]
 
 
-def cmdlines_are_equal(cmd1, cmd2):
-    """Validate provided lists containing output of /proc/cmdline are equal
-
-    This function ignores absolute paths of executables in order to have
-    correct results in case one list uses absolute path and the other does not.
-    """
-    cmd1 = remove_abs_path(cmd1)
-    cmd2 = remove_abs_path(cmd2)
-    return cmd1 == cmd2
+def cmd_matches_expected(cmd, expected_cmd):
+    abs_cmd = remove_abs_path(cmd)
+    abs_expected_cmd = remove_abs_path(expected_cmd)
+    if abs_cmd != abs_expected_cmd:
+        # Commands executed with #! are prefixed with the script
+        # executable. Check for the expected cmd being a subset of the
+        # actual cmd to cover this possibility.
+        abs_cmd = remove_abs_path(abs_cmd[1:])
+    return abs_cmd == abs_expected_cmd
 
 
 def pid_invoked_with_cmdline(pid, expected_cmd):
     """Validate process with given pid is running with provided parameters
 
     """
-    cmdline = get_cmdline_from_pid(pid)
-    return cmdlines_are_equal(expected_cmd, cmdline)
+    cmd = get_cmdline_from_pid(pid)
+    return cmd_matches_expected(cmd, expected_cmd)
 
 
 def wait_until_true(predicate, timeout=60, sleep=1, exception=None):

neutron/tests/api/base_v2.py

@@ -55,19 +55,6 @@ import testtools
 from neutron.tests import sub_base
 
 
-class AttributeDict(dict):
-
-    """
-    Provide attribute access (dict.key) to dictionary values.
-    """
-
-    def __getattr__(self, name):
-        """Allow attribute access for all keys in the dict."""
-        if name in self:
-            return self[name]
-        raise AttributeError(_("Unknown attribute '%s'.") % name)
-
-
 @six.add_metaclass(abc.ABCMeta)
 class BaseNeutronClient(object):
     """

neutron/tests/api/test_v2_rest.py

@@ -22,6 +22,7 @@ from tempest_lib import exceptions
 import testscenarios
 
 from neutron.tests.api import base_v2
+from neutron.tests import sub_base
 from neutron.tests.tempest import test as t_test
 
 # Required to generate tests from scenarios.  Not compatible with nose.
@@ -55,19 +56,19 @@ class TempestRestClient(base_v2.BaseNeutronClient):
     def _create_network(self, **kwargs):
         # Internal method - use create_network() instead
         body = self.client.create_network(**kwargs)
-        return base_v2.AttributeDict(body['network'])
+        return sub_base.AttributeDict(body['network'])
 
     def update_network(self, id_, **kwargs):
         body = self.client.update_network(id_, **kwargs)
-        return base_v2.AttributeDict(body['network'])
+        return sub_base.AttributeDict(body['network'])
 
     def get_network(self, id_, **kwargs):
         body = self.client.show_network(id_, **kwargs)
-        return base_v2.AttributeDict(body['network'])
+        return sub_base.AttributeDict(body['network'])
 
     def get_networks(self, **kwargs):
         body = self.client.list_networks(**kwargs)
-        return [base_v2.AttributeDict(x) for x in body['networks']]
+        return [sub_base.AttributeDict(x) for x in body['networks']]
 
     def delete_network(self, id_):
         self.client.delete_network(id_)

neutron/tests/common/helpers.py

@@ -0,0 +1,31 @@
+# Copyright 2015 Red Hat, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+
+import neutron
+
+
+def find_file(filename, path):
+    """Find a file with name 'filename' located in 'path'."""
+    for root, _, files in os.walk(path):
+        if filename in files:
+            return os.path.abspath(os.path.join(root, filename))
+
+
+def find_sample_file(filename):
+    """Find a file with name 'filename' located in the sample directory."""
+    return find_file(
+        filename,
+        path=os.path.join(neutron.__path__[0], '..', 'etc'))

neutron/tests/fullstack/base.py

@@ -0,0 +1,60 @@
+# Copyright 2015 Red Hat, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_config import cfg
+from oslo_db.sqlalchemy import test_base
+
+from neutron.db.migration.models import head  # noqa
+from neutron.db import model_base
+from neutron.tests.fullstack import fullstack_fixtures as f_fixtures
+
+
+class BaseFullStackTestCase(test_base.MySQLOpportunisticTestCase):
+    """Base test class for full-stack tests.
+
+    :param process_fixtures: a list of fixture classes (not instances).
+    """
+
+    def setUp(self):
+        super(BaseFullStackTestCase, self).setUp()
+        self.create_db_tables()
+
+        self.neutron_server = self.useFixture(
+            f_fixtures.NeutronServerFixture())
+        self.client = self.neutron_server.client
+
+    @property
+    def test_name(self):
+        """Return the name of the test currently running."""
+        return self.id().split(".")[-1]
+
+    def create_db_tables(self):
+        """Populate the new database.
+
+        MySQLOpportunisticTestCase creates a new database for each test, but
+        these need to be populated with the appropriate tables. Before we can
+        do that, we must change the 'connection' option which the Neutron code
+        knows to look at.
+
+        Currently, the username and password options are hard-coded by
+        oslo.db and neutron/tests/functional/contrib/gate_hook.sh. Also,
+        we only support MySQL for now, but the groundwork for adding Postgres
+        is already laid.
+        """
+        conn = "mysql://%(username)s:%(password)s@127.0.0.1/%(db_name)s" % {
+            'username': test_base.DbFixture.USERNAME,
+            'password': test_base.DbFixture.PASSWORD,
+            'db_name': self.engine.url.database}
+        cfg.CONF.set_override('connection', conn, group='database')
+        model_base.BASEV2.metadata.create_all(self.engine)

neutron/tests/fullstack/config_fixtures.py

@@ -0,0 +1,183 @@
+# Copyright 2015 Red Hat, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os.path
+import tempfile
+
+import fixtures
+import six
+
+from neutron.common import constants
+from neutron.tests.common import helpers as c_helpers
+from neutron.tests.functional.agent.linux import helpers
+from neutron.tests import sub_base
+
+
+class ConfigDict(sub_base.AttributeDict):
+    def update(self, other):
+        self.convert_to_attr_dict(other)
+        super(ConfigDict, self).update(other)
+
+    def convert_to_attr_dict(self, other):
+        """Convert nested dicts to AttributeDict.
+
+        :param other: dictionary to be directly modified.
+        """
+        for key, value in other.iteritems():
+            if isinstance(value, dict):
+                if not isinstance(value, sub_base.AttributeDict):
+                    other[key] = sub_base.AttributeDict(value)
+                self.convert_to_attr_dict(value)
+
+
+class ConfigFileFixture(fixtures.Fixture):
+    """A fixture that knows how to translate configurations to files.
+
+    :param base_filename: the filename to use on disk.
+    :param config: a ConfigDict instance.
+    :param temp_dir: an existing temporary directory to use for storage.
+    """
+
+    def __init__(self, base_filename, config, temp_dir):
+        super(ConfigFileFixture, self).__init__()
+        self.base_filename = base_filename
+        self.config = config
+        self.temp_dir = temp_dir
+
+    def setUp(self):
+        super(ConfigFileFixture, self).setUp()
+        config_parser = self.dict_to_config_parser(self.config)
+        # Need to randomly generate a unique folder to put the file in
+        self.filename = os.path.join(self.temp_dir, self.base_filename)
+        with open(self.filename, 'w') as f:
+            config_parser.write(f)
+            f.flush()
+
+    def dict_to_config_parser(self, config_dict):
+        config_parser = six.moves.configparser.SafeConfigParser()
+        for section, section_dict in six.iteritems(config_dict):
+            if section != 'DEFAULT':
+                config_parser.add_section(section)
+            for option, value in six.iteritems(section_dict):
+                config_parser.set(section, option, value)
+        return config_parser
+
+
+class ConfigFixture(fixtures.Fixture):
+    """A fixture that holds an actual Neutron configuration.
+
+    Note that 'self.config' is intended to only be updated once, during
+    the constructor, so if this fixture is re-used (setUp is called twice),
+    then the dynamic configuration values won't change. The correct usage
+    is initializing a new instance of the class.
+    """
+    def __init__(self, temp_dir, base_filename):
+        self.config = ConfigDict()
+        self.temp_dir = temp_dir
+        self.base_filename = base_filename
+
+    def setUp(self):
+        super(ConfigFixture, self).setUp()
+        cfg_fixture = ConfigFileFixture(
+            self.base_filename, self.config, self.temp_dir)
+        self.useFixture(cfg_fixture)
+        self.filename = cfg_fixture.filename
+
+
+class NeutronConfigFixture(ConfigFixture):
+
+    def __init__(self, temp_dir, connection):
+        super(NeutronConfigFixture, self).__init__(
+            temp_dir, base_filename='neutron.conf')
+
+        self.config.update({
+            'DEFAULT': {
+                'host': self._generate_host(),
+                'state_path': self._generate_state_path(temp_dir),
+                'bind_port': self._generate_port(),
+                'api_paste_config': self._generate_api_paste(),
+                'policy_file': self._generate_policy_json(),
+                'core_plugin': 'neutron.plugins.ml2.plugin.Ml2Plugin',
+                'rabbit_userid': 'stackrabbit',
+                'rabbit_password': 'secretrabbit',
+                'rabbit_hosts': '127.0.0.1',
+                'auth_strategy': 'noauth',
+                'verbose': 'True',
+                'debug': 'True',
+            },
+            'database': {
+                'connection': connection,
+            }
+        })
+
+    def _generate_host(self):
+        return sub_base.get_rand_name(prefix='host-')
+
+    def _generate_state_path(self, temp_dir):
+        # Assume that temp_dir will be removed by the caller
+        self.state_path = tempfile.mkdtemp(prefix='state_path', dir=temp_dir)
+        return self.state_path
+
+    def _generate_port(self):
+        """Get a free TCP port from the Operating System and return it.
+
+        This might fail if some other process occupies this port after this
+        function finished but before the neutron-server process started.
+        """
+        return str(helpers.get_free_namespace_port())
+
+    def _generate_api_paste(self):
+        return c_helpers.find_sample_file('api-paste.ini')
+
+    def _generate_policy_json(self):
+        return c_helpers.find_sample_file('policy.json')
+
+
+class ML2ConfigFixture(ConfigFixture):
+
+    def __init__(self, temp_dir):
+        super(ML2ConfigFixture, self).__init__(
+            temp_dir, base_filename='ml2_conf.ini')
+
+        self.config.update({
+            'ml2': {
+                'tenant_network_types': 'vlan',
+                'mechanism_drivers': 'openvswitch',
+            },
+            'ml2_type_vlan': {
+                'network_vlan_ranges': 'physnet1:1000:2999',
+            },
+            'ml2_type_gre': {
+                'tunnel_id_ranges': '1:1000',
+            },
+            'ml2_type_vxlan': {
+                'vni_ranges': '1001:2000',
+            },
+            'ovs': {
+                'enable_tunneling': 'False',
+                'local_ip': '127.0.0.1',
+                'bridge_mappings': self._generate_bridge_mappings(),
+                'integration_bridge': self._generate_integration_bridge(),
+            }
+        })
+
+    def _generate_bridge_mappings(self):
+        return ('physnet1:%s' %
+                sub_base.get_rand_name(
+                    prefix='br-eth',
+                    max_length=constants.DEVICE_NAME_MAX_LEN))
+
+    def _generate_integration_bridge(self):
+        return sub_base.get_rand_name(prefix='br-int',
+                                      max_length=constants.DEVICE_NAME_MAX_LEN)

neutron/tests/fullstack/fullstack_fixtures.py

@@ -0,0 +1,104 @@
+# Copyright 2015 Red Hat, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from distutils import spawn
+
+import fixtures
+from neutronclient.common import exceptions as nc_exc
+from neutronclient.v2_0 import client
+from oslo_config import cfg
+from oslo_log import log as logging
+from oslo_utils import timeutils
+
+from neutron.agent.linux import async_process
+from neutron.agent.linux import utils
+from neutron.tests.fullstack import config_fixtures
+
+LOG = logging.getLogger(__name__)
+
+# This should correspond the directory from which infra retrieves log files
+DEFAULT_LOG_DIR = '/opt/stack/logs'
+
+
+class ProcessFixture(fixtures.Fixture):
+    def __init__(self, name, exec_name, config_filenames):
+        super(ProcessFixture, self).__init__()
+        self.name = name
+        self.exec_name = exec_name
+        self.config_filenames = config_filenames
+        self.process = None
+
+    def setUp(self):
+        super(ProcessFixture, self).setUp()
+        self.start()
+
+    def start(self):
+        fmt = self.name + "--%Y-%m-%d--%H%M%S.log"
+        cmd = [spawn.find_executable(self.exec_name),
+               '--log-dir', DEFAULT_LOG_DIR,
+               '--log-file', timeutils.strtime(fmt=fmt)]
+        for filename in self.config_filenames:
+            cmd += ['--config-file', filename]
+        self.process = async_process.AsyncProcess(cmd)
+        self.process.start(block=True)
+
+    def stop(self):
+        self.process.stop(block=True)
+
+    def cleanUp(self, *args, **kwargs):
+        self.stop()
+        super(ProcessFixture, self, *args, **kwargs)
+
+
+class NeutronServerFixture(fixtures.Fixture):
+
+    def setUp(self):
+        super(NeutronServerFixture, self).setUp()
+        self.temp_dir = self.useFixture(fixtures.TempDir()).path
+
+        self.neutron_cfg_fixture = config_fixtures.NeutronConfigFixture(
+            self.temp_dir, cfg.CONF.database.connection)
+        self.plugin_cfg_fixture = config_fixtures.ML2ConfigFixture(
+            self.temp_dir)
+
+        self.useFixture(self.neutron_cfg_fixture)
+        self.useFixture(self.plugin_cfg_fixture)
+
+        self.neutron_config = self.neutron_cfg_fixture.config
+
+        config_filenames = [self.neutron_cfg_fixture.filename,
+                            self.plugin_cfg_fixture.filename]
+
+        self.process_fixture = self.useFixture(ProcessFixture(
+            name='neutron_server',
+            exec_name='neutron-server',
+            config_filenames=config_filenames,
+        ))
+
+        utils.wait_until_true(self.processes_are_ready)
+
+    @property
+    def client(self):
+        url = "http://127.0.0.1:%s" % self.neutron_config.DEFAULT.bind_port
+        return client.Client(auth_strategy="noauth", endpoint_url=url)
+
+    def processes_are_ready(self):
+        # ProcessFixture will ensure that the server has started, but
+        # that doesn't mean that the server will be serving commands yet, nor
+        # that all processes are up.
+        try:
+            return len(self.client.list_agents()['agents']) == 0
+        except nc_exc.NeutronClientException:
+            LOG.debug("Processes aren't up yet.")
+            return False

neutron/tests/fullstack/test_sanity.py

@@ -0,0 +1,25 @@
+# Copyright 2015 Red Hat, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+#TODO(jschwarz): This is an example test file which demonstrates the
+# general usage of fullstack. Once we add more FullStack tests, this should
+# be deleted.
+
+from neutron.tests.fullstack import base
+
+
+class TestSanity(base.BaseFullStackTestCase):
+
+    def test_sanity(self):
+        self.assertEqual(self.client.list_networks(), {'networks': []})

neutron/tests/functional/__init__.py

@@ -0,0 +1,44 @@
+# Copyright 2015 Red Hat, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Previously, running 'tox -e dsvm-functional' simply ran a normal test discovery
+of the ./neutron/tests/functional tree. In order to save gate resources, we
+decided to forgo adding a new gate job specifically for the full-stack
+framework, and instead discover tests that are present in
+./neutron/tests/fullstack.
+
+In short, running 'tox -e dsvm-functional' now runs both functional tests and
+full-stack tests, and this code allows for the test discovery needed.
+"""
+
+import os
+import unittest
+
+
+def _discover(loader, path, pattern):
+    return loader.discover(path, pattern=pattern, top_level_dir=".")
+
+
+def load_tests(_, tests, pattern):
+    suite = unittest.TestSuite()
+    suite.addTests(tests)
+
+    loader = unittest.loader.TestLoader()
+    suite.addTests(_discover(loader, "./neutron/tests/functional", pattern))
+
+    if os.getenv('OS_RUN_FULLSTACK') == '1':
+        suite.addTests(_discover(loader, "./neutron/tests/fullstack", pattern))
+
+    return suite

neutron/tests/functional/api/test_v2_plugin.py

@@ -25,6 +25,7 @@ from neutron.common import exceptions as q_exc
 from neutron import context
 from neutron import manager
 from neutron.tests.api import base_v2
+from neutron.tests import sub_base
 from neutron.tests.unit.ml2 import test_ml2_plugin
 from neutron.tests.unit import testlib_api
 from neutron.tests.unit import testlib_plugin
@@ -68,20 +69,20 @@ class PluginClient(base_v2.BaseNeutronClient):
         kwargs.setdefault('shared', False)
         data = dict(network=kwargs)
         result = self.plugin.create_network(self.ctx, data)
-        return base_v2.AttributeDict(result)
+        return sub_base.AttributeDict(result)
 
     def update_network(self, id_, **kwargs):
         data = dict(network=kwargs)
         result = self.plugin.update_network(self.ctx, id_, data)
-        return base_v2.AttributeDict(result)
+        return sub_base.AttributeDict(result)
 
     def get_network(self, *args, **kwargs):
         result = self.plugin.get_network(self.ctx, *args, **kwargs)
-        return base_v2.AttributeDict(result)
+        return sub_base.AttributeDict(result)
 
     def get_networks(self, *args, **kwargs):
         result = self.plugin.get_networks(self.ctx, *args, **kwargs)
-        return [base_v2.AttributeDict(x) for x in result]
+        return [sub_base.AttributeDict(x) for x in result]
 
     def delete_network(self, id_):
         self.plugin.delete_network(self.ctx, id_)

neutron/tests/sub_base.py

@@ -52,6 +52,19 @@ def bool_from_env(key, strict=False, default=False):
     return strutils.bool_from_string(value, strict=strict, default=default)
 
 
+class AttributeDict(dict):
+
+    """
+    Provide attribute access (dict.key) to dictionary values.
+    """
+
+    def __getattr__(self, name):
+        """Allow attribute access for all keys in the dict."""
+        if name in self:
+            return self[name]
+        raise AttributeError(_("Unknown attribute '%s'.") % name)
+
+
 class SubBaseTestCase(testtools.TestCase):
 
     def setUp(self):

neutron/tests/unit/agent/linux/test_utils.py

@@ -198,15 +198,16 @@ class TestPathUtilities(base.BaseTestCase):
         self.assertEqual(['ping', '8.8.8.8'],
                          utils.remove_abs_path(['/usr/bin/ping', '8.8.8.8']))
 
-    def test_cmdlines_are_equal(self):
-        self.assertTrue(utils.cmdlines_are_equal(
-            ['ping', '8.8.8.8'],
-            ['/usr/bin/ping', '8.8.8.8']))
+    def test_cmd_matches_expected_matches_abs_path(self):
+        cmd = ['/bar/../foo']
+        self.assertTrue(utils.cmd_matches_expected(cmd, cmd))
 
-    def test_cmdlines_are_equal_different_commands(self):
-        self.assertFalse(utils.cmdlines_are_equal(
-            ['ping', '8.8.8.8'],
-            ['/usr/bin/ping6', '8.8.8.8']))
+    def test_cmd_matches_expected_matches_script(self):
+        self.assertTrue(utils.cmd_matches_expected(['python', 'script'],
+                                                   ['script']))
+
+    def test_cmd_matches_expected_doesnt_match(self):
+        self.assertFalse(utils.cmd_matches_expected('foo', 'bar'))
 
 
 class TestBaseOSUtils(base.BaseTestCase):

tox.ini

@@ -43,6 +43,25 @@ setenv = OS_TEST_PATH=./neutron/tests/functional
          OS_ROOTWRAP_DAEMON_CMD=sudo {envbindir}/neutron-rootwrap-daemon {envdir}/etc/neutron/rootwrap.conf
          OS_FAIL_ON_MISSING_DEPS=1
          OS_TEST_TIMEOUT=90
+         OS_RUN_FULLSTACK=1
+sitepackages=True
+deps =
+  {[testenv:functional]deps}
+
+[testenv:fullstack]
+setenv = OS_TEST_PATH=./neutron/tests/fullstack
+         OS_TEST_TIMEOUT=90
+deps =
+  {[testenv]deps}
+  -r{toxinidir}/neutron/tests/functional/requirements.txt
+
+[testenv:dsvm-fullstack]
+setenv = OS_TEST_PATH=./neutron/tests/fullstack
+         OS_SUDO_TESTING=1
+         OS_ROOTWRAP_CMD=sudo {envbindir}/neutron-rootwrap {envdir}/etc/neutron/rootwrap.conf
+         OS_ROOTWRAP_DAEMON_CMD=sudo {envbindir}/neutron-rootwrap-daemon {envdir}/etc/neutron/rootwrap.conf
+         OS_FAIL_ON_MISSING_DEPS=1
+         OS_TEST_TIMEOUT=90
 sitepackages=True
 deps =
   {[testenv:functional]deps}