[Fullstack] Use new DB for each running test

Until now neutron fullstack tests which were run by the same worker were
using same DB but after test content of the DB was cleaned.
This could cause problems e.g. for default security group rules which
weren't created properly in second test run by the same worker.

To fix that issue patch [1] was proposed and merged some time ago. But
this didn't solve the problem so this patch is effectively reverting [1]
and proposing another solution which will make each fullstack test to
use own DB and run db migration script.

As running DB migration before every test makes this jobs to run a bit
longer than it took before, this patch also increases timeout for the
fullstack job(s) to 3h (10800 seconds).

[1] https://review.opendev.org/c/openstack/neutron/+/891040

Related-bug: #1983053
Change-Id: Ia261b4c62db9a99ef6eb161acb4609520e45d101

neutron/tests/fullstack/base.py

@@ -48,6 +48,11 @@ class BaseFullStackTestCase(testlib_api.MySQLTestCaseMixin,
 
     BUILD_WITH_MIGRATIONS = True
 
+    # NOTE(slaweq): In fullstack tests there need to be new database created
+    # for every test, and one db shouldn't be really shared between tests
+    # running by the same worker
+    CLEAN_DB_AFTER_TEST = True
+
     def setUp(self, environment):
         super(BaseFullStackTestCase, self).setUp()
 

neutron/tests/fullstack/test_securitygroup.py

@@ -49,12 +49,6 @@ class OVSVersionChecker(object):
 
 class BaseSecurityGroupsSameNetworkTest(base.BaseFullStackTestCase):
 
-    # NOTE(slaweq): Those SG tests are run serially by same worker thus we need
-    # to run db migration script before each test because in some cases cleanup
-    # after one test may clean data from the db and next test run by same
-    # worker will have empty tables
-    FORCE_DB_MIGRATION = True
-
     def setUp(self):
         debug_iptables = self.firewall_driver.startswith("iptables")
         host_descriptions = [

neutron/tests/unit/testlib_api.py

@@ -122,10 +122,7 @@ class OpportunisticSqlFixture(lib_fixtures.SqlFixture):
     def _init_resources(self):
         testresources.setUpResources(
             self.test, self.test.resources, testresources._get_result())
-        self.addCleanup(
-            testresources.tearDownResources,
-            self.test, self.test.resources, testresources._get_result()
-        )
+        self.addCleanup(self._cleanup_resources)
 
         # unfortunately, fixtures won't let us call a skip() from
         # here.  So the test has to check this also.
@@ -133,6 +130,14 @@ class OpportunisticSqlFixture(lib_fixtures.SqlFixture):
         if hasattr(self.test, 'db'):
             self.engine = self.test.engine = self.test.db.engine
 
+    def _cleanup_resources(self):
+        testresources.tearDownResources(
+            self.test, self.test.resources, testresources._get_result())
+
+        if self.test.CLEAN_DB_AFTER_TEST:
+            self.test._database_resources.pop(self.test.DRIVER)
+            self.test._schema_resources.pop((self.test.DRIVER, None))
+
     @classmethod
     def resources_collection(cls, test):
         # reimplement current oslo.db code.
@@ -156,7 +161,7 @@ class OpportunisticSqlFixture(lib_fixtures.SqlFixture):
 
         key = (driver, None)
         if test.BUILD_SCHEMA:
-            if test.FORCE_DB_MIGRATION or key not in test._schema_resources:
+            if key not in test._schema_resources:
                 test._schema_resources[key] = provision.SchemaResource(
                     database_resource,
                     cls._generate_schema_w_migrations
@@ -209,7 +214,7 @@ class OpportunisticDBTestMixin(object):
     FIXTURE = OpportunisticSqlFixture
 
     BUILD_WITH_MIGRATIONS = False
-    FORCE_DB_MIGRATION = False
+    CLEAN_DB_AFTER_TEST = False
 
     def _setup_database_fixtures(self):
         self.useFixture(self.FIXTURE(self))

zuul.d/base.yaml

@@ -59,7 +59,7 @@
 - job:
     name: neutron-fullstack
     parent: neutron-functional
-    timeout: 9000
+    timeout: 10800
     vars:
       tox_envlist: dsvm-fullstack-gate
       Q_BUILD_OVS_FROM_GIT: False