From 7efc0113ef03b792b382d4bc32aeffcd3667bdda Mon Sep 17 00:00:00 2001 From: "Castulo J. Martinez" Date: Wed, 11 Nov 2015 09:05:33 -0800 Subject: [PATCH] Re-run failed Tempest tests Add the ability to re-run the tempest tests that failed in the last test execution. Sometimes Tempest tests fail due to a special temporary condition in the environment, in such cases it is very useful to be able to re-execute those tests. Running the following command will re-run all the test that failed during the last test execution regardless of what test suite was run. $ rally verify start --failing Change-Id: Ieccb6d2a28175c3445d028e81ae2718e4fce375b Implements: blueprint re-run-failed-tempest-tests --- etc/rally.bash_completion | 2 +- rally/api.py | 9 +++++-- rally/cli/commands/verify.py | 20 +++++++++++++--- rally/verification/tempest/tempest.py | 12 ++++++---- tests/unit/cli/commands/test_verify.py | 32 +++++++++++++++++++++---- tests/unit/test_api.py | 9 ++++--- tests/unit/verification/test_tempest.py | 30 +++++++++++++++++++---- 7 files changed, 90 insertions(+), 24 deletions(-) diff --git a/etc/rally.bash_completion b/etc/rally.bash_completion index e6b3b831d7..f07a6b4923 100644 --- a/etc/rally.bash_completion +++ b/etc/rally.bash_completion @@ -56,7 +56,7 @@ _rally() OPTS["verify_results"]="--uuid --html --json --output-file" OPTS["verify_show"]="--uuid --sort-by --detailed" OPTS["verify_showconfig"]="--deployment" - OPTS["verify_start"]="--deployment --set --regex --tests-file --tempest-config --xfails-file --no-use --system-wide --concurrency" + OPTS["verify_start"]="--deployment --set --regex --tests-file --tempest-config --xfails-file --no-use --system-wide --concurrency --failing" OPTS["verify_uninstall"]="--deployment" OPTS["verify_use"]="--uuid" diff --git a/rally/api.py b/rally/api.py index 7acb325472..8f92cafde1 100644 --- a/rally/api.py +++ b/rally/api.py @@ -359,9 +359,10 @@ class Verification(object): @classmethod def verify(cls, deployment, set_name="", regex=None, tests_file=None, tempest_config=None, expected_failures=None, system_wide=False, - concur=0): + concur=0, failing=False): """Start verification. + :param deployment: UUID or name of a deployment :param deployment: UUID or name of a deployment :param set_name: Name of a Tempest test set :param regex: Regular expression of test @@ -376,8 +377,11 @@ class Verification(object): env when running the tests :param concur: How many processes to use to run Tempest tests. The default value (0) auto-detects CPU count + :param failing: Re-run tests that failed during the last + execution :returns: Verification object """ + deployment_uuid = objects.Deployment.get(deployment)["uuid"] verification = objects.Verification(deployment_uuid=deployment_uuid) verifier = tempest.Tempest(deployment_uuid, @@ -395,7 +399,8 @@ class Verification(object): LOG.info("Starting verification of deployment: %s" % deployment_uuid) verification.set_running() verifier.verify(set_name=set_name, regex=regex, tests_file=tests_file, - expected_failures=expected_failures, concur=concur) + expected_failures=expected_failures, concur=concur, + failing=failing) return verification diff --git a/rally/cli/commands/verify.py b/rally/cli/commands/verify.py index 235aea107c..20242431b9 100644 --- a/rally/cli/commands/verify.py +++ b/rally/cli/commands/verify.py @@ -78,10 +78,13 @@ class VerifyCommands(object): required=False, help="How many processes to use to run Tempest tests. " "The default value (0) auto-detects your CPU count") + @cliutils.args("--failing", dest="failing", required=False, + help="Re-run the tests that failed in the last execution", + action="store_true") @envutils.with_default_deployment(cli_arg_name="deployment") def start(self, deployment=None, set_name="", regex=None, tests_file=None, tempest_config=None, xfails_file=None, - do_use=True, system_wide=False, concur=0): + do_use=True, system_wide=False, concur=0, failing=False): """Start verification (run Tempest tests). :param deployment: UUID or name of a deployment @@ -98,7 +101,9 @@ class VerifyCommands(object): env when running the tests :param concur: How many processes to use to run Tempest tests. The default value (0) auto-detects CPU count + :param failing: Re-run tests that failed during the last execution """ + msg = _("Arguments '%s' and '%s' are not compatible. " "You can use only one of the mentioned arguments.") if regex and set_name: @@ -111,7 +116,7 @@ class VerifyCommands(object): print(msg % ("tests_file", "regex")) return 1 - if not (regex or set_name or tests_file): + if not (regex or set_name or tests_file or failing): set_name = "full" if set_name and set_name not in AVAILABLE_SETS: @@ -124,6 +129,14 @@ class VerifyCommands(object): print(_("File '%s' not found.") % tests_file) return 1 + if failing and set_name: + print(msg % ("failing", "set")) + return 1 + + if failing and tests_file: + print(msg % ("failing", "tests_file")) + return 1 + expected_failures = None if xfails_file: if os.path.exists(xfails_file): @@ -136,7 +149,8 @@ class VerifyCommands(object): verification = api.Verification.verify( deployment, set_name=set_name, regex=regex, tests_file=tests_file, tempest_config=tempest_config, expected_failures=expected_failures, - system_wide=system_wide, concur=concur) + system_wide=system_wide, concur=concur, failing=failing) + if do_use: self.use(verification["uuid"]) diff --git a/rally/verification/tempest/tempest.py b/rally/verification/tempest/tempest.py index c28501a66f..ee2b3e06a9 100644 --- a/rally/verification/tempest/tempest.py +++ b/rally/verification/tempest/tempest.py @@ -297,13 +297,16 @@ class Tempest(object): shutil.rmtree(self.path()) @logging.log_verification_wrapper(LOG.info, _("Run verification.")) - def _prepare_and_run(self, set_name, regex, tests_file, concur): + def _prepare_and_run(self, set_name, regex, tests_file, concur, failing): if not self.is_configured(): self.generate_config_file() testr_args = "--concurrency %d" % concur - if set_name: + if failing: + testr_args += " --failing" + set_name = "re-run-failed" + elif set_name: if set_name == "full": pass elif set_name in consts.TempestTestsSets: @@ -392,8 +395,9 @@ class Tempest(object): else: self.verification.set_failed() - def verify(self, set_name, regex, tests_file, expected_failures, concur): - self._prepare_and_run(set_name, regex, tests_file, concur) + def verify(self, set_name, regex, tests_file, expected_failures, concur, + failing): + self._prepare_and_run(set_name, regex, tests_file, concur, failing) self._save_results(expected_failures=expected_failures) def import_results(self, set_name, log_file): diff --git a/tests/unit/cli/commands/test_verify.py b/tests/unit/cli/commands/test_verify.py index 243d90d316..c748416b3b 100644 --- a/tests/unit/cli/commands/test_verify.py +++ b/tests/unit/cli/commands/test_verify.py @@ -57,8 +57,8 @@ class VerifyCommandsTestCase(test.TestCase): mock_verification_verify.assert_called_once_with( deployment_id, set_name="full", regex=None, tests_file=None, - tempest_config=None, expected_failures=None, system_wide=False, - concur=0) + tempest_config=None, expected_failures=None, + system_wide=False, concur=0, failing=False) @mock.patch("rally.osclients.Clients") @mock.patch("rally.api.Verification.verify") @@ -76,7 +76,7 @@ class VerifyCommandsTestCase(test.TestCase): mock_verification_verify.assert_called_once_with( deployment_id, set_name="full", regex=None, tests_file=None, tempest_config=tempest_config.name, expected_failures=None, - system_wide=False, concur=0) + system_wide=False, concur=0, failing=False) tempest_config.close() @mock.patch("rally.api.Verification.verify") @@ -91,7 +91,7 @@ class VerifyCommandsTestCase(test.TestCase): mock_verification_verify.assert_called_once_with( deployment_id, set_name="", regex=None, tests_file=tests_file, tempest_config=None, expected_failures=None, system_wide=False, - concur=0) + concur=0, failing=False) @mock.patch("rally.api.Verification.verify") @mock.patch("six.moves.builtins.open", @@ -107,7 +107,7 @@ class VerifyCommandsTestCase(test.TestCase): mock_verification_verify.assert_called_once_with( deployment_id, set_name="full", regex=None, tests_file=None, tempest_config=None, expected_failures={"test": "reason of fail"}, - system_wide=False, concur=0) + system_wide=False, concur=0, failing=False) @mock.patch("rally.api.Verification.verify") def test_start_with_wrong_set_name(self, mock_verification_verify): @@ -121,6 +121,28 @@ class VerifyCommandsTestCase(test.TestCase): consts.TempestTestsAPI) self.assertFalse(mock_verification_verify.called) + @mock.patch("rally.api.Verification.verify") + def test_start_with_failing_and_set_name(self, mock_verification_verify): + deployment_id = "f2009aae-6ef3-468e-96b2-3c987d584010" + + set_name = "some_value" + self.verify.start(set_name=set_name, deployment=deployment_id, + do_use=False, failing=True) + + self.assertFalse(mock_verification_verify.called) + + @mock.patch("rally.api.Verification.verify") + @mock.patch("os.path.exists", return_value=True) + def test_start_with_failing_and_test_files(self, mock_exists, + mock_verification_verify): + deployment_id = "f2009aae-6ef3-468e-96b2-3c987d584010" + tests_file = "/path/to/tests/file" + + self.verify.start(tests_file=tests_file, deployment=deployment_id, + do_use=False, failing=True) + + self.assertFalse(mock_verification_verify.called) + @mock.patch("rally.api.Verification.import_results") def test_import_results(self, mock_verification_import_results): deployment_id = "fake_uuid" diff --git a/tests/unit/test_api.py b/tests/unit/test_api.py index cb6b2de3de..b3437f65f6 100644 --- a/tests/unit/test_api.py +++ b/tests/unit/test_api.py @@ -410,7 +410,7 @@ class VerificationAPITestCase(BaseDeploymentTestCase): self.tempest.is_installed.assert_called_once_with() self.tempest.verify.assert_called_once_with( set_name="smoke", regex=None, tests_file=None, - expected_failures=None, concur=0) + expected_failures=None, concur=0, failing=False) @mock.patch("rally.api.objects.Deployment.get") @mock.patch("rally.api.objects.Verification") @@ -424,12 +424,11 @@ class VerificationAPITestCase(BaseDeploymentTestCase): api.Verification.verify( self.deployment_uuid, set_name="smoke", regex=None, tests_file=None, tempest_config=None) - self.tempest.is_installed.assert_called_once_with() self.tempest.install.assert_called_once_with() self.tempest.verify.assert_called_once_with( set_name="smoke", regex=None, tests_file=None, - expected_failures=None, concur=0) + expected_failures=None, concur=0, failing=False) @mock.patch("os.path.exists", return_value=True) @mock.patch("rally.api.objects.Deployment.get") @@ -443,11 +442,11 @@ class VerificationAPITestCase(BaseDeploymentTestCase): tests_file = "/path/to/tests/file" api.Verification.verify( self.deployment_uuid, set_name="", regex=None, - tests_file=tests_file, tempest_config=None) + tests_file=tests_file, tempest_config=None, failing=False) self.tempest.verify.assert_called_once_with( set_name="", regex=None, tests_file=tests_file, - expected_failures=None, concur=0) + expected_failures=None, concur=0, failing=False) @mock.patch("rally.common.objects.Deployment.get") @mock.patch("rally.api.objects.Verification") diff --git a/tests/unit/verification/test_tempest.py b/tests/unit/verification/test_tempest.py index ad43ce62c7..9331dfe943 100644 --- a/tests/unit/verification/test_tempest.py +++ b/tests/unit/verification/test_tempest.py @@ -358,7 +358,7 @@ class TempestVerifyTestCase(BaseTestCase): set_name = "compute" fake_call = self._get_fake_call("tempest.api.%s" % set_name) - self.verifier.verify(set_name, None, None, None, 0) + self.verifier.verify(set_name, None, None, None, 0, False) self.assertEqual(2, mock_tempest_is_configured.call_count) mock_tempest_config.assert_called_once_with(self.verifier.deployment) @@ -388,7 +388,7 @@ class TempestVerifyTestCase(BaseTestCase): set_name = "identity" fake_call = self._get_fake_call("tempest.api.%s" % set_name) - self.verifier.verify(set_name, None, None, None, 0) + self.verifier.verify(set_name, None, None, None, 0, False) mock_tempest_is_configured.assert_called_once_with() self.assertFalse(mock_tempest_config.called) @@ -417,7 +417,7 @@ class TempestVerifyTestCase(BaseTestCase): fake_call = self._get_fake_call("tempest.api.%s" % set_name) mock_subprocess.side_effect = subprocess.CalledProcessError - self.verifier.verify(set_name, None, None, None, 0) + self.verifier.verify(set_name, None, None, None, 0, False) mock_tempest_is_configured.assert_called_once_with() self.assertFalse(mock_tempest_config.called) @@ -444,7 +444,7 @@ class TempestVerifyTestCase(BaseTestCase): tests_file = "/path/to/tests/file" fake_call = self._get_fake_call("--load-list %s" % tests_file) - self.verifier.verify("", None, tests_file, None, 0) + self.verifier.verify("", None, tests_file, None, 0, False) self.verifier.verification.start_verifying.assert_called_once_with("") mock_subprocess.check_call.assert_called_once_with( @@ -452,6 +452,28 @@ class TempestVerifyTestCase(BaseTestCase): shell=True) mock_tempest_parse_results.assert_called_once_with(None, None) + @mock.patch(TEMPEST_PATH + ".tempest.Tempest.parse_results", + return_value=(None)) + @mock.patch(TEMPEST_PATH + ".tempest.Tempest.env") + @mock.patch(TEMPEST_PATH + ".tempest.subprocess") + @mock.patch(TEMPEST_PATH + ".config.TempestResourcesContext") + @mock.patch(TEMPEST_PATH + ".tempest.Tempest.is_configured", + return_value=True) + def test_verify_run_failed_tests_(self, mock_tempest_is_configured, + mock_tempest_resources_context, + mock_subprocess, mock_tempest_env, + mock_tempest_parse_results): + fake_call = self._get_fake_call("--failing") + self.verifier.verify("", None, None, None, 0, True) + + self.verifier.verification.start_verifying.assert_called_once_with( + "re-run-failed") + + mock_subprocess.check_call.assert_called_once_with( + fake_call, env=mock_tempest_env, cwd=self.verifier.path(), + shell=True) + mock_tempest_parse_results.assert_called_once_with(None, None) + def test_import_results(self): set_name = "identity" log_file = "log_file"