openstack/rally
Code Issues Proposed changes

[Verify] Introduce rally verify rerun cmd

Browse Source 
This command can re-run either all tests or only failed ones
from the specified verification.

Change-Id: I89ebfb3fa3ab87f66cdffdf02d204efa0ac96fee
This commit is contained in:
Yaroslav Lobankov 2017-01-09 16:21:59 +04:00
parent 3e565cb380
commit ef2cd31a65
5 changed files with 127 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
etc/rally.bash_completion
View File

@ -55,8 +55,9 @@ _rally()
OPTS["verify_list-verifier-tests"]="--id --pattern"
OPTS["verify_list-verifiers"]="--status"
OPTS["verify_report"]="--uuid --html --file --open"
OPTS["verify_rerun"]="--uuid --deployment-id --failed"
OPTS["verify_show"]="--uuid --sort-by --detailed"
OPTS["verify_start"]="--verifier-id --deployment-id --pattern --concurrency --load-list --skip-list --xfail-list --failed --no-use"
OPTS["verify_start"]="--verifier-id --deployment-id --pattern --concurrency --load-list --skip-list --xfail-list --no-use"
OPTS["verify_update-verifier"]="--id --update-venv --version --system-wide --no-system-wide"
OPTS["verify_use"]="--uuid"
OPTS["verify_use-verifier"]="--id"

31
rally/api.py
View File

@ -832,6 +832,37 @@ class _Verification(object):
return verification, results
@classmethod
def rerun(cls, verification_uuid, deployment_id=None, failed=False):
"""Re-run tests from a verification.
:param verification_uuid: Verification UUID
:param deployment_id: Deployment name or UUID
:param failed: Re-run only failed tests
"""
# TODO(ylobankov): Improve this method in the future: put some
# information about re-run in run_args.
verification = cls.get(verification_uuid)
tests = verification.tests
if failed:
tests = [t for t, r in tests.items() if r["status"] == "fail"]
if not tests:
raise exceptions.RallyException(
"There are no failed tests from verification (UUID=%s)."
% verification_uuid)
else:
tests = tests.keys()
deployment = _Deployment.get(deployment_id or
verification.deployment_uuid)
LOG.info("Re-running %stests from verification (UUID=%s) for "
"deployment '%s' (UUID=%s).", "failed " if failed else "",
verification.uuid, deployment["name"], deployment["uuid"])
return cls.start(verification.verifier_uuid, deployment["uuid"],
load_list=tests)
@staticmethod
def get(verification_uuid):
"""Get a verification.

42
rally/cli/commands/verify.py
View File

@ -382,9 +382,6 @@ class VerifyCommands(object):
"considered as expected failures. "
"Format: json or yaml like a dictionary where keys "
"are test names and values are reasons.")
@cliutils.args("--failed", dest="failed", required=False,
help="Re-run tests that failed in the last verification.",
action="store_true")
@cliutils.args("--no-use", dest="do_use", action="store_false",
help="Not to set the finished verification as the default "
"verification for future operations.")
@ -393,19 +390,13 @@ class VerifyCommands(object):
@plugins.ensure_plugins_are_loaded
def start(self, api, verifier_id=None, deployment=None, pattern=None,
concur=0, load_list=None, skip_list=None, xfail_list=None,
failed=False, do_use=True):
do_use=True):
"""Start a verification (run verifier tests)."""
incompatible_args_map = [{"load-list": load_list, "pattern": pattern},
{"failed": failed, "pattern": pattern},
{"failed": failed, "load-list": load_list},
{"failed": failed, "skip-list": skip_list}]
msg = _("Arguments '--%s' and '--%s' cannot be used simultaneously. "
"You can use only one of the mentioned arguments.")
for args in incompatible_args_map:
args_keys = list(args)
if args[args_keys[0]] and args[args_keys[1]]:
print(msg % (args_keys[0], args_keys[1]))
return 1
if pattern and load_list:
print(_("Arguments '--pattern' and '--load-list' cannot be used "
"simultaneously. You can use only one of the mentioned "
"arguments."))
return 1
def parse(filename):
with open(filename, "r") as f:
@ -433,7 +424,7 @@ class VerifyCommands(object):
run_args = {key: value for key, value in (
("pattern", pattern), ("load_list", load_list),
("skip_list", skip_list), ("xfail_list", xfail_list),
("concurrency", concur), ("failed", failed)) if value}
("concurrency", concur)) if value}
verification, results = api.verification.start(verifier_id, deployment,
**run_args)
@ -455,6 +446,25 @@ class VerifyCommands(object):
print(_("Using verification (UUID=%s) as the default verification "
"for the future operations.") % verification.uuid)
@cliutils.help_group("verification")
@cliutils.args("--uuid", dest="verification_uuid", type=str,
help="Verification UUID. " + LIST_VERIFICATIONS_HINT)
@cliutils.args("--deployment-id", dest="deployment", type=str,
metavar="<id>",
help="Deployment name or UUID. " + LIST_DEPLOYMENTS_HINT)
@cliutils.args("--failed", dest="failed", required=False,
help="Re-run only failed tests.", action="store_true")
@envutils.with_default_verification_uuid
@envutils.with_default_deployment(cli_arg_name="deployment-id")
@plugins.ensure_plugins_are_loaded
def rerun(self, api, verification_uuid=None, deployment=None,
failed=False):
"""Re-run tests from a verification for a specific deployment."""
verification, results = api.verification.rerun(verification_uuid,
deployment, failed)
self._print_totals(results.totals)
print(_("Verification UUID: %s.") % verification.uuid)
@cliutils.help_group("verification")
@cliutils.args("--uuid", dest="verification_uuid", type=str,
help="Verification UUID. " + LIST_VERIFICATIONS_HINT)

17
tests/unit/cli/commands/test_verify.py
View File

@ -170,7 +170,7 @@ class VerifyCommandsTestCase(test.TestCase):
@mock.patch("rally.cli.commands.verify.os.path.exists")
def test_start(self, mock_exists, mock_update_globals_file):
self.verify.start(self.fake_api, "v_id", "d_id", pattern="pattern",
failed=True)
load_list="load-list")
self.assertFalse(self.fake_api.verification.start.called)
mock_exists.return_value = False
@ -240,6 +240,21 @@ class VerifyCommandsTestCase(test.TestCase):
mock_update_globals_file.assert_called_once_with(
envutils.ENV_VERIFICATION, "v_uuid")
def test_rerun(self):
verification = mock.Mock(uuid="v_uuid")
results = mock.Mock(totals={"tests_count": 2,
"tests_duration": 4,
"success": 2,
"skipped": 0,
"expected_failures": 0,
"unexpected_success": 0,
"failures": 0})
self.fake_api.verification.rerun.return_value = (verification, results)
self.verify.rerun(self.fake_api, "v_uuid", "d_id", failed=True)
self.fake_api.verification.rerun.assert_called_once_with("v_uuid",
"d_id", True)
def test_show(self):
deployment_name = "Some Deploy"
verifier_name = "My Verifier"

52
tests/unit/test_api.py
View File

@ -1295,3 +1295,55 @@ class VerificationAPITestCase(test.TestCase):
self.assertFalse(verification_obj.finish.called)
self.assertFalse(mock_configure.called)
@mock.patch("rally.api._Verification.start")
@mock.patch("rally.api._Deployment.get")
@mock.patch("rally.api._Verification.get")
def test_rerun(self, mock___verification_get, mock___deployment_get,
mock___verification_start):
tests = {"test_1": {"status": "success"},
"test_2": {"status": "fail"}}
mock___verification_get.return_value = mock.Mock(
uuid="uuid", verifier_uuid="v_uuid", deployment_uuid="d_uuid",
tests=tests)
mock___deployment_get.return_value = {"name": "d_name",
"uuid": "d_uuid"}
api._Verification.rerun("uuid")
mock___verification_start.assert_called_once_with(
"v_uuid", "d_uuid", load_list=tests.keys())
@mock.patch("rally.api._Verification.start")
@mock.patch("rally.api._Deployment.get")
@mock.patch("rally.api._Verification.get")
def test_rerun_failed_tests(
self, mock___verification_get, mock___deployment_get,
mock___verification_start):
tests = {"test_1": {"status": "success"},
"test_2": {"status": "fail"},
"test_3": {"status": "fail"}}
mock___verification_get.return_value = mock.Mock(
uuid="uuid", verifier_uuid="v_uuid", deployment_uuid="d_uuid",
tests=tests)
mock___deployment_get.return_value = {"name": "d_name",
"uuid": "d_uuid"}
api._Verification.rerun("uuid", failed=True)
expected_tests = [t for t, r in tests.items() if r["status"] == "fail"]
mock___verification_start.assert_called_once_with(
"v_uuid", "d_uuid", load_list=expected_tests)
@mock.patch("rally.api._Verification.get")
def test_rerun_failed_tests_raise_exc(
self, mock___verification_get):
tests = {"test_1": {"status": "success"},
"test_2": {"status": "success"},
"test_3": {"status": "skip"}}
mock___verification_get.return_value = mock.Mock(
uuid="uuid", verifier_uuid="v_uuid", deployment_uuid="d_uuid",
tests=tests)
e = self.assertRaises(exceptions.RallyException,
api._Verification.rerun, "uuid", failed=True)
self.assertEqual("There are no failed tests from verification "
"(UUID=uuid).", "%s" % e)