diff --git a/doc/source/contribute.rst b/doc/source/contribute.rst index 1510bd4ae8..6af0c5b82d 100644 --- a/doc/source/contribute.rst +++ b/doc/source/contribute.rst @@ -142,7 +142,7 @@ To run a single unit test e.g. test_deployment $ tox -e -- #NOTE: is one of py34, py27 or pep8 - # is the unit test case name + # is the unit test case name, e.g tests.unit.test_osclients To debug issues on the unit test: diff --git a/tests/README.rst b/tests/README.rst index 9d00ca5fd1..0a88662d0a 100644 --- a/tests/README.rst +++ b/tests/README.rst @@ -25,28 +25,32 @@ To run unit tests locally:: $ pip install tox $ tox -To run py26, py27 or pep8 only:: +To run py27, py34, py35 or pep8 only:: $ tox -e - #NOTE: is one of py26, py27 or pep8 + # NOTE: is one of py27, py34, py35 or pep8 -To run py26, py27 against mysql or psql +To run py27/py34/py35 against mysql or psql $ export RALLY_UNITTEST_DB_URL="mysql://user:secret@localhost/rally" $ tox -epy27 +To run specific test of py27/py34/py35:: + + $ tox -e py27 -- tests.unit.test_osclients + To get test coverage:: $ tox -e cover - #NOTE: Results will be in /cover/index.html + # NOTE: Results will be in ./cover/index.html To generate docs:: $ tox -e docs - #NOTE: Documentation will be in doc/source/_build/html/index.html + # NOTE: Documentation will be in doc/source/_build/html/index.html Functional tests ---------------- @@ -62,7 +66,7 @@ To run functional tests locally:: $ rally deployment create --fromenv --name testing $ tox -e cli - #NOTE: openrc file with OpenStack admin credentials + # NOTE: openrc file with OpenStack admin credentials Output of every Rally execution will be collected under some reports root in directory structure like: reports_root/ClassName/MethodName_suffix.extension diff --git a/tests/ci/pytest_launcher.py b/tests/ci/pytest_launcher.py new file mode 100755 index 0000000000..9558dbc368 --- /dev/null +++ b/tests/ci/pytest_launcher.py @@ -0,0 +1,107 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import argparse +import os +import subprocess +import sys + + +PYTEST_REPORT = os.environ.get("PYTEST_REPORT", + ".test_results/pytest_results.html") +TESTR_REPORT = "testr_results.html" + + +def error(msg): + print(msg) + exit(1) + + +def main(args): + parser = argparse.ArgumentParser(args[0]) + parser.add_argument("discovery_path", metavar="", type=str, + help="Path to location of all tests.") + parser.add_argument("--posargs", metavar="", type=str, default="", + help="TOX posargs. Currently supported only string to " + "partial test or tests group to launch.") + args = parser.parse_args(args[1:]) + + # We allow only one parameter - path to partial test or tests group + path = args.posargs + if len(path.split(" ")) > 1: + error("Wrong value of posargs. It should include only path to single " + "test or tests group to launch.") + # NOTE(andreykurilin): Previously, next format was supported: + # tests.unit.test_osclients.SomeTestCase.some_method + # It is more simple and pythonic than native pytest-way: + # tests/unit/test_osclients.py::SomeTestCase::some_method + # Let's return this support + if path: + if "/" not in path: + path = path.split(".") + module = "" + for i in range(0, len(path)): + part = os.path.join(module, path[i]) + if os.path.exists(part): + module = part + continue + if os.path.exists("%s.py" % part): + if i != (len(path) - 1): + module = "%s.py::%s" % (part, "::".join(path[i + 1:])) + else: + module = "%s.py" % part + break + + error("Non-existing path to single test or tests group to " + "launch. %s %s" % (module, part)) + path = module + + path = os.path.abspath(os.path.expanduser(path)) + if not path.startswith(os.path.abspath(args.discovery_path)): + # Prevent to launch functional tests from unit tests launcher. + error("Wrong path to single test or tests group to launch. It " + "should be in %s." % args.discovery_path) + else: + path = args.discovery_path + + print("Test(s) to launch (pytest format): %s" % path) + + # NOTE(andreykurilin): we cannot publish pytest reports at gates, but we + # can mask them as testr reports. It looks like a dirty hack and I + # prefer to avoid it, but I see no other solutions at this point. + + # apply dirty hack only in gates. + if os.environ.get("ZUUL_PROJECT"): + pytest_report = TESTR_REPORT + else: + pytest_report = PYTEST_REPORT + + try: + subprocess.check_call(["py.test", "--html=%s" % pytest_report, + "--durations=10", "-n", "auto", path], + stderr=subprocess.STDOUT) + except subprocess.CalledProcessError: + # NOTE(andreykurilin): it is ok, since tests can fail. + exit_code = 1 + else: + exit_code = 0 + + if os.path.exists(pytest_report) and os.environ.get("ZUUL_PROJECT"): + subprocess.check_call(["gzip", "-9", "-f", pytest_report], + stderr=subprocess.STDOUT) + + if exit_code == 1: + error("") + +if __name__ == "__main__": + main(sys.argv) diff --git a/tests/unit/test_pytest_launcher.py b/tests/unit/test_pytest_launcher.py new file mode 100644 index 0000000000..eda1705451 --- /dev/null +++ b/tests/unit/test_pytest_launcher.py @@ -0,0 +1,76 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +import mock + +from tests.ci import pytest_launcher +from tests.unit import test + + +PATH = "tests.ci.pytest_launcher" + + +class ExitError(Exception): + pass + + +class PyTestLauncherTestCase(test.TestCase): + def setUp(self): + super(PyTestLauncherTestCase, self).setUp() + + sp_patcher = mock.patch("%s.subprocess" % PATH) + self.sp = sp_patcher.start() + self.addCleanup(sp_patcher.stop) + + exit_patcher = mock.patch("%s.exit" % PATH, side_effect=ExitError) + self.exit = exit_patcher.start() + self.addCleanup(exit_patcher.stop) + + os_patcher = mock.patch("%s.os" % PATH) + self.os = os_patcher.start() + self.addCleanup(os_patcher.stop) + # emulate local run by default + self.os.environ = {} + self.os.path.join.side_effect = os.path.join + self.os.path.abspath.side_effect = os.path.abspath + self.os.path.expanduser.side_effect = os.path.expanduser + + def test_wrong_posargs(self): + self.assertRaises(ExitError, pytest_launcher.main, + ["script name", "test_path", + "--posargs='posargs with spaces'"]) + + self.assertFalse(self.sp.called) + self.assertFalse(self.os.called) + + def test_parsing_path(self): + def os_path_exists(path): + dpath = "some/path/to/some/test" + return dpath.startswith(path) or path == "%s/module.py" % dpath + + self.os.path.exists.side_effect = os_path_exists + + pytest_launcher.main( + ["script_name", "some/path", + "--posargs=some.path.to.some.test.module.TestCase.test"]) + + expected_path = os.path.abspath( + "some/path/to/some/test/module.py::TestCase::test") + + self.assertEqual(1, self.sp.check_call.call_count) + call_args_obj = self.sp.check_call.call_args_list[0] + call_args = call_args_obj[0] + self.assertEqual(expected_path, call_args[0][-1]) \ No newline at end of file diff --git a/tox.ini b/tox.ini index 440577254a..4fc9241ad9 100644 --- a/tox.ini +++ b/tox.ini @@ -19,7 +19,7 @@ install_command = pip install -U {opts} {packages} usedevelop = True commands = find . -type f -name "*.pyc" -delete - py.test --html=.test_results/pytest_results.html --durations=10 -n auto "tests/unit" {posargs} + python {toxinidir}/tests/ci/pytest_launcher.py tests/unit --posargs={posargs} distribute = false basepython = python2.7 passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY @@ -55,8 +55,7 @@ commands = oslo_debug_helper -t tests {posargs} sitepackages = True commands = find . -type f -name "*.pyc" -delete - py.test --html=pytest_results.html --durations=10 -n auto "tests/functional" {posargs} - + python {toxinidir}/tests/ci/pytest_launcher.py "tests/functional" --posargs={posargs} [testenv:cover] commands = {toxinidir}/tests/ci/cover.sh {posargs}