05fcfef6ce
During a mailing list discussion on some OOM issues neutron has been seeing [0], Mike Bayer recommended we should change from using subquery to selectin DB load strategy. A full description of this strategy can be found here [1], but in short: - “subquery” loading incurs additional performance / complexity issues when used on a many-levels-deep eager load, as subqueries will be nested repeatedly. - "The subqueryload() eager loader is mostly legacy at this point, superseded by selectinload() - "The only scenario in which selectin eager loading is not feasible is when the model is using composite primary keys, and the backend database does not support tuples with IN, which currently includes SQL Server." So that does not apply to us. The plan agreed to at the neutron drivers meeting [2] was to make this change early in the cycle so we would be able to see if there were any issues through the D cycle. Added hacking checks so new code using subquery loads is not added back. [0] https://lists.openstack.org/archives/list/openstack-discuss@lists.openstack.org/thread/EHLQQXNG3NLLZYPDGG2ES3DINIJ7YT3N/ [1] https://docs.sqlalchemy.org/en/20/orm/queryguide/relationships.html#selectin-eager-loading [2] https://meetings.opendev.org/meetings/neutron_drivers/2024/neutron_drivers.2024-05-31-14.00.log.html#l-67 Closes-bug: #2067770 Depends-on: https://review.opendev.org/c/openstack/neutron-lib/+/920936 Change-Id: I6e40a15284da392a3d48d45205a7a5770c14c297
308 lines
11 KiB
INI
308 lines
11 KiB
INI
[tox]
|
|
envlist = docs,py3,pep8
|
|
minversion = 3.18.0
|
|
skipsdist = False
|
|
ignore_basepython_conflict = True
|
|
|
|
[testenv]
|
|
description =
|
|
Run unit tests.
|
|
basepython = {env:TOX_PYTHON:python3}
|
|
setenv = VIRTUAL_ENV={envdir}
|
|
OS_LOG_CAPTURE={env:OS_LOG_CAPTURE:true}
|
|
OS_STDOUT_CAPTURE={env:OS_STDOUT_CAPTURE:true}
|
|
OS_STDERR_CAPTURE={env:OS_STDERR_CAPTURE:true}
|
|
OS_TEST_TIMEOUT={env:OS_TEST_TIMEOUT:180}
|
|
PYTHONWARNINGS=default::DeprecationWarning,ignore::DeprecationWarning:distutils,ignore::DeprecationWarning:site
|
|
passenv = TRACE_FAILONLY
|
|
GENERATE_HASHES
|
|
http_proxy
|
|
HTTP_PROXY
|
|
https_proxy
|
|
HTTPS_PROXY
|
|
no_proxy
|
|
NO_PROXY
|
|
TOX_ENV_SRC_MODULES
|
|
usedevelop = True
|
|
deps =
|
|
-c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
|
|
-r{toxinidir}/requirements.txt
|
|
-r{toxinidir}/test-requirements.txt
|
|
hacking>=6.1.0,<6.2.0 # Apache-2.0
|
|
allowlist_externals = bash
|
|
commands =
|
|
bash {toxinidir}/tools/pip_install_src_modules.sh "{toxinidir}"
|
|
stestr run {posargs}
|
|
|
|
# there is also secret magic in ostestr which lets you run in a fail only
|
|
# mode. To do this define the TRACE_FAILONLY environmental variable.
|
|
|
|
[testenv:debug]
|
|
description =
|
|
Run specified tests through oslo_debug_helper, which allows use of pdb.
|
|
commands = oslo_debug_helper -t neutron/tests {posargs}
|
|
|
|
[testenv:common]
|
|
# Fake job to define environment variables shared between dsvm/non-dsvm jobs
|
|
setenv = OS_TEST_TIMEOUT={env:OS_TEST_TIMEOUT:180}
|
|
commands = false
|
|
|
|
[testenv:dsvm]
|
|
# Fake job to define environment variables shared between dsvm jobs
|
|
setenv = OS_SUDO_TESTING=1
|
|
OS_ROOTWRAP_CMD=sudo {envdir}/bin/neutron-rootwrap {envdir}/etc/neutron/rootwrap.conf
|
|
OS_ROOTWRAP_DAEMON_CMD=sudo {envdir}/bin/neutron-rootwrap-daemon {envdir}/etc/neutron/rootwrap.conf
|
|
OS_FAIL_ON_MISSING_DEPS=1
|
|
OS_LOG_PATH={env:OS_LOG_PATH:/opt/stack/logs}
|
|
commands = false
|
|
|
|
[testenv:functional]
|
|
description =
|
|
Run functional tests.
|
|
setenv = {[testenv]setenv}
|
|
{[testenv:common]setenv}
|
|
OS_TEST_PATH=./neutron/tests/functional
|
|
OS_LOG_PATH={env:OS_LOG_PATH:/opt/stack/logs}
|
|
# Because of issue with stestr and Python3, we need to avoid too much output
|
|
# to be produced during tests, so we will ignore python warnings here
|
|
PYTHONWARNINGS=ignore
|
|
deps =
|
|
{[testenv]deps}
|
|
-r{toxinidir}/neutron/tests/functional/requirements.txt
|
|
|
|
[testenv:dsvm-functional]
|
|
description =
|
|
Run functional tests that require sudo privileges.
|
|
setenv = {[testenv:functional]setenv}
|
|
{[testenv:dsvm]setenv}
|
|
deps =
|
|
{[testenv:functional]deps}
|
|
commands =
|
|
bash {toxinidir}/tools/deploy_rootwrap.sh {toxinidir} {envdir}/etc {envdir}/bin
|
|
stestr run {posargs}
|
|
|
|
[testenv:dsvm-functional-gate]
|
|
description =
|
|
Run functional gate tests that require sudo privileges.
|
|
setenv = {[testenv:dsvm-functional]setenv}
|
|
deps = {[testenv:dsvm-functional]deps}
|
|
commands =
|
|
bash {toxinidir}/tools/deploy_rootwrap.sh {toxinidir} {envdir}/etc {envdir}/bin
|
|
stestr run --slowest --exclude-regex (.*MySQL\.|.*PostgreSQL\.|.*test_get_all_devices|.*TestMetadataAgent\.|.*BaseOVSTestCase\.) {posargs}
|
|
stestr run --slowest --combine --concurrency 1 (.*MySQL\.|.*PostgreSQL\.|.*test_get_all_devices|.*TestMetadataAgent\.|.*BaseOVSTestCase\.) {posargs}
|
|
|
|
[testenv:dsvm-fullstack]
|
|
description =
|
|
Run fullstack tests.
|
|
setenv = {[testenv]setenv}
|
|
{[testenv:common]setenv}
|
|
{[testenv:dsvm]setenv}
|
|
# workaround for DB teardown lock contention (bug/1541742)
|
|
OS_TEST_TIMEOUT={env:OS_TEST_TIMEOUT:600}
|
|
OS_TEST_PATH=./neutron/tests/fullstack
|
|
# Because of issue with stestr and Python3, we need to avoid too much output
|
|
# to be produced during tests, so we will ignore python warnings here
|
|
PYTHONWARNINGS=ignore
|
|
deps =
|
|
{[testenv:functional]deps}
|
|
commands =
|
|
bash {toxinidir}/tools/generate_dhclient_script_for_fullstack.sh {envdir}
|
|
bash {toxinidir}/tools/deploy_rootwrap.sh {toxinidir} {envdir}/etc {envdir}/bin
|
|
stestr run --concurrency 2 {posargs}
|
|
|
|
[testenv:dsvm-fullstack-gate]
|
|
description =
|
|
Run fullstack gate tests that require sudo privileges.
|
|
setenv = {[testenv:dsvm-fullstack]setenv}
|
|
deps = {[testenv:dsvm-fullstack]deps}
|
|
commands =
|
|
bash {toxinidir}/tools/generate_dhclient_script_for_fullstack.sh {envdir}
|
|
bash {toxinidir}/tools/deploy_rootwrap.sh {toxinidir} {envdir}/etc {envdir}/bin
|
|
stestr run --slowest --concurrency 2 --exclude-regex neutron.tests.fullstack.test_securitygroup.TestSecurityGroupsSameNetwork.test_securitygroup {posargs}
|
|
stestr run --slowest --combine --concurrency 1 neutron.tests.fullstack.test_securitygroup.TestSecurityGroupsSameNetwork.test_securitygroup {posargs}
|
|
|
|
[testenv:releasenotes]
|
|
description =
|
|
Build release note documentation in HTML format.
|
|
deps = -r{toxinidir}/doc/requirements.txt
|
|
commands = sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html
|
|
|
|
[testenv:pep8]
|
|
description =
|
|
Run style and lint checks.
|
|
deps =
|
|
{[testenv]deps}
|
|
bashate>=2.1.1 # Apache-2.0
|
|
bandit>=1.7.5 # Apache-2.0
|
|
flake8-import-order>=0.18.2,<0.19.0 # LGPLv3
|
|
pylint==3.2.0 # GPLv2
|
|
commands=
|
|
# If it is easier to add a check via a shell script, consider adding it in this file
|
|
bash ./tools/misc-sanity-checks.sh
|
|
bash {toxinidir}/tools/check_unit_test_structure.sh
|
|
# Checks for coding and style guidelines
|
|
flake8
|
|
bash ./tools/coding-checks.sh --pylint '{posargs}'
|
|
neutron-db-manage --config-file neutron/tests/etc/neutron.conf check_migration
|
|
{[testenv:genconfig]commands}
|
|
{[testenv:bashate]commands}
|
|
{[testenv:bandit]commands}
|
|
{[testenv:genpolicy]commands}
|
|
allowlist_externals = bash
|
|
|
|
[testenv:cover]
|
|
description =
|
|
Run unit tests and generate coverage report.
|
|
setenv =
|
|
{[testenv]setenv}
|
|
PYTHON=coverage run --source neutron --parallel-mode
|
|
commands =
|
|
stestr run --no-subunit-trace {posargs}
|
|
coverage combine
|
|
coverage report --fail-under=82 --skip-covered
|
|
coverage html -d cover
|
|
coverage xml -o cover/coverage.xml
|
|
|
|
[testenv:venv]
|
|
description =
|
|
Run specified command in a virtual environment with all dependencies
|
|
installed.
|
|
deps =
|
|
-c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
|
|
-r{toxinidir}/doc/requirements.txt
|
|
-r{toxinidir}/requirements.txt
|
|
commands = {posargs}
|
|
|
|
[testenv:docs]
|
|
description =
|
|
Build documentation in HTML format.
|
|
# Do not remove requirements.txt from deps list as without it
|
|
# upper constraints will not be used for deps listed in requirements.txt
|
|
# and may cause issues
|
|
deps =
|
|
-c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
|
|
-r{toxinidir}/doc/requirements.txt
|
|
-r{toxinidir}/requirements.txt
|
|
commands = sphinx-build -W -b html doc/source doc/build/html
|
|
|
|
[testenv:pdf-docs]
|
|
description =
|
|
Build documentation in PDF format.
|
|
deps = {[testenv:docs]deps}
|
|
allowlist_externals =
|
|
make
|
|
commands =
|
|
sphinx-build -W -b latex doc/source doc/build/pdf
|
|
make -C doc/build/pdf
|
|
|
|
[testenv:linkcheck]
|
|
description =
|
|
Build documentation link checks.
|
|
deps = -r{toxinidir}/doc/requirements.txt
|
|
commands = sphinx-build -W -b linkcheck doc/source doc/build/linkcheck
|
|
|
|
[flake8]
|
|
# E126 continuation line over-indented for hanging indent
|
|
# E128 continuation line under-indented for visual indent
|
|
# E231 missing whitespace after ','
|
|
# E275 missing whitespace after keyword
|
|
# H405 multi line docstring summary not separated with an empty line
|
|
# I202 Additional newline in a group of imports
|
|
# N530 direct neutron imports not allowed
|
|
# TODO(amotoki) check the following new rules should be fixed or ignored
|
|
# E731 do not assign a lambda expression, use a def
|
|
# W504 line break after binary operator
|
|
ignore = E126,E128,E231,E275,E731,I202,H405,N530,W504
|
|
# H106: Don't put vim configuration in source files
|
|
# H203: Use assertIs(Not)None to check for None
|
|
# H204: Use assert(Not)Equal to check for equality
|
|
# H205: Use assert(Greater|Less)(Equal) for comparison
|
|
# H904: Delay string interpolations at logging calls
|
|
enable-extensions=H106,H203,H204,H205,H904
|
|
show-source = true
|
|
exclude = ./.*,build,dist,doc
|
|
import-order-style = pep8
|
|
|
|
[flake8:local-plugins]
|
|
extension =
|
|
# Checks specific to neutron repo
|
|
N322 = neutron.hacking.checks:check_assert_called_once_with
|
|
N328 = neutron.hacking.checks:check_asserttruefalse
|
|
N329 = neutron.hacking.checks:check_assertitemsequal
|
|
N330 = neutron.hacking.checks:check_assertempty
|
|
N331 = neutron.hacking.checks:check_assertisinstance
|
|
N332 = neutron.hacking.checks:check_assertequal_for_httpcode
|
|
N340 = neutron.hacking.checks:check_oslo_i18n_wrapper
|
|
N341 = neutron.hacking.checks:check_builtins_gettext
|
|
N343 = neutron.hacking.checks:check_no_imports_from_tests
|
|
N344 = neutron.hacking.checks:check_python3_no_filter
|
|
N346 = neutron.hacking.checks:check_no_sqlalchemy_event_import
|
|
N348 = neutron.hacking.checks:check_no_import_six
|
|
N349 = neutron.hacking.checks:check_no_import_packaging
|
|
N350 = neutron.hacking.checks:check_no_sqlalchemy_lazy_subquery
|
|
# Checks from neutron-lib
|
|
N521 = neutron_lib.hacking.checks:use_jsonutils
|
|
N524 = neutron_lib.hacking.checks:check_no_contextlib_nested
|
|
N529 = neutron_lib.hacking.checks:no_mutable_default_args
|
|
N530 = neutron_lib.hacking.checks:check_neutron_namespace_imports
|
|
N532 = neutron_lib.hacking.translation_checks:check_log_warn_deprecated
|
|
N534 = neutron_lib.hacking.translation_checks:check_raised_localized_exceptions
|
|
N536 = neutron_lib.hacking.checks:assert_equal_none
|
|
N537 = neutron_lib.hacking.translation_checks:no_translate_logs
|
|
|
|
[hacking]
|
|
import_exceptions = neutron._i18n
|
|
|
|
[testenv:bandit]
|
|
description =
|
|
Run bandit security checks.
|
|
deps = {[testenv:pep8]deps}
|
|
# B104: Possible binding to all interfaces
|
|
commands = bandit -r neutron -x tests -n5 -s B104
|
|
|
|
[testenv:bashate]
|
|
description =
|
|
Run bashate checks.
|
|
deps = {[testenv:pep8]deps}
|
|
commands = bash -c "find {toxinidir} \
|
|
-not \( -type d -name .tox\* -prune \) \
|
|
-not \( -type d -name .venv\* -prune \) \
|
|
-type f \
|
|
-name \*.sh \
|
|
# E005 file does not begin with #! or have a .sh prefix
|
|
# E006 check for lines longer than 79 columns
|
|
# E042 local declaration hides errors
|
|
# E043 Arithmetic compound has inconsistent return semantics
|
|
-print0 | xargs -0 bashate -v -iE006 -eE005,E042,E043"
|
|
|
|
[testenv:genconfig]
|
|
description =
|
|
Generate configuration example files.
|
|
commands = bash {toxinidir}/tools/generate_config_file_samples.sh
|
|
|
|
[testenv:genpolicy]
|
|
description =
|
|
Generate policy example files.
|
|
commands = oslopolicy-sample-generator --config-file=etc/oslo-policy-generator/policy.conf
|
|
|
|
# This environment can be used to quickly validate that all needed system
|
|
# packages required to successfully execute test targets are installed
|
|
[testenv:bindep]
|
|
description =
|
|
Run binary dependency checks.
|
|
# Do not install any requirements. We want this to be fast and work even if
|
|
# system dependencies are missing, since it's used to tell you what system
|
|
# dependencies are missing! This also means that bindep must be installed
|
|
# separately, outside of the requirements files.
|
|
deps = bindep
|
|
commands = bindep test
|
|
|
|
[testenv:requirements]
|
|
description =
|
|
Run requirement checks.
|
|
deps =
|
|
-egit+https://opendev.org/openstack/requirements#egg=openstack-requirements
|
|
allowlist_externals = bash
|
|
commands =
|
|
bash -c '{envdir}/src/openstack-requirements/playbooks/files/project-requirements-change.py --req {envdir}/src/openstack-requirements --local {toxinidir} master'
|