pre-commit: Migrate to ruff, enable autopep8

This allows us to replace bandit and flake8. It also ensures we run
pyupgrade-like checks continuously to prevent adding Python < 3.10 style
syntax in the future.

Long-term, we should look at replacing autopep8 with ruff, but that's a
different discussion :)

Change-Id: I12f058d95a0745d895b3257dbaa4e866835c7008
Signed-off-by: Stephen Finucane <stephenfin@redhat.com>

.pre-commit-config.yaml

@@ -17,6 +17,11 @@ repos:
       - id: check-yaml
         files: .*\.(yaml|yml)$
         exclude: 'rally-jobs/task-neutron.yaml'
+  - repo: https://github.com/lucas-c/pre-commit-hooks
+    rev: v1.5.5
+    hooks:
+      - id: remove-tabs
+        exclude: '.*\.(svg)$'
   - repo: https://opendev.org/openstack/bashate.git
     rev: 2.1.1
     hooks:
@@ -28,27 +33,24 @@ repos:
         args: ['-v', '-iE006', '-iE005,E042,E043']
         files: .*\.sh
         exclude: '.tox/.*'
-  - repo: https://github.com/PyCQA/bandit
+  - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 1.8.3
+    rev: v0.12.1
     hooks:
-      - id: bandit
+      - id: ruff-check
-        # B104: Possible binding to all interfaces
+        args: []
-        args: ['-n5', '-sB104']
-        files: 'neutron/'
-        exclude: 'neutron/tests'
-  - repo: https://github.com/lucas-c/pre-commit-hooks
-    rev: v1.5.5
-    hooks:
-      - id: remove-tabs
-        exclude: '.*\.(svg)$'
   - repo: https://opendev.org/openstack/hacking
     rev: 7.0.0
     hooks:
       - id: hacking
         additional_dependencies: ['neutron', 'neutron-lib']
         exclude: '^(doc|releasenotes|tools)/.*$'
+  - repo: https://github.com/hhatto/autopep8
+    rev: v2.3.2
+    hooks:
+      - id: autopep8
+        files: '^.*\.py$'
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.15.0
+    rev: v1.16.1
     hooks:
       - id: mypy
         pass_filenames: false
@@ -56,15 +58,48 @@ repos:
         # necessary to detect one of 'ignored' errors and complains about
         # unnecessary ignore; other dependencies can be left out it seems
         additional_dependencies: ['alembic']
+  - repo: https://github.com/PyCQA/doc8
+    rev: v2.0.0
+    hooks:
+      - id: doc8
+        exclude: |
+          (?x)(
+            doc/source/admin/config-qos-min-pps.rst
+            | doc/source/admin/deploy-provider-verifynetworkoperation.txt
+            | doc/source/admin/deploy-selfservice-verifynetworkoperation.txt
+            | doc/source/admin/shared/deploy-ha-vrrp-initialnetworks.txt
+            | doc/source/admin/shared/deploy-ha-vrrp-verifynetworkoperation.txt
+            | doc/source/admin/shared/deploy-provider-initialnetworks.txt
+            | doc/source/configuration/metering-agent.rst
+            | doc/source/contributor/internals/images
+            | doc/source/contributor/policies/bugs.rst
+          )
+        files: |
+          (?x)(
+            doc/source/.*
+            | neutron/.*
+            | CONTRIBUTING.rst
+            | README.rst
+            | TESTING.rst
+          )
   - repo: local
     hooks:
-      - id: flake8
+      - id: misc-sanity-checks
-        name: flake8
+        name: misc sanity checks
-        additional_dependencies:
+        language: script
-          - hacking>=6.1.0,<6.2.0
+        require_serial: true
-          - neutron
+        pass_filenames: false
-        language: python
+        entry: './tools/misc-sanity-checks.sh'
-        entry: flake8
+        files: '^.*\.py$'
+        exclude: '^(doc|releasenotes|tools)/.*$'
+  - repo: local
+    hooks:
+      - id: check-unit-test-structure
+        name: check unit test structure
+        language: script
+        require_serial: true
+        pass_filenames: false
+        entry: './tools/check_unit_test_structure.sh'
         files: '^.*\.py$'
         exclude: '^(doc|releasenotes|tools)/.*$'
 # todo(slaweq): enable pylint check once all issues in the current code will

neutron/cmd/ovn/ml2ovn_trace.py

@@ -68,7 +68,7 @@ class OvnTrace:
         return ('ovn-trace', *self.extra_args, self.microflow)
 
     def run(self):
-        return subprocess.run(self.args, check=True)  # nosec
+        return subprocess.run(self.args, check=True)  # noqa: S603
 
     def __str__(self):
         return " ".join(self.args[:-1] + ("'%s'" % self.args[-1],))

neutron/tests/fullstack/base.py

@@ -24,6 +24,7 @@ from neutron_lib.tests import tools
 from oslo_config import cfg
 from oslo_log import log as logging
 
+from neutron._i18n import _
 from neutron.agent.linux import ip_lib
 from neutron.common import utils as common_utils
 from neutron.conf.agent import common as config

neutron/tests/functional/db/test_migrations.py

@@ -505,7 +505,7 @@ class TestWalkDowngrade(oslotest_base.BaseTestCase):
             return True
 
 
-class _BaseTestWalkMigrations(object):
+class _BaseTestWalkMigrations:
 
     BUILD_SCHEMA = False
 

pyproject.toml

@@ -17,3 +17,27 @@ files = "neutron"
 # Finally, when the whole repo is migrated this option can be deleted
 # and rules applied to the whole repo.
 exclude = "(?x)(^neutron/tests/$)"
+
+[tool.ruff]
+line-length = 79
+target-version = "py310"
+
+[tool.ruff.lint]
+select = ["E4", "E7", "E9", "F", "S", "U"]
+ignore = [
+    "E731",  # Do not assign a `lambda` expression, use a `def`
+    "E741",  # Ambiguous variable name: `l`
+    "S104",  # Possible binding to all interfaces
+    "UP031", # Use format specifiers instead of percent format
+    "UP032", # Use f-string instead of `format` call
+    # FIXME(stephenfin): These can all be auto-fixed by ruff and
+    # should be enabled
+    "UP018",  # Unnecessary `int` call (rewrite as a literal)
+    "UP024",  # Replace aliased errors with `OSError`
+    "UP030",  # Use implicit references for positional format fields
+    "UP038",  # Use `X | Y` in `isinstance` call instead of `(X, Y)`
+    "UP039",  # Unnecessary parentheses after class definition
+]
+
+[tool.ruff.lint.per-file-ignores]
+"neutron/tests/*" = ["S"]

rally-jobs/plugins/README.rst

@@ -1,7 +1,7 @@
 Rally plugins
 =============
 
-All *.py modules from this directory will be auto-loaded by Rally and all
+All ``*.py`` modules from this directory will be auto-loaded by Rally and all
 plugins will be discoverable. There is no need of any extra configuration
 and there is no difference between writing them here and in rally code base.
 

tox.ini

@@ -122,10 +122,6 @@ commands =
   stestr run --slowest --concurrency 2 --exclude-regex neutron.tests.fullstack.test_securitygroup.TestSecurityGroupsSameNetwork.test_securitygroup {posargs}
   stestr run --slowest --combine --concurrency 1 neutron.tests.fullstack.test_securitygroup.TestSecurityGroupsSameNetwork.test_securitygroup {posargs}
 
-[testenv:mypy]
-deps = {[testenv:pep8]deps}
-commands = mypy
-
 [testenv:releasenotes]
 description =
     Build release note documentation in HTML format.
@@ -137,40 +133,14 @@ description =
     Run style and lint checks.
 deps =
   {[testenv]deps}
-  bashate>=2.1.1 # Apache-2.0
+  pylint~=3.3 # GPLv2
-  bandit>=1.8.3 # Apache-2.0
+  pre-commit~=4.0 # MIT
-  flake8-import-order>=0.18.2,<0.19.0 # LGPLv3
+commands =
-  pylint==3.3.6 # GPLv2
+  pre-commit run --all-files --show-diff-on-failure
-  mypy==1.15.0
+  pylint --rcfile=.pylintrc --output-format=colorized neutron
-commands=
-  # If it is easier to add a check via a shell script, consider adding it in this file
-  bash ./tools/misc-sanity-checks.sh
-  bash {toxinidir}/tools/check_unit_test_structure.sh
-  # Checks for coding and style guidelines
-  flake8
-  bash ./tools/coding-checks.sh --pylint '{posargs}'
   neutron-db-manage --config-file neutron/tests/etc/neutron.conf check_migration
-  # RST linter - remove the ignores once files are updated
-  doc8 \
-  --ignore-path doc/source/admin/config-qos-min-pps.rst \
-  --ignore-path doc/source/admin/deploy-provider-verifynetworkoperation.txt \
-  --ignore-path doc/source/admin/deploy-selfservice-verifynetworkoperation.txt \
-  --ignore-path doc/source/admin/shared/deploy-ha-vrrp-initialnetworks.txt \
-  --ignore-path doc/source/admin/shared/deploy-ha-vrrp-verifynetworkoperation.txt \
-  --ignore-path doc/source/admin/shared/deploy-provider-initialnetworks.txt \
-  --ignore-path doc/source/configuration/metering-agent.rst \
-  --ignore-path doc/source/contributor/internals/images \
-  --ignore-path doc/source/contributor/policies/bugs.rst \
-  doc/source neutron CONTRIBUTING.rst README.rst TESTING.rst
   {[testenv:genconfig]commands}
-  {[testenv:bashate]commands}
-  {[testenv:mypy]commands}
-  {[testenv:bandit]commands}
   {[testenv:genpolicy]commands}
-allowlist_externals = bash
-
-[doc8]
-max-line-length = 79
 
 [testenv:cover]
 description =
@@ -187,8 +157,8 @@ commands =
 
 [testenv:venv]
 description =
-    Run specified command in a virtual environment with all dependencies
+  Run specified command in a virtual environment with all dependencies
-    installed.
+  installed.
 deps =
   -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
   -r{toxinidir}/doc/requirements.txt
@@ -210,7 +180,8 @@ commands = sphinx-build -W -b html doc/source doc/build/html
 [testenv:pdf-docs]
 description =
     Build documentation in PDF format.
-deps = {[testenv:docs]deps}
+deps =
+  {[testenv:docs]deps}
 allowlist_externals =
   make
 commands =
@@ -224,26 +195,11 @@ deps = -r{toxinidir}/doc/requirements.txt
 commands = sphinx-build -W -b linkcheck doc/source doc/build/linkcheck
 
 [flake8]
-# E126 continuation line over-indented for hanging indent
+# We only enable the hacking (H) and neutron (N) checks
-# E128 continuation line under-indented for visual indent
+select = H,N
-# E231 missing whitespace after ','
+ignore = H405,H701,H702,H703,N530
-# E275 missing whitespace after keyword
-# H405 multi line docstring summary not separated with an empty line
-# I202 Additional newline in a group of imports
-# N530 direct neutron imports not allowed
-# TODO(amotoki) check the following new rules should be fixed or ignored
-# E731 do not assign a lambda expression, use a def
-# W504 line break after binary operator
-ignore = E126,E128,E231,E275,E731,I202,H405,N530,W504
-# H106: Don't put vim configuration in source files
-# H203: Use assertIs(Not)None to check for None
-# H204: Use assert(Not)Equal to check for equality
-# H205: Use assert(Greater|Less)(Equal) for comparison
-# H904: Delay string interpolations at logging calls
-enable-extensions=H106,H203,H204,H205,H904
 show-source = true
 exclude = ./.*,build,dist,doc
-import-order-style = pep8
 
 [flake8:local-plugins]
 extension =
@@ -271,36 +227,16 @@ extension =
   N536 = neutron_lib.hacking.checks:assert_equal_none
   N537 = neutron_lib.hacking.translation_checks:no_translate_logs
 
+[doc8]
+max-line-length = 79
+
 [hacking]
 import_exceptions = neutron._i18n
 
-[testenv:bandit]
-description =
-    Run bandit security checks.
-deps = {[testenv:pep8]deps}
-# B104: Possible binding to all interfaces
-# B311: Standard pseudo-random generators are not suitable for
-#       security/cryptographic purposes
-commands = bandit -r neutron -x tests -n5 -s B104,B311
-
-[testenv:bashate]
-description =
-    Run bashate checks.
-deps = {[testenv:pep8]deps}
-commands = bash -c "find {toxinidir}             \
-         -not \( -type d -name .tox\* -prune \)  \
-         -not \( -type d -name .venv\* -prune \) \
-         -type f                                 \
-         -name \*.sh                             \
-# E005 file does not begin with #! or have a .sh prefix
-# E006 check for lines longer than 79 columns
-# E042 local declaration hides errors
-# E043 Arithmetic compound has inconsistent return semantics
-         -print0 | xargs -0 bashate -v -iE006 -eE005,E042,E043"
-
 [testenv:genconfig]
 description =
     Generate configuration example files.
+allowlist_externals = bash
 commands = bash {toxinidir}/tools/generate_config_file_samples.sh
 
 [testenv:genpolicy]