Migrate lint to pre-commit

Migrate all existing linters to pre-commit. This consolodates our bandit
and codespell job into the general pep8 job.

Change-Id: I6b40a3338d98fab500e22918b6bd5b8bff2106fd

.pre-commit-config.yaml

@@ -7,9 +7,23 @@ repos:
     rev: v4.5.0
     hooks:
       - id: trailing-whitespace
+        # NOTE(JayF): We shouldn't modify release notes after their
+        #  associated release. Instead, ignore these minor lint issues.
+        exclude: |
+          (?x)(
+          ^releasenotes/notes/redfish-raid-get-drives-fix-18d46f3e7275b0ef.yaml$|
+          ^releasenotes/notes/provide_mountpoint-58cfd25b6dd4cfde.yaml$|
+          ^releasenotes/notes/ipmi-retries-min-command-interval-070cd7eff5eb74dd.yaml$|
+          ^releasenotes/notes/deprecate-ibmc-9106cc3a81171738.yaml$|
+          ^releasenotes/notes/fix-cve-2016-4985-b62abae577025365.yaml$
+          )
       - id: mixed-line-ending
         args: ['--fix', 'lf']
-        exclude: '.*\.(svg)$'
+        exclude: |
+          (?x)(
+          .*.svg$|
+          ^releasenotes/notes/ibmc-driver-45fcf9f50ebf0193.yaml$|
+          )
       - id: fix-byte-order-marker
       - id: check-merge-conflict
       - id: debug-statements
@@ -17,6 +31,7 @@ repos:
         files: .*\.json$
       - id: check-yaml
         files: .*\.(yaml|yml)$
+        exclude: releasenotes/.*$
   - repo: https://github.com/Lucas-C/pre-commit-hooks
     rev: v1.5.4
     hooks:
@@ -37,5 +52,44 @@ repos:
     hooks:
       - id: sphinx-lint
         args: [--enable=default-role]
-        files: ^doc/|releasenotes|api-guide
+        files: ^doc/|releasenotes|api-ref
         types: [rst]
+  - repo: https://opendev.org/openstack/bashate
+    rev: 2.1.0
+    hooks:
+      - id: bashate
+        args: ["-iE006,E044", "-eE005,E042"]
+        name: bashate
+        description: This hook runs bashate for linting shell scripts
+        entry: bashate
+        language: python
+        types: [shell]
+  - repo: https://github.com/PyCQA/bandit
+    rev: 1.7.10
+    hooks:
+      - id: bandit
+        args: ["-x", "tests/", "-n5", "-ll", "-c", "tools/bandit.yml"]
+        name: bandit
+        description: 'Bandit is a tool for finding common security issues in Python code'
+        entry: bandit
+        language: python
+        language_version: python3
+        types: [ python ]
+        require_serial: true
+  - repo: https://github.com/PyCQA/doc8
+    rev: v1.1.1
+    hooks:
+      - id: doc8
+        args: ["--ignore", "D001"]
+        name: doc8
+        description: This hook runs doc8 for linting docs
+        entry: python -m doc8
+        language: python
+        files: \.rst$
+        require_serial: true
+  - repo: local
+    hooks:
+      - id: check-releasenotes
+        name: check-releasenotes
+        language: python
+        entry: python tools/check-releasenotes.py

tools/bandit.yml

@@ -90,6 +90,7 @@ tests:
 skips:
   - B104
   - B604
+  - B701
 
 ### (optional) plugin settings - some test plugins require configuration data
 ### that may be given here, per-plugin. All bandit test plugins have a built in

tox.ini

@@ -39,26 +39,19 @@ deps = {[testenv]deps}
 commands = {toxinidir}/tools/states_to_dot.py -f {toxinidir}/doc/source/images/states.svg --format svg
 
 [testenv:pep8]
-usedevelop = False
+deps = pre-commit
-deps=
+allowlist_externals = pre-commit
-    hacking~=6.1.0 # Apache-2.0
+commands = pre-commit run --all-files --show-diff-on-failure {posargs}
-    doc8~=1.1.0 # Apache-2.0
+
-    pycodestyle>=2.0.0,<3.0.0 # MIT
+[testenv:bandit]
-    flake8-import-order~=0.18.0 # LGPLv3
+deps = pre-commit
-    bashate~=2.1.0 # Apache-2.0
+commands = pre-commit run --all-files --show-diff-on-failure bandit
-    -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
+
-allowlist_externals = bash
+[testenv:codespell]
-                      {toxinidir}/tools/run_bashate.sh
+description =
-                      {toxinidir}/tools/check-releasenotes.py
+  Run codespell to check spelling
-commands =
+deps = pre-commit
-  bash tools/flake8wrap.sh {posargs}
+commands = pre-commit run --all-files --show-diff-on-failure codespell
-  # Run bashate during pep8 runs to ensure violations are caught by
-  # the check and gate queues.
-  {toxinidir}/tools/run_bashate.sh {toxinidir}
-  # Check the *.rst files
-  doc8 README.rst CONTRIBUTING.rst doc/source api-ref/source --ignore D001
-  # Check to make sure reno releasenotes created with 'reno new'
-  {toxinidir}/tools/check-releasenotes.py
 
 [testenv:cover]
 setenv = {[testenv]setenv}
@@ -103,7 +96,6 @@ commands =
   sphinx-build -W -b latex doc/source doc/build/pdf
   make -C doc/build/pdf
 
-
 [testenv:api-ref]
 # NOTE(Mahnoor): documentation building process requires importing ironic API modules
 usedevelop = False
@@ -126,12 +118,41 @@ commands =
 
 [testenv:venv]
 setenv = PYTHONHASHSEED=0
+# NOTE(JayF) The generic venv target exists to allow people to run arbitrary
+#   things, e.g. `reno`. For this reason, allow all externals in this
+#   environment.
+allowlist_externals = *
 deps =
   -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
   -r{toxinidir}/test-requirements.txt
   -r{toxinidir}/doc/requirements.txt
 commands = {posargs}
 
+[testenv:local-ironic-dev]
+constrain_package_deps = true
+setenv = VIRTUAL_ENV={envdir}
+         PYTHONDONTWRITEBYTECODE=1
+         LANGUAGE=en_US
+         LC_ALL=en_US.UTF-8
+         PYTHONUNBUFFERED=1
+         SQLALCHEMY_WARN_20=true
+deps =
+  -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
+  -r{toxinidir}/requirements.txt
+  -r{toxinidir}/driver-requirements.txt
+  python-ironicclient
+  python-openstackclient
+allowlist_externals = bash
+commands =
+    bash -c 'ironic-dbsync --config-file={toxinidir}/tools/ironic.conf.localdev create_schema || echo "===== DB already exists; continuing ====="'
+    ironic --config-file={toxinidir}/tools/ironic.conf.localdev {posargs}
+passenv = http_proxy
+          HTTP_PROXY
+          https_proxy
+          HTTPS_PROXY
+          no_proxy
+          NO_PROXY
+
 [flake8]
 # [E129] visually indented line with same indent as next logical line
 # [E741] ambiguous variable name
@@ -154,50 +175,11 @@ per-file-ignores =
     ironic/cmd/__init__.py:E402
     ironic/tests/base.py:E402
 
-[hacking]
-import_exceptions = testtools.matchers, ironic.common.i18n
-
 [flake8:local-plugins]
 # [N323] Found use of _() without explicit import of _!
 extension =
   N323 = checks:check_explicit_underscore_import
 paths = ./ironic/hacking/
 
-[testenv:bandit]
+[hacking]
-usedevelop = False
+import_exceptions = testtools.matchers, ironic.common.i18n
-deps = -r{toxinidir}/test-requirements.txt
-commands = bandit -r ironic -x tests -n5 -ll -c tools/bandit.yml
-
-[testenv:codespell]
-description =
-  Run codespell to check spelling
-deps = codespell
-# note(JayF): {posargs} lets us run `tox -ecodespell -- -w` to get codespell
-#             to correct spelling issues in our code it's aware of.
-commands =
-  codespell {posargs}
-
-[testenv:local-ironic-dev]
-constrain_package_deps = true
-usedevelop = True
-setenv = VIRTUAL_ENV={envdir}
-         PYTHONDONTWRITEBYTECODE=1
-         LANGUAGE=en_US
-         LC_ALL=en_US.UTF-8
-         PYTHONUNBUFFERED=1
-deps =
-  -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
-  -r{toxinidir}/requirements.txt
-  -r{toxinidir}/driver-requirements.txt
-  python-ironicclient
-  python-openstackclient
-allowlist_externals = bash
-commands =
-    bash -c 'ironic-dbsync --config-file={toxinidir}/tools/ironic.conf.localdev create_schema || echo "===== DB already exists; continuing ====="'
-    ironic --config-file={toxinidir}/tools/ironic.conf.localdev {posargs}
-passenv = http_proxy
-          HTTP_PROXY
-          https_proxy
-          HTTPS_PROXY
-          no_proxy
-          NO_PROXY

zuul.d/ironic-jobs.yaml

@@ -881,28 +881,6 @@
         IRONIC_RAMDISK_TYPE: tinyipa
         IRONIC_AUTOMATED_CLEAN_ENABLED: False
 
-- job:
-    # Security testing for known issues
-    name: ironic-tox-bandit
-    parent: openstack-tox
-    timeout: 2400
-    vars:
-      tox_envlist: bandit
-    required-projects:
-      - opendev.org/openstack/ironic
-    irrelevant-files:
-      - ^.*\.rst$
-      - ^api-ref/.*$
-      - ^doc/.*$
-      - ^driver-requirements.txt$
-      - ^install-guide/.*$
-      - ^ironic/locale/.*$
-      - ^ironic/tests/.*$
-      - ^redfish-interop-profiles/.*$
-      - ^releasenotes/.*$
-      - ^setup.cfg$
-      - ^tox.ini$
-
 - job:
     name: ironic-tempest-ipa-wholedisk-bios-ipmi-direct-dib
     parent: ironic-base
@@ -1319,10 +1297,3 @@
       grenade_localrc:
         # NOTE(iurygregory): This is required for skip upgrades.
         NOVA_ENABLE_UPGRADE_WORKAROUND: True
-
-- job:
-    name: ironic-tox-codespell
-    parent: openstack-tox
-    timeout: 7200
-    vars:
-      tox_envlist: codespell

zuul.d/project.yaml

@@ -10,8 +10,6 @@
       - release-notes-jobs-python3
     check:
       jobs:
-        - ironic-tox-bandit
-        - ironic-tox-codespell
         - ironic-tox-unit-mysql-migrations
         - ironic-tox-unit-with-driver-libs
         - ironic-cross-sushy:
@@ -70,8 +68,6 @@
             voting: false
     gate:
       jobs:
-        - ironic-tox-bandit
-        - ironic-tox-codespell
         - ironic-tox-unit-mysql-migrations
         - ironic-tox-unit-with-driver-libs
         - ironic-tempest-functional-python3