Add flake8-import-order and use python3.9 on tox

Improving the code quality of utilities by adding the flake8 check on
zuul and adding the flake8-import-order plugin to standardize imports.
Also, defaults testenv to python3.9 configuration that should be used
for now on with the debian migration.

Story: 2010100
Task: 45671
Signed-off-by: Thiago Brito <thiago.brito@windriver.com>
Change-Id: I007302bdfb873aed4ebb7b9023db2461282b9fba
This commit is contained in:
Thiago Brito 2022-06-22 18:55:26 -03:00 committed by Thiago Paiva Brito
parent 62a6bafae8
commit c213a3832d
35 changed files with 1008 additions and 770 deletions

View File

@ -2,29 +2,42 @@
- project:
check:
jobs:
- openstack-tox-linters
- stx-tox-pylint
- utilities-ceph-manager-tox-bandit
- utilities-ceph-client-tox-bandit
- utilities-pci-irq-affinity-agent-tox-py27
- utilities-pci-irq-affinity-agent-tox-py39
- utilities-pci-irq-affinity-agent-tox-flake8
- stx-utilities-tox-linters
- stx-utilities-tox-pep8
- stx-utilities-tox-pylint
- stx-utilities-ceph-manager-tox-bandit
- stx-utilities-ceph-client-tox-bandit
- stx-utilities-pci-irq-affinity-agent-tox-py27
- stx-utilities-pci-irq-affinity-agent-tox-py39
- stx-utilities-pci-irq-affinity-agent-tox-pep8
gate:
jobs:
- openstack-tox-linters
- stx-tox-pylint
- utilities-ceph-manager-tox-bandit
- utilities-ceph-client-tox-bandit
- utilities-pci-irq-affinity-agent-tox-py27
- utilities-pci-irq-affinity-agent-tox-py39
- utilities-pci-irq-affinity-agent-tox-flake8
- stx-utilities-tox-linters
- stx-utilities-tox-pep8
- stx-utilities-tox-pylint
- stx-utilities-ceph-manager-tox-bandit
- stx-utilities-ceph-client-tox-bandit
- stx-utilities-pci-irq-affinity-agent-tox-py27
- stx-utilities-pci-irq-affinity-agent-tox-py39
- stx-utilities-pci-irq-affinity-agent-tox-pep8
post:
jobs:
- stx-utilities-upload-git-mirror
- job:
name: stx-tox-pylint
name: stx-utilities-tox-linters
parent: openstack-tox-linters
nodeset: debian-bullseye
- job:
name: stx-utilities-tox-pep8
parent: openstack-tox-pep8
nodeset: debian-bullseye
- job:
name: stx-utilities-tox-pylint
parent: tox
nodeset: debian-bullseye
description: |
Run pylint tests
required-projects:
@ -32,6 +45,7 @@
- starlingx/config
vars:
tox_envlist: pylint
- job:
name: stx-utilities-upload-git-mirror
parent: upload-git-mirror
@ -46,7 +60,7 @@
pass-to-parent: true
- job:
name: utilities-ceph-manager-tox-bandit
name: stx-utilities-ceph-manager-tox-bandit
parent: tox
description: |
Run bandit test for utilities ceph-mananger
@ -58,7 +72,7 @@
tox_extra_args: -c ./ceph/ceph-manager/ceph-manager/tox.ini
- job:
name: utilities-ceph-client-tox-bandit
name: stx-utilities-ceph-client-tox-bandit
parent: tox
description: |
Run bandit test for utilities python-cephclient
@ -70,7 +84,7 @@
tox_extra_args: -c ./ceph/python-cephclient/python-cephclient/tox.ini
- job:
name: utilities-pci-irq-affinity-agent-tox-py27
name: stx-utilities-pci-irq-affinity-agent-tox-py27
parent: tox
description: |
Run py27 unittests for utilities/pci-irq-affinity-agent
@ -82,7 +96,7 @@
tox_extra_args: -c ./utilities/pci-irq-affinity-agent/pci_irq_affinity/tox.ini
- job:
name: utilities-pci-irq-affinity-agent-tox-py39
name: stx-utilities-pci-irq-affinity-agent-tox-py39
voting: false
parent: tox
description: |
@ -96,7 +110,7 @@
tox_extra_args: -c ./utilities/pci-irq-affinity-agent/pci_irq_affinity/tox.ini
- job:
name: utilities-pci-irq-affinity-agent-tox-flake8
name: stx-utilities-pci-irq-affinity-agent-tox-pep8
parent: tox
description: |
Run flake8 checks for utilities/pci-irq-affinity-agent
@ -104,7 +118,7 @@
files:
- ./utilities/pci-irq-affinity-agent/*
vars:
tox_envlist: flake8
tox_envlist: pep8
tox_extra_args: -c ./utilities/pci-irq-affinity-agent/pci_irq_affinity/tox.ini
- secret:

View File

@ -4,11 +4,12 @@
# SPDX-License-Identifier: Apache-2.0
#
from ceph_manager import exception
from ceph_manager.i18n import _LI
# noinspection PyUnresolvedReferences
from oslo_log import log as logging
from ceph_manager import exception
from ceph_manager.i18n import _LI
LOG = logging.getLogger(__name__)

View File

@ -4,10 +4,12 @@
# SPDX-License-Identifier: Apache-2.0
#
from ceph_manager.i18n import _
# noinspection PyUnresolvedReferences
from sysinv.common import constants as sysinv_constants
from ceph_manager.i18n import _
CEPH_POOL_OBJECT_GATEWAY_NAME_JEWEL = \
sysinv_constants.CEPH_POOL_OBJECT_GATEWAY_NAME_JEWEL
CEPH_POOL_OBJECT_GATEWAY_NAME_HAMMER = \

View File

@ -4,11 +4,12 @@
# SPDX-License-Identifier: Apache-2.0
#
# noinspection PyUnresolvedReferences
from oslo_log import log as logging
# noinspection PyUnresolvedReferences
from ceph_manager.i18n import _
from ceph_manager.i18n import _LW
# noinspection PyUnresolvedReferences
from oslo_log import log as logging
LOG = logging.getLogger(__name__)

View File

@ -6,23 +6,22 @@
import time
# noinspection PyUnresolvedReferences
from fm_api import fm_api
# noinspection PyUnresolvedReferences
from fm_api import constants as fm_constants
# noinspection PyUnresolvedReferences
from fm_api import fm_api
# noinspection PyUnresolvedReferences
from oslo_log import log as logging
# noinspection PyProtectedMember
from ceph_manager.i18n import _
from ceph_manager.i18n import _LI
from ceph_manager.i18n import _LW
from ceph_manager.i18n import _LE
from tsconfig import tsconfig
from ceph_manager import constants
from ceph_manager import exception
# noinspection PyProtectedMember
from ceph_manager.i18n import _
from ceph_manager.i18n import _LE
from ceph_manager.i18n import _LI
from ceph_manager.i18n import _LW
from tsconfig import tsconfig
LOG = logging.getLogger(__name__)
@ -721,4 +720,3 @@ class Monitor(HandleUpgradesMixin):
self.current_health_alarm = self.service.fm_api.get_fault(
fm_constants.FM_ALARM_ID_STORAGE_CEPH,
self.service.entity_instance_id)

View File

@ -12,28 +12,27 @@ import sys
# noinspection PyUnresolvedReferences
import eventlet
# noinspection PyUnresolvedReferences
import oslo_messaging as messaging
# noinspection PyUnresolvedReferences
from fm_api import fm_api
# noinspection PyUnresolvedReferences
from oslo_config import cfg
# noinspection PyUnresolvedReferences
from oslo_log import log as logging
# noinspection PyUnresolvedReferences
import oslo_messaging as messaging
# noinspection PyUnresolvedReferences
from oslo_service import service
# noinspection PyUnresolvedReferences
from oslo_service.periodic_task import PeriodicTasks
# noinspection PyUnresolvedReferences
from cephclient import wrapper
from retrying import retry
from ceph_manager.monitor import Monitor
from ceph_manager import constants
from ceph_manager import utils
from ceph_manager.i18n import _LI
from ceph_manager.i18n import _LW
from retrying import retry
from ceph_manager.monitor import Monitor
from cephclient import wrapper
eventlet.monkey_patch(all=True)
@ -136,11 +135,12 @@ class Service(SysinvConductorUpgradeApi, service.Service):
# pylint: disable=protected-access
sysinv_conf = self.conf._namespace._normalized[0]['DEFAULT']
url = "rabbit://{user}:{password}@{host}:{port}"\
"".format(user=sysinv_conf['rabbit_userid'][0],
password=sysinv_conf['rabbit_password'][0],
host=utils.ipv6_bracketed(sysinv_conf['rabbit_host'][0]),
port=sysinv_conf['rabbit_port'][0])
url = "rabbit://{user}:{password}@{host}:{port}".format(
user=sysinv_conf['rabbit_userid'][0],
password=sysinv_conf['rabbit_password'][0],
host=utils.ipv6_bracketed(sysinv_conf['rabbit_host'][0]),
port=sysinv_conf['rabbit_port'][0]
)
transport = messaging.get_transport(self.conf, url=url)
self.sysinv_conductor = messaging.RPCClient(
transport,

View File

@ -9,3 +9,4 @@ eventlet
pytest
oslo.log
oslo.i18n
flake8-import-order

View File

@ -2,26 +2,32 @@
[tox]
minversion = 1.6
envlist = py27,pep8
envlist = py27,pep8,bandit
skipsdist = True
# tox does not work if the path to the workdir is too long, so move it to /tmp
toxworkdir = /tmp/{env:USER}_ceph_manager_tox
toxworkdir = /tmp/{env:USER}_utilities_ceph_manager_tox
[testenv]
basepython = python3.9
setenv = VIRTUAL_ENV={envdir}
usedevelop = True
install_command = pip install -U --force-reinstall {opts} {packages}
deps = -r{toxinidir}/test-requirements.txt
commands = py.test {posargs}
commands = pytest {posargs}
whitelist_externals = bash
passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY
[testenv:py27]
basepython = python2.7
[testenv:pep8]
commands =
flake8 {posargs}
[flake8]
exclude = .venv,.git,.tox,dist,doc,etc,*glance/locale*,*lib/python*,*egg,build
import-order-style = google
application-import-names = ceph_manager,cephclient
[bandit]
# The following bandit tests are being skipped:

File diff suppressed because it is too large Load Diff

View File

@ -11,7 +11,8 @@ class CephClientException(Exception):
def __init__(self, *args, **kwargs):
if "message" not in kwargs:
try:
message = self.message.format(*args, **kwargs) # pylint: disable=W1645
# pylint: disable=W1645
message = self.message.format(*args, **kwargs)
except Exception: # noqa
message = '{}, args:{}, kwargs: {}'.format(
self.message, args, kwargs) # pylint: disable=W1645

View File

@ -69,7 +69,8 @@ class CephWrapper(CephClient):
return self._request('osd create', **kwargs)
def osd_rm(self, ids, body='json', timeout=None):
"""remove osd(s) <id> [<id>...], or use <any|all> to remove all osds """
"""remove osd(s) <id> [<id>...], or use <any|all> to remove all osds
"""
if isinstance(ids, list):
ids = [self._sanitize_osdid_to_str(_id)
for _id in ids]
@ -82,7 +83,8 @@ class CephWrapper(CephClient):
return self.osd_rm(ids, body=body, timeout=timeout)
def osd_down(self, ids, body='json', timeout=None):
"""set osd(s) <id> [<id>...] down, or use <any|all> to set all osds down """
"""set osd(s) <id> [<id>...] down, or use <any|all> to set all osds down
"""
if isinstance(ids, list):
ids = [self._sanitize_osdid_to_str(_id)
for _id in ids]
@ -219,7 +221,11 @@ class CephWrapper(CephClient):
return caps
def auth_add(self, entity, caps=None, body='json', timeout=None):
"""add auth info for <entity> from input file, or random key if no input is given, and/or any caps specified in the command """
"""Add auth info
Adds auth info for <entity> from input file, or random key if no input
is given, and/or any caps specified in the command
"""
caps = self._auth_convert_caps(caps)
return super(CephWrapper, self).auth_add(
entity, caps=caps, body=body, timeout=timeout)
@ -231,15 +237,23 @@ class CephWrapper(CephClient):
entity, caps=caps, body=body, timeout=timeout)
def auth_get_or_create(self, entity, caps=None, body='json', timeout=None):
"""add auth info for <entity> from input file, or random key if no input given, and/or any caps specified in the command """
"""Get or create auth info
Adds auth info for <entity> from input file, or random key if no input
given, and/or any caps specified in the command
"""
caps = self._auth_convert_caps(caps)
return super(CephWrapper, self).auth_get_or_create(
entity, caps, body=body, timeout=timeout)
def auth_get_or_create_key(self, entity, caps=None,
body='json', timeout=None):
"""Get or add auth key
"""get, or add, key for <name> from system/caps pairs specified in the command. If key already exists, any given caps must match the existing caps for that key. """
Gets, or adds, key for <name> from system/caps pairs specified in the
command. If key already exists, any given caps must match the
existing caps for that key.
"""
caps = self._auth_convert_caps(caps)
response, _body = super(CephWrapper, self).auth_get_or_create_key(
entity, caps, body=body, timeout=timeout)

View File

@ -5,3 +5,4 @@
bandit;python_version>="3.0"
flake8
pytest
flake8-import-order

View File

@ -1,22 +1,28 @@
[tox]
envlist = py27,pep8
envlist = py27,pep8,bandit
skipsdist = True
toxworkdir = /tmp/{env:USER}_python_cephclient_tox
toxworkdir = /tmp/{env:USER}_utilities_python_cephclient_tox
[testenv]
basepython = python3.9
setenv = VIRTUAL_ENV={envdir}
usedevelop = True
install_command = pip install -U --force-reinstall {opts} {packages}
deps = -r{toxinidir}/test-requirements.txt
commands = py.test {posargs}
commands = pytest {posargs}
whitelist_externals = bash
[testenv:py27]
basepython = python2.7
[testenv:pep8]
commands =
flake8 {posargs}
[flake8]
exclude = .venv,.git,.tox,dist,doc,*lib/python*,*egg,build
import-order-style = google
application-import-names = cephclient
[bandit]
# The following bandit tests are being skipped:

View File

@ -4,5 +4,5 @@ bashate >= 0.2
isort<5;python_version>="3.0"
pylint<2.1.0;python_version<"3.0" # GPLv2
pylint<2.3.0;python_version>="3.0" # GPLv2
python-ldap<3.4.0
flake8-import-order

View File

@ -1,44 +1,43 @@
import os
import shutil
import logging
import zipfile
import atexit
from datetime import datetime
from datetime import timedelta
from functools import wraps
import logging
import os
import shutil
import tarfile
from urllib.parse import quote
from urllib.parse import unquote
import zipfile
from apscheduler.schedulers.background import BackgroundScheduler
from flask import abort
from flask import after_this_request
from flask import flash
from flask import Flask
from flask import g
from flask import jsonify
from flask import make_response
from flask import redirect
from flask import render_template
from flask import request
from flask import send_file
from flask import session
from flask import url_for
from flask_mail import Mail
from flask_mail import Message
import flask_openid
from launchpadlib.launchpad import Launchpad
import magic
from mail_config import custom_mail_password
from mail_config import custom_mail_server
from mail_config import custom_mail_username
from mail_config import custom_server_admins
import magic
from flask_mail import Mail
from flask_mail import Message
from urllib.parse import quote
from urllib.parse import unquote
from flask import Flask
from flask import flash
from flask import request
from flask import redirect
from flask import render_template
from flask import g
from flask import session
from flask import url_for
from flask import abort
from flask import send_file
from flask import make_response
from flask import jsonify
from flask import after_this_request
import flask_openid
from openid.extensions import pape
import pymysql.cursors
from werkzeug.exceptions import RequestEntityTooLarge
from werkzeug.utils import secure_filename
import pymysql.cursors
from launchpadlib.launchpad import Launchpad
import atexit
from apscheduler.schedulers.background import BackgroundScheduler
logging.basicConfig(filename='collect.log', level=logging.INFO, format='%(asctime)s:%(levelname)s:%(message)s')

View File

@ -14,15 +14,16 @@
#
########################################################################
import shutil
import algorithms
from datetime import datetime
import gzip
import logging
import os
import re
import shutil
import subprocess
from datetime import datetime
import algorithms
logger = logging.getLogger(__name__)

View File

@ -16,11 +16,12 @@
from datetime import datetime
import algorithms
import json
import logging
import os
import algorithms
logger = logging.getLogger(__name__)

View File

@ -38,12 +38,12 @@
import argparse
from cmath import log
from datetime import datetime
from datetime import timezone
import logging
import os
import time
from datetime import datetime
from datetime import timezone
from execution_engine import ExecutionEngine
from plugin import Plugin

View File

@ -10,11 +10,11 @@ by querying /proc/buddyinfo
Example:
# python3 buddyinfo.py
"""
from collections import defaultdict
import logging
import optparse
import os
import re
from collections import defaultdict
import logging
class Logger:

View File

@ -1,26 +1,25 @@
#!/usr/bin/python3
"""
Copyright (c) 2017 Wind River Systems, Inc.
Copyright (c) 2017-2022 Wind River Systems, Inc.
SPDX-License-Identifier: Apache-2.0
"""
import os
import sys
import time
from collections import OrderedDict
import datetime
import psutil
import fcntl
import logging
from six.moves import configparser
import itertools
import six
from multiprocessing import Process
from multiprocessing import cpu_count
from subprocess import Popen
from multiprocessing import Process
import os
from subprocess import PIPE
from collections import OrderedDict
from subprocess import Popen
import sys
import time
import psutil
from six.moves import configparser
from six.moves import input

17
tox.ini
View File

@ -1,14 +1,18 @@
[tox]
envlist = linters,pylint
envlist = linters,pylint,pep8
minversion = 2.3
skipsdist = True
toxworkdir = /tmp/{env:USER}_python_utilities_tox
stxdir = {toxinidir}/..
[testenv]
sitepackages = False
basepython = python3
install_command = pip install {opts} {packages}
basepython = python3.9
install_command = pip install \
-v -v -v \
-U \
-c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/starlingx/root/raw/branch/master/build-tools/requirements/debian/upper-constraints.txt} \
{opts} {packages}
setenv =
VIRTUAL_ENV={envdir}
OS_STDOUT_CAPTURE=1
@ -75,8 +79,10 @@ commands =
ignore = F401,F841,W291,W391,W503,W504,W605,E128,E221,E226,E227,E241,E265,E302,E305,E501,E502,E702,E722,H101,H104,H201,H238,H306,H401,H405
# max-line-length is not referenced if E501 is suppressed
max-line-length=80
import-order-style = google
application-import-names = cephclient,ceph_manager,logmgmt,pci_irq_affinity,platform_util
[testenv:flake8]
[testenv:pep8]
commands =
flake8
@ -99,7 +105,6 @@ commands = pylint {posargs} \
[testenv:linters]
commands =
{[testenv:bashate]commands}
{[testenv:flake8]commands}
[testenv:bandit]
basepython = python3

View File

@ -9,14 +9,15 @@ SPDX-License-Identifier: Apache-2.0
# IMPORTS
###################
from __future__ import absolute_import
import glob
import logging
import logging.handlers
import time
import os
import subprocess
import glob
import re
import subprocess
import sys
import time
from daemon import runner

View File

@ -11,8 +11,8 @@ SPDX-License-Identifier: Apache-2.0
import array
import fcntl
import struct
import glob
import struct
EXT2_APPEND_FL = 0x00000020
EXT4_EXTENTS_FL = 0x00080000

View File

@ -12,10 +12,11 @@
""" Define pci_irq_affinity_provider class"""
import os
from pci_irq_affinity.driver import AffinePciIrqDriver
from pci_irq_affinity.log import LOG
from pci_irq_affinity import nova_provider
from pci_irq_affinity import utils as pci_utils
from pci_irq_affinity.driver import AffinePciIrqDriver
from pci_irq_affinity.log import LOG
COMPUTE_IRQ = os.getenv("COMPUTE_IRQ", default="/proc/irq/")

View File

@ -12,22 +12,22 @@
""" Pci interrupt affinity agent daemon entry"""
import os
import sys
import signal
import socket
import eventlet
import sys
import threading
import time
import eventlet
import oslo_messaging
from oslo_service import periodic_task
from oslo_service import service
import oslo_messaging
from pci_irq_affinity import affinity
from pci_irq_affinity.config import CONF
from pci_irq_affinity.log import LOG
from pci_irq_affinity import nova_provider
from pci_irq_affinity import utils as pci_utils
from pci_irq_affinity.config import CONF
from pci_irq_affinity.log import LOG
stay_on = True

View File

@ -11,8 +11,8 @@
""" Define configuration info for pci-irq-affinity-agent"""
import sys
from os import path
import sys
from oslo_config import cfg

View File

@ -11,13 +11,13 @@
""" Define AffinePciIrqDriver class"""
from oslo_service import loopingcall
from oslo_concurrency import lockutils
from oslo_service import loopingcall
from pci_irq_affinity.config import CONF
from pci_irq_affinity.log import LOG
from pci_irq_affinity import nova_provider
from pci_irq_affinity import utils as pci_utils
from pci_irq_affinity.config import CONF
from pci_irq_affinity.log import LOG
synchronized = lockutils.synchronized_with_prefix('pci_irq_affinity-')

View File

@ -11,14 +11,16 @@
""" Encapsulate libvirt related interfaces"""
import libvirt
import os
import sys
import signal
from xml.dom import minidom
from xml.etree import ElementTree
import libvirt
from pci_irq_affinity.log import LOG
debug = 0
# libvirt timeout parameters
LIBVIRT_TIMEOUT_SEC = 5.0

View File

@ -14,17 +14,17 @@ This class wraps novaclient access interface and expose get_instance() and
get_instances() to other agent classes.
"""
import keyring
import os
import socket
import keyring
from keystoneauth1 import loading
from keystoneauth1 import session
from novaclient import client
from pci_irq_affinity.config import CONF
from pci_irq_affinity import guest
from pci_irq_affinity import instance
from pci_irq_affinity.config import CONF
from pci_irq_affinity.log import LOG

View File

@ -6,11 +6,12 @@
# All Rights Reserved.
#
import mock
import unittest
from pci_irq_affinity.config import CONF
import mock
from pci_irq_affinity import nova_provider
from pci_irq_affinity.config import CONF
class TestNovaProvider(unittest.TestCase):

View File

@ -11,12 +11,12 @@
""" Define utility functions for this agent"""
import os
import errno
from itertools import groupby
import os
from pci_irq_affinity.log import LOG
import pci_irq_affinity.instance
COMPUTE_PCI_DEVICES = os.getenv("COMPUTE_PCI_DEVICES", default="/sys/bus/pci/devices")

View File

@ -6,3 +6,4 @@ pylint<2.1.0;python_version<"3.0" # GPLv2
pylint<2.3.0;python_version>="3.0" # GPLv2
pytest
mock
flake8-import-order

View File

@ -1,7 +1,8 @@
[tox]
envlist = py27,flake8
envlist = py39,py27,pep8
minversion = 2.3
skipsdist = True
toxworkdir = /tmp/{env:USER}_utilities_pci_irq_affinity_agent_tox
[testenv]
sitepackages = False
@ -59,8 +60,10 @@ basepython = python3.9
ignore = F401,F841,W291,W391,W503,W504,W605,E128,E221,E226,E227,E241,E265,E302,E305,E501,E502,E702,E722,H101,H104,H201,H238,H306,H401,H405
# max-line-length is not referenced if E501 is suppressed
max-line-length=80
import-order-style = google
application-import-names = pci_irq_affinity
[testenv:flake8]
[testenv:pep8]
commands =
flake8

View File

@ -5,6 +5,7 @@
#
import abc
import six

View File

@ -1,6 +1,6 @@
#!/usr/bin/python
################################################################################
# Copyright (c) 2013 Wind River Systems, Inc.
# Copyright (c) 2013-2022 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -11,9 +11,10 @@
# total memory, and numa nodes
from __future__ import print_function
import os
import sys
import re
import sys
class Topology(object):
""" Build up topology information.