starlingx/utilities
Code Issues Proposed changes

Refactoring novaClient instantiation and unittests

Browse Source 
And adding infra for unittest execution.

Partial-Bug: 1951579
Signed-off-by: Thiago Brito <thiago.brito@windriver.com>
Change-Id: I58cefac9076db52333b41633bf2cbaa5441dc98c
This commit is contained in:
Thiago Brito 2021-11-19 18:41:26 -03:00
parent ed01aa74e1
commit 5d0b69af84
12 changed files with 271 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -2,3 +2,5 @@
*.egg-info
*.swp
.idea
__pycache__
*.py[cod]

44
.zuul.yaml
View File

@ -6,12 +6,18 @@
- stx-tox-pylint
- utilities-ceph-manager-tox-bandit
- utilities-ceph-client-tox-bandit
- utilities-pci-irq-affinity-agent-tox-py27
- utilities-pci-irq-affinity-agent-tox-py39
- utilities-pci-irq-affinity-agent-tox-flake8
gate:
jobs:
- openstack-tox-linters
- stx-tox-pylint
- utilities-ceph-manager-tox-bandit
- utilities-ceph-client-tox-bandit
- utilities-pci-irq-affinity-agent-tox-py27
- utilities-pci-irq-affinity-agent-tox-py39
- utilities-pci-irq-affinity-agent-tox-flake8
post:
jobs:
- stx-utilities-upload-git-mirror
@ -63,6 +69,44 @@
tox_envlist: bandit
tox_extra_args: -c ./ceph/python-cephclient/python-cephclient/tox.ini
- job:
name: utilities-pci-irq-affinity-agent-tox-py27
parent: tox
description: |
Run py27 unittests for utilities/pci-irq-affinity-agent
nodeset: ubuntu-bionic
files:
- ./utilities/pci-irq-affinity-agent/*
vars:
tox_envlist: py27
tox_extra_args: -c ./utilities/pci-irq-affinity-agent/pci_irq_affinity/tox.ini
- job:
name: utilities-pci-irq-affinity-agent-tox-py39
voting: false
parent: tox
description: |
Run py39 unittests for utilities/pci-irq-affinity-agent
nodeset: debian-bullseye
files:
- ./utilities/pci-irq-affinity-agent/*
vars:
tox_envlist: py39
python_version: 3.9
tox_extra_args: -c ./utilities/pci-irq-affinity-agent/pci_irq_affinity/tox.ini
- job:
name: utilities-pci-irq-affinity-agent-tox-flake8
parent: tox
description: |
Run flake8 checks for utilities/pci-irq-affinity-agent
nodeset: ubuntu-bionic
files:
- ./utilities/pci-irq-affinity-agent/*
vars:
tox_envlist: flake8
tox_extra_args: -c ./utilities/pci-irq-affinity-agent/pci_irq_affinity/tox.ini
- secret:
name: stx-utilities-github-secret
data:

13
utilities/pci-irq-affinity-agent/pci_irq_affinity/bindep.txt Normal file
View File

@ -0,0 +1,13 @@
# This is a cross-platform list tracking distribution packages needed for install and tests;
# see https://docs.openstack.org/infra/bindep/ for additional information.
build-essential [platform:dpkg test]
gcc [platform:rpm test]
libvirt-dev [platform:dpkg]
python3-dev [platform:dpkg (requirements python3)]
python3-devel [platform:rpm (requirements python3)]
python-dev [platform:dpkg test]
python3-all [platform:dpkg]
python3-all-dev [platform:dpkg]
python3 [platform:rpm test]
python3-devel [platform:rpm test]

11
utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/affinity.py
View File

@ -12,15 +12,15 @@
""" Define pci_irq_affinity_provider class"""
import os
import pci_irq_affinity.utils as pci_utils
from pci_irq_affinity.driver import AffinePciIrqDriver
from pci_irq_affinity.nova_provider import novaClient
from pci_irq_affinity.log import LOG
from pci_irq_affinity import nova_provider
from pci_irq_affinity import utils as pci_utils
COMPUTE_IRQ = os.getenv("COMPUTE_IRQ", default="/proc/irq/")
class pci_irq_affinity_provider:
class PciIrqAffinityProvider:
def __init__(self):
self.affinePciIrqDriver = AffinePciIrqDriver()
@ -90,9 +90,10 @@ class pci_irq_affinity_provider:
'task_state': None,
'deleted': False,
'all_tenants': True}
instances = novaClient.get_instances(filters)
nova_client = nova_provider.get_nova_client()
instances = nova_client.get_instances(filters)
for inst in instances:
self.affine_pci_dev_instance(inst, wait_for_irqs=False)
pciIrqAffinity = pci_irq_affinity_provider()
pci_irq_affinity = PciIrqAffinityProvider()

21
utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/agent.py
View File

@ -23,11 +23,12 @@ from oslo_service import periodic_task
from oslo_service import service
import oslo_messaging
from pci_irq_affinity.affinity import pciIrqAffinity
from pci_irq_affinity import affinity
from pci_irq_affinity.config import CONF
from pci_irq_affinity.log import LOG
from pci_irq_affinity.nova_provider import novaClient
import pci_irq_affinity.utils as pci_utils
from pci_irq_affinity import nova_provider
from pci_irq_affinity import utils as pci_utils
stay_on = True
@ -44,7 +45,8 @@ def process_signal_handler(signum, frame):
def get_inst(instance_uuid, callback):
# get instance info from nova
inst = novaClient.get_instance(instance_uuid)
nova_client = nova_provider.get_nova_client()
inst = nova_client.get_instance(instance_uuid)
if inst is not None:
LOG.debug("inst:%s" % inst)
callback(inst)
@ -52,12 +54,12 @@ def get_inst(instance_uuid, callback):
def query_instance_callback(inst):
LOG.debug("query inst:%s" % inst)
pciIrqAffinity.affine_pci_dev_instance(inst)
affinity.pci_irq_affinity.affine_pci_dev_instance(inst)
@periodic_task.periodic_task(spacing=CONF.parameters.pci_affine_interval)
def audit_affinity(self, context):
pciIrqAffinity.audit_pci_irq_affinity()
affinity.pci_irq_affinity.audit_pci_irq_affinity()
def audit_work(srv, callback):
@ -150,7 +152,7 @@ class InstanceOfflineNotificationEndpoint(BaseInstanceEndpoint):
if instance_uuid:
LOG.info("Instance offline: uuid=%s, instance_host=%s, event_type=%s" % (
instance_uuid, instance_host, event_type))
pciIrqAffinity.reset_irq_affinity(instance_uuid)
affinity.pci_irq_affinity.reset_irq_affinity(instance_uuid)
def rpc_work(srv):
@ -194,11 +196,12 @@ def process_main():
LOG.info("Enter PCIInterruptAffinity Agent")
nova_client = nova_provider.get_nova_client()
try:
signal.signal(signal.SIGTSTP, process_signal_handler)
openstack_enabled = CONF.openstack.openstack_enabled
if openstack_enabled:
novaClient.open_libvirt_connect()
nova_client.open_libvirt_connect()
audit_srv = audits_initialize()
rabbit_client = start_rabbitmq_client()
@ -216,7 +219,7 @@ def process_main():
finally:
LOG.error("process_main finalized!!!")
if openstack_enabled:
novaClient.close_libvirt_connect()
nova_client.close_libvirt_connect()
audit_srv.tg.stop()
rabbit_client.stop()

7
utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/driver.py
View File

@ -14,10 +14,10 @@
from oslo_service import loopingcall
from oslo_concurrency import lockutils
import pci_irq_affinity.utils as pci_utils
from pci_irq_affinity.config import CONF
from pci_irq_affinity.log import LOG
from pci_irq_affinity.nova_provider import novaClient
from pci_irq_affinity import nova_provider
from pci_irq_affinity import utils as pci_utils
synchronized = lockutils.synchronized_with_prefix('pci_irq_affinity-')
@ -94,7 +94,8 @@ class AffinePciIrqDriver:
_msi_irqs = set()
# refresh instance info.
if refresh_need:
_inst = novaClient.get_instance(inst.uuid)
nova_client = nova_provider.get_nova_client()
_inst = nova_client.get_instance(inst.uuid)
if _inst is None:
return

15
utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/nova_provider.py
View File

@ -14,18 +14,18 @@ This class wraps novaclient access interface and expose get_instance() and
get_instances() to other agent classes.
"""
import keyring
import os
import socket
import keyring
from keystoneauth1 import loading
from keystoneauth1 import session
from novaclient import client
from pci_irq_affinity.log import LOG
from pci_irq_affinity.config import CONF
from pci_irq_affinity import instance
from pci_irq_affinity import guest
from pci_irq_affinity import instance
from pci_irq_affinity.log import LOG
class NovaProvider:
@ -142,7 +142,8 @@ class NovaProvider:
return instances
if CONF.openstack.openstack_enabled:
novaClient = NovaProvider()
else:
novaClient = None
def get_nova_client():
if CONF.openstack.openstack_enabled:
return NovaProvider()
else:
return None

19
utilities/pci-irq-affinity-agent/pci_irq_affinity/requirements.txt Normal file
View File

@ -0,0 +1,19 @@
eventlet==0.33.0
keyring==18.0.1
keyrings.alt==3.2.0
pycrypto==2.6.1
keystoneauth1==4.0.1
kombu==4.6.11
pkgconfig==1.5.2
libvirt-python;python_version>="3.0" # GPLv2
libvirt-python<6.0.0;python_version<"3.0" # GPLv2
oslo.concurrency==3.31.0
oslo.config==7.0.0
oslo.i18n==3.25.1
oslo.log==3.45.2
oslo.messaging==10.5.0
oslo.service==1.41.1
python-daemon==2.3.0
python-novaclient==16.0.0
retrying==1.3.3
psutil==5.8.0

8
utilities/pci-irq-affinity-agent/pci_irq_affinity/test-requirements.txt Normal file
View File

@ -0,0 +1,8 @@
# hacking pulls in flake8
hacking>=1.1.0,<=2.0.0 # Apache-2.0
bashate >= 0.2
isort<5;python_version>="3.0"
pylint<2.1.0;python_version<"3.0" # GPLv2
pylint<2.3.0;python_version>="3.0" # GPLv2
pytest
mock

0
utilities/pci-irq-affinity-agent/pci_irq_affinity/tests/__init__.py Normal file
View File

73
utilities/pci-irq-affinity-agent/pci_irq_affinity/tests/test_nova_provider.py Normal file
View File

@ -0,0 +1,73 @@
#
# Copyright (c) 2021 StarlingX.
#
# SPDX-License-Identifier: Apache-2.0
#
# All Rights Reserved.
#
import mock
import unittest
from pci_irq_affinity.config import CONF
from pci_irq_affinity import nova_provider
class TestNovaProvider(unittest.TestCase):
AUTH_URL = "http://keystone.local"
USERNAME = "my-name"
CACERT = "cacert.pem"
KEYRING_SERVICE = "CGCS"
PASSWORD = "my-pass"
def setUp(self):
CONF.set_override("auth_url", self.AUTH_URL, group="openstack")
CONF.set_override("username", self.USERNAME, group="openstack")
CONF.set_override("cacert", self.CACERT, group="openstack")
CONF.set_override("keyring_service", self.KEYRING_SERVICE,
group="openstack")
def tearDown(self):
CONF.clear_override("auth_url", group="openstack")
CONF.clear_override("username", group="openstack")
CONF.clear_override("cacert", group="openstack")
CONF.clear_override("keyring_service", group="openstack")
CONF.clear_override("password", group="openstack")
def test__get_keystone_creds(self):
CONF.set_override("password", "my-pass", group="openstack")
instance = nova_provider.NovaProvider()
creds = instance._get_keystone_creds()
expected = {
"auth_url": self.AUTH_URL,
"username": self.USERNAME,
"password": self.PASSWORD,
# "project_domain_name": None,
# "project_name": None,
# "user_domain_name": None
}
self.assertDictEqual(expected, creds)
@mock.patch.object(nova_provider, "keyring")
def test__get_keystone_creds_no_password(self, mock_keyring):
keyring_pass = "keyring-pass"
mock_keyring.get_password.return_value = keyring_pass
instance = nova_provider.NovaProvider()
mock_keyring.get_password.reset_mock()
creds = instance._get_keystone_creds()
expected = {
"auth_url": self.AUTH_URL,
"username": self.USERNAME,
"password": keyring_pass,
# "project_domain_name": None,
# "project_name": None,
}
self.assertDictEqual(expected, creds)
mock_keyring.get_password.assert_called_once_with(
self.KEYRING_SERVICE,
self.USERNAME,
)

82
utilities/pci-irq-affinity-agent/pci_irq_affinity/tox.ini Normal file
View File

@ -0,0 +1,82 @@
[tox]
envlist = py27,flake8
minversion = 2.3
skipsdist = True
[testenv]
sitepackages = False
basepython = python2.7
install_command = pip install -vvv -U {opts} {packages}
setenv =
VIRTUAL_ENV={envdir}
OS_STDOUT_CAPTURE=1
OS_STDERR_CAPTURE=1
OS_DEBUG=1
OS_LOG_CAPTURE=1
deps =
-r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
whitelist_externals =
bash
commands =
pytest tests/
[testenv:py36]
basepython = python3.6
[testenv:py39]
basepython = python3.9
[flake8]
# Note: hacking pulls in flake8 2.5.5 which can not parse an ignore list spanning multiple lines
# F errors are high priority to fix. W are warnings. E series are pep8, H series are hacking
# F401 'FOO' imported but unused
# F841 local variable 'FOO' is assigned to but never used
# W291 trailing whitespace
# W391 blank line at end of file
# W503 line break before binary operator
# W504 line break after binary operator
# W605 invalid escape sequence '\s'
# E128 continuation line under-indented for visual indent
# E221 multiple spaces before operator
# E226 missing whitespace around arithmetic operator
# E227 missing whitespace around bitwise or shift operator
# E241 multiple spaces after ','
# E265 block comment should start with '# '
# E302 expected 2 blank lines, found 1
# E305 expected 2 blank lines after class or function definition, found 1
# E501 line too long
# E502 the backslash is redundant between brackets
# E702 multiple statements on one line (semicolon)
# E722 do not use bare 'except'
# H101 is TODO
# H104 File contains nothing but comments
# H201 no 'except:' at least use 'except Exception:'
# H238 old style class declaration, use new style (inherit from `object`)
# H306 imports not in alphabetical order (sys, re)
# H401 docstring should not start with a space
# H405 multi line docstring summary not separated with an empty line
ignore = F401,F841,W291,W391,W503,W504,W605,E128,E221,E226,E227,E241,E265,E302,E305,E501,E502,E702,E722,H101,H104,H201,H238,H306,H401,H405
# max-line-length is not referenced if E501 is suppressed
max-line-length=80
[testenv:flake8]
commands =
flake8
[testenv:pylint]
commands = pylint {posargs} pci_irq_affinity --rcfile=../../../pylint.rc
[testenv:bandit]
description = Dummy environment to allow bandit to be run in subdir tox
[testenv:bindep]
# Do not install any requirements. We want this to be fast and work even if
# system dependencies are missing, since it's used to tell you what system
# dependencies are missing! This also means that bindep must be installed
# separately, outside of the requirements files, and develop mode disabled
# explicitly to avoid unnecessarily installing the checked-out repo too (this
# further relies on "tox.skipsdist = True" above).
deps = bindep
commands = bindep test
usedevelop = False