openstack/os-traits
Code Issues Proposed changes

Automate the import of traits

Browse Source 
The previous patch broke apart the single monolithic declaration of
traits into a branching directory structure. That was great, but it
added a lot of repetitive code to get the subpackages and submodules
into the proper namespace. This patch automates that for any file in the
tree that contains a 'TRAITS' property.

Change-Id: I904789cbec76d83a14e213ce6bb474c9967d588b
This commit is contained in:
EdLeafe
2017-04-05 20:11:02 +00:00
parent 842a43b515
commit 2eb278ce1b
9 changed files with 156 additions and 190 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
os_traits/__init__.py
View File

@@ -12,25 +12,57 @@
# License for the specific language governing permissions and limitations
# under the License.
import importlib
import pkgutil
import sys
import pbr.version
import six
__version__ = pbr.version.VersionInfo(
'os_traits').version_string()
this_name = __name__
this_lib = sys.modules[this_name]
__version__ = pbr.version.VersionInfo(this_name).version_string()
# Any user-specified feature/trait is prefixed with the custom namespace
CUSTOM_NAMESPACE = 'CUSTOM_'
# Each submodule registers its symbols with the os_traits module namespace
from os_traits.hw.cpu import x86 # noqa
from os_traits.hw import nic # noqa
from os_traits.hw.nic import accel # noqa
from os_traits.hw.nic import dcb # noqa
from os_traits.hw.nic import offload # noqa
from os_traits.hw.nic import sriov # noqa
from os_traits.storage import disk # noqa
def symbolize(mod_name, name):
"""Given a reference to a Python module object and a short string name for
a trait, registers a symbol in the module that corresponds to the full
namespaced trait name.
"""
leaf_mod = sys.modules[mod_name]
value_base = '_'.join([m.upper() for m in mod_name.split('.')[1:]])
value = value_base + '_' + name.upper()
setattr(this_lib, value, value) # os_traits.HW_CPU_X86_SSE
setattr(leaf_mod, name.upper(), value) # os_traits.hw.cpu.x86.SSE
def import_submodules(package, recursive=True):
"""Import all submodules of a module, recursively, including subpackages
:param package: package (name or actual module)
:type package: str | module
:rtype: dict[str, types.ModuleType]
"""
if isinstance(package, str):
package = importlib.import_module(package)
for loader, name, is_pkg in pkgutil.walk_packages(package.__path__):
full_name = package.__name__ + '.' + name
test_dir = "%s.tests" % this_name
if test_dir in full_name:
continue
imported = importlib.import_module(full_name)
for prop in getattr(imported, "TRAITS", []):
symbolize(full_name, prop)
if recursive and is_pkg:
import_submodules(full_name)
# This is where the names defined in submodules are imported
import_submodules(sys.modules.get(__name__))
def get_symbol_names(prefix=None):
@@ -39,11 +71,12 @@ def get_symbol_names(prefix=None):
:param prefix: Optional string prefix to filter by. e.g. 'HW_'
"""
prefix = prefix or ""
return [
k for k, v in sys.modules[__name__].__dict__.items()
if isinstance(v, six.string_types) and
not k.startswith('_') and
(prefix is None or v.startswith(prefix))
v.startswith(prefix)
]
@@ -53,11 +86,12 @@ def get_traits(prefix=None):
:param prefix: Optional string prefix to filter by. e.g. 'HW_'
"""
prefix = prefix or ""
return [
v for k, v in sys.modules[__name__].__dict__.items()
if isinstance(v, six.string_types) and
not k.startswith('_') and
(prefix is None or v.startswith(prefix))
v.startswith(prefix)
]

79
os_traits/hw/cpu/x86.py
View File

@@ -12,53 +12,54 @@
# License for the specific language governing permissions and limitations
# under the License.
from os_traits import utils
register = utils.register_fn(__name__)
TRAITS = [
# ref: https://en.wikipedia.org/wiki/Streaming_SIMD_Extensions
register('AVX')
register('AVX2')
register('CLMUL')
register('FMA3')
register('FMA4')
register('F16C')
register('MMX')
register('SSE')
register('SSE2')
register('SSE3')
register('SSSE3')
register('SSE41')
register('SSE42')
register('SSE4A')
register('XOP')
register('3DNOW')
'AVX',
'AVX2',
'CLMUL',
'FMA3',
'FMA4',
'F16C',
'MMX',
'SSE',
'SSE2',
'SSE3',
'SSSE3',
'SSE41',
'SSE42',
'SSE4A',
'XOP',
'3DNOW',
# ref: https://en.wikipedia.org/wiki/AVX-512
register('AVX512F') # foundation
register('AVX512CD') # conflict detection
register('AVX512PF') # prefetch
register('AVX512ER') # exponential + reciprocal
register('AVX512VL') # vector length extensions
register('AVX512BW') # byte + word
register('AVX512DQ') # double word + quad word
'AVX512F', # foundation
'AVX512CD', # conflict detection
'AVX512PF', # prefetch
'AVX512ER', # exponential + reciprocal
'AVX512VL', # vector length extensions
'AVX512BW', # byte + word
'AVX512DQ', # double word + quad word
# ref: https://en.wikipedia.org/wiki/Bit_Manipulation_Instruction_Sets
register('ABM')
register('BMI')
register('BMI2')
register('TBM')
'ABM',
'BMI',
'BMI2',
'TBM',
# ref: https://en.wikipedia.org/wiki/AES_instruction_set
register('AES-NI')
'AES-NI',
# ref: https://en.wikipedia.org/wiki/Intel_SHA_extensions
register('SHA')
'SHA',
# ref: https://en.wikipedia.org/wiki/Intel_MPX
register('MPX')
'MPX',
# ref: https://en.wikipedia.org/wiki/Software_Guard_Extensions
register('SGX')
# ref: https://en.wikipedia.org/wiki/Transactional_Synchronization_Extensions
register('TSX')
'SGX',
# ref:
# https://en.wikipedia.org/wiki/Transactional_Synchronization_Extensions
'TSX',
# ref: https://en.wikipedia.org/wiki/Advanced_Synchronization_Facility
register('ASF')
'ASF',
# ref: https://en.wikipedia.org/wiki/VT-x
register('VMX')
'VMX',
# ref: https://en.wikipedia.org/wiki/AMD-V
register('SVM')
'SVM',
]

18
os_traits/hw/nic/__init__.py
View File

@@ -12,14 +12,12 @@
# License for the specific language governing permissions and limitations
# under the License.
from os_traits import utils
register = utils.register_fn(__name__)
# A few generalized capabilities of some NICs
register('SRIOV') # NIC supports partitioning via SR-IOV
register('MULTIQUEUE') # >1 receive and transmit queues
register('VMDQ') # Virtual machine device queues
# Some NICs allow processing pipelines to be programmed via FPGAs embedded in
# the NIC itself...
register('PROGRAMMABLE_PIPELINE')
TRAITS = [
'SRIOV', # NIC supports partitioning via SR-IOV
'MULTIQUEUE', # >1 receive and transmit queues
'VMDQ', # Virtual machine device queues
# Some NICs allow processing pipelines to be programmed via FPGAs embedded
# in the NIC itself...
'PROGRAMMABLE_PIPELINE',
]

22
os_traits/hw/nic/accel.py
View File

@@ -12,15 +12,13 @@
# License for the specific language governing permissions and limitations
# under the License.
from os_traits import utils
register = utils.register_fn(__name__)
register('SSL') # SSL crypto
register('IPSEC') # IP-Sec crypto
register('TLS') # TLS crypto
register('DIFFIEH') # Diffie-Hellmann crypto
register('RSA') # RSA crypto
register('ECC') # Eliptic Curve crypto
register('LZS') # LZS compression
register('DEFLATE') # Deflate compression
TRAITS = [
'SSL', # SSL crypto
'IPSEC', # IP-Sec crypto
'TLS', # TLS crypto
'DIFFIEH', # Diffie-Hellmann crypto
'RSA', # RSA crypto
'ECC', # Eliptic Curve crypto
'LZS', # LZS compression
'DEFLATE', # Deflate compression
]

12
os_traits/hw/nic/dcb.py
View File

@@ -12,13 +12,11 @@
# License for the specific language governing permissions and limitations
# under the License.
from os_traits import utils
register = utils.register_fn(__name__)
TRAITS = [
# IEEE 802.1Qbb Priority-flow control
register('PFC')
'PFC',
# IEEE 802.1Qaz Enhanced Transmission Selection
register('ETS')
'ETS',
# IEEE 802.1Qau Quantized Congestion Notification
register('QCN')
'QCN',
]

46
os_traits/hw/nic/offload.py
View File

@@ -12,27 +12,25 @@
# License for the specific language governing permissions and limitations
# under the License.
from os_traits import utils
register = utils.register_fn(__name__)
register('TSO') # TCP segmentation
register('GRO') # Generic receive
register('GSO') # Generic segmentation
register('UFO') # UDP Fragmentation
register('LRO') # Large receive
register('LSO') # Large send
register('TCS') # TCP Checksum
register('UCS') # UDP Checksum
register('SCS') # SCTP Checksum
register('L2CRC') # Layer-2 CRC
register('FDF') # Intel Flow-Director Filter
register('RXVLAN') # VLAN receive tunnel segmentation
register('TXVLAN') # VLAN transmit tunnel segmentation
register('VXLAN') # VxLAN tunneling
register('GRE') # GRE tunneling
register('GENEVE') # Geneve tunneling
register('TXUDP') # UDP transmit tunnel segmentation
register('QINQ') # QinQ specification
register('RDMA') # remote direct memory access
register('RXHASH') # receive hashing
TRAITS = [
'TSO', # TCP segmentation
'GRO', # Generic receive
'GSO', # Generic segmentation
'UFO', # UDP Fragmentation
'LRO', # Large receive
'LSO', # Large send
'TCS', # TCP Checksum
'UCS', # UDP Checksum
'SCS', # SCTP Checksum
'L2CRC', # Layer-2 CRC
'FDF', # Intel Flow-Director Filter
'RXVLAN', # VLAN receive tunnel segmentation
'TXVLAN', # VLAN transmit tunnel segmentation
'VXLAN', # VxLAN tunneling
'GRE', # GRE tunneling
'GENEVE', # Geneve tunneling
'TXUDP', # UDP transmit tunnel segmentation
'QINQ', # QinQ specification
'RDMA', # remote direct memory access
'RXHASH', # receive hashing
]

18
os_traits/hw/nic/sriov.py
View File

@@ -12,14 +12,12 @@
# License for the specific language governing permissions and limitations
# under the License.
from os_traits import utils
register = utils.register_fn(__name__)
# The function (virtual or physical) can restrict transmit rates
register('QOS_TX')
# The function (virtual or physical) can restrict receive rates
register('QOS_RX')
# The function (virtual or physical) can set up multiple receive and transmit
TRAITS = [
# Individual virtual functions can restrict transmit rates
'QOS_TX',
# Individual virtual functions can restrict receive rates
'QOS_RX',
# Individual virtual functions can set up multiple receive and transmit
# queues for receive-side scaling
register('MULTIQUEUE')
'MULTIQUEUE',
]

11
os_traits/storage/disk.py
View File

@@ -12,10 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
from os_traits import utils
register = utils.register_fn(__name__)
register('HDD') # spinning oxide
register('SSD') # solid-state disks
TRAITS = [
'HDD', # spinning oxide
'SSD', # solid-state disks
]

56
os_traits/utils.py
View File

@@ -1,56 +0,0 @@
# -*- coding: utf-8 -*-
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import functools
import sys
import os_traits
def symbolize(mod_name, name):
"""Given a reference to a Python module object and a short string name for
a trait, registers a symbol in the module that corresponds to the full
namespaced trait name.
For example, if called like so:
:code:
# In file /os_traits/hw/cpu/x86.py
import functools
from os_traits import utils
mod_register = functools.partial(utils.symbolize, __name__)
mod_register('AVX2')
mod_register('SSE')
Would end up creating the following symbols:
os_traits.hw.cpu.x86.AVX2 with the value of 'HW_CPU_X86_AVX2'
os_traits.hw.cpu.x86.SSE with the value of 'HW_CPU_X86_SSE'
os_traits.HW_CPU_X86_AVX2 with the value of 'HW_CPU_X86_AVX2'
os_traits.HW_CPU_X86_SSE with the value of 'HW_CPU_X86_SSE'
"""
leaf_mod = sys.modules[mod_name]
value_base = '_'.join([m.upper() for m in mod_name.split('.')[1:]])
value = value_base + '_' + name.upper()
setattr(os_traits, value, value) # os_traits.HW_CPU_X86_SSE
setattr(leaf_mod, name.upper(), value) # os_traits.hw.cpu.x86.SSE
def register_fn(mod_name):
return functools.partial(symbolize, mod_name)