mypy: enable for all non-test modules

The patch also bumps mypy to the latest release on pypi. It fixes one
discrepancy in error reporting between different basepython versions.

Change-Id: I359e82463911821568be7939f9c08209401d0d08

mypy.ini

@@ -15,71 +15,4 @@ follow_imports = silent
 # NOTE: Gradually enable type checking for each package.
 # Finally, when the whole repo is migrated this option can be deleted
 # and rules applied to the whole repo.
-exclude = (?x)(
+exclude = (?x)(^neutron/tests/$)
-    ^neutron/agent/linux/dhcp.py
-    | ^neutron/agent/metadata/agent.py$
-    | ^neutron/agent/metadata/driver.py$
-    | ^neutron/agent/ovn/extensions/metadata.py$
-    | ^neutron/agent/ovn/metadata/driver.py$
-    | ^neutron/agent/ovn/metadata/server.py$
-    | ^neutron/agent/securitygroups_rpc.py$
-    | ^neutron/api/rpc/callbacks/version_manager.py$
-    | ^neutron/conf/db/migration_cli.py$
-    | ^neutron/db/l3_dvr_db.py$
-    | ^neutron/db/l3_hamode_db.py$
-    | ^neutron/db/migration/alembic_migrations/env.py$
-    | ^neutron/extensions/tagging.py$
-    | ^neutron/manager.py$
-    | ^neutron/objects/address_group.py$
-    | ^neutron/objects/address_scope.py$
-    | ^neutron/objects/agent.py$
-    | ^neutron/objects/auto_allocate.py$
-    | ^neutron/objects/conntrack_helper.py$
-    | ^neutron/objects/flavor.py$
-    | ^neutron/objects/floatingip.py$
-    | ^neutron/objects/ipam.py$
-    | ^neutron/objects/l3_hamode.py$
-    | ^neutron/objects/l3agent.py$
-    | ^neutron/objects/local_ip.py$
-    | ^neutron/objects/logapi/logging_resource.py$
-    | ^neutron/objects/metering.py$
-    | ^neutron/objects/ndp_proxy.py$
-    | ^neutron/objects/network.py$
-    | ^neutron/objects/network_segment_range.py$
-    | ^neutron/objects/plugins/ml2/flatallocation.py$
-    | ^neutron/objects/plugins/ml2/geneveallocation.py$
-    | ^neutron/objects/plugins/ml2/greallocation.py$
-    | ^neutron/objects/plugins/ml2/vlanallocation.py$
-    | ^neutron/objects/plugins/ml2/vxlanallocation.py$
-    | ^neutron/objects/port/extensions/allowedaddresspairs.py$
-    | ^neutron/objects/port/extensions/data_plane_status.py$
-    | ^neutron/objects/port/extensions/extra_dhcp_opt.py$
-    | ^neutron/objects/port/extensions/port_device_profile.py$
-    | ^neutron/objects/port/extensions/port_hardware_offload_type.py$
-    | ^neutron/objects/port/extensions/port_hints.py$
-    | ^neutron/objects/port/extensions/port_numa_affinity_policy.py$
-    | ^neutron/objects/port/extensions/port_security.py$
-    | ^neutron/objects/port/extensions/port_trusted.py$
-    | ^neutron/objects/port/extensions/uplink_status_propagation.py$
-    | ^neutron/objects/port_forwarding.py$
-    | ^neutron/objects/ports.py$
-    | ^neutron/objects/provisioning_blocks.py$
-    | ^neutron/objects/qos/binding.py$
-    | ^neutron/objects/qos/policy.py$
-    | ^neutron/objects/qos/rule.py$
-    | ^neutron/objects/quota.py$
-    | ^neutron/objects/router.py$
-    | ^neutron/objects/securitygroup.py$
-    | ^neutron/objects/securitygroup_default_rules.py$
-    | ^neutron/objects/servicetype.py$
-    | ^neutron/objects/subnet.py$
-    | ^neutron/objects/subnetpool.py$
-    | ^neutron/objects/tag.py$
-    | ^neutron/objects/trunk.py$
-    | ^neutron/plugins/ml2/drivers/linuxbridge/agent/arp_protect.py$
-    | ^neutron/plugins/ml2/drivers/ovn/mech_driver/ovsdb/commands.py$
-    | ^neutron/plugins/ml2/drivers/ovn/mech_driver/ovsdb/ovn_client.py$
-    | ^neutron/plugins/ml2/drivers/ovn/mech_driver/ovsdb/ovsdb_monitor.py$
-    | ^neutron/tests/$
-    )
-

neutron/agent/metadata/driver_base.py

@@ -47,8 +47,8 @@ listen listener
 
 
 class HaproxyConfiguratorBase(metaclass=abc.ABCMeta):
-    PROXY_CONFIG_DIR = None
+    PROXY_CONFIG_DIR: str
-    HEADER_CONFIG_TEMPLATE = None
+    HEADER_CONFIG_TEMPLATE: str
 
     def __init__(self, network_id, router_id, unix_socket_path, host, port,
                  user, group, state_path, pid_file, rate_limiting_config,

neutron/agent/metadata/proxy_base.py

@@ -40,8 +40,8 @@ MODE_MAP = {
 
 
 class MetadataProxyHandlerBase(metaclass=abc.ABCMeta):
-    NETWORK_ID_HEADER = None
+    NETWORK_ID_HEADER: str
-    ROUTER_ID_HEADER = None
+    ROUTER_ID_HEADER: str
 
     def __init__(self, conf, has_cache=False):
         self.conf = conf

neutron/agent/ovn/extensions/metadata.py

@@ -126,10 +126,18 @@ class MetadataExtension(extension_manager.OVNAgentExtension,
     def nb_idl(self):
         return self.agent_api.nb_idl
 
+    @nb_idl.setter
+    def nb_idl(self, val):
+        self.agent_api.nb_idl = val
+
     @property
     def sb_idl(self):
         return self.agent_api.sb_idl
 
+    @sb_idl.setter
+    def sb_idl(self, val):
+        self.agent_api.sb_idl = val
+
     @property
     def ovs_idl(self):
         return self.agent_api.ovs_idl

neutron/agent/ovn/metadata/server.py

@@ -30,6 +30,7 @@ LOG = logging.getLogger(__name__)
 
 class MetadataProxyHandler(proxy_base.MetadataProxyHandlerBase):
     NETWORK_ID_HEADER = 'X-OVN-Network-ID'
+    ROUTER_ID_HEADER = ''
 
     def __init__(self, conf, chassis, sb_idl):
         super().__init__(conf)

neutron/agent/securitygroups_rpc.py

@@ -66,6 +66,40 @@ def disable_security_group_extension_by_config(aliases):
         _disable_extension(sg_rules_default_sg_def.ALIAS, aliases)
 
 
+def skip_if_noopfirewall_or_firewall_disabled(func):
+    @functools.wraps(func)
+    def decorated_function(self, *args, **kwargs):
+        if self.noopfirewall_or_firewall_disabled:
+            LOG.info("Skipping method %s as firewall is disabled or "
+                     "configured as NoopFirewallDriver.", func.__name__)
+            return
+        return func(self, *args, **kwargs)
+    return decorated_function
+
+
+def _port_filter_wait(func):
+    """Decorator to wait for the latest port filter lock to be released"""
+    @functools.wraps(func)
+    def decorated_function(self, *args, **kwargs):
+        with self._latest_port_filter_lock.read_lock():
+            return func(self, *args, **kwargs)
+    return decorated_function
+
+
+def _port_filter_lock(func):
+    """Decorator to acquire a new lock while applying port filters"""
+    @functools.wraps(func)
+    def decorated_function(self, *args, **kwargs):
+        lock = lockutils.ReaderWriterLock()
+        # Tracking the most recent lock at the instance level allows
+        # waiters to only wait for the most recent lock to be released
+        # instead of waiting until all locks have been released.
+        self._latest_port_filter_lock = lock
+        with lock.write_lock():
+            return func(self, *args, **kwargs)
+    return decorated_function
+
+
 class SecurityGroupAgentRpc:
     """Enables SecurityGroup agent support in agent implementations."""
 
@@ -97,27 +131,6 @@ class SecurityGroupAgentRpc:
                 trusted_devices.append(device_id)
         return trusted_devices
 
-    def _port_filter_lock(func):
-        """Decorator to acquire a new lock while applying port filters"""
-        @functools.wraps(func)
-        def decorated_function(self, *args, **kwargs):
-            lock = lockutils.ReaderWriterLock()
-            # Tracking the most recent lock at the instance level allows
-            # waiters to only wait for the most recent lock to be released
-            # instead of waiting until all locks have been released.
-            self._latest_port_filter_lock = lock
-            with lock.write_lock():
-                return func(self, *args, **kwargs)
-        return decorated_function
-
-    def _port_filter_wait(func):
-        """Decorator to wait for the latest port filter lock to be released"""
-        @functools.wraps(func)
-        def decorated_function(self, *args, **kwargs):
-            with self._latest_port_filter_lock.read_lock():
-                return func(self, *args, **kwargs)
-        return decorated_function
-
     def init_firewall(self, defer_refresh_firewall=False,
                       integration_bridge=None):
         firewall_driver = cfg.CONF.SECURITYGROUP.firewall_driver or 'noop'
@@ -135,18 +148,6 @@ class SecurityGroupAgentRpc:
         # deferred refresh is enabled.
         self.devices_to_refilter = set()
 
-    def skip_if_noopfirewall_or_firewall_disabled(func):
-        @functools.wraps(func)
-        def decorated_function(self, *args, **kwargs):
-            if self.noopfirewall_or_firewall_disabled:
-                LOG.info("Skipping method %s as firewall is disabled "
-                         "or configured as NoopFirewallDriver.",
-                         func.__name__)
-            else:
-                return func(self,  # pylint: disable=not-callable
-                            *args, **kwargs)
-        return decorated_function
-
     @skip_if_noopfirewall_or_firewall_disabled
     def init_ovs_dvr_firewall(self, dvr_agent):
         dvr_agent.set_firewall(self.firewall)

neutron/api/rpc/callbacks/version_manager.py

@@ -12,6 +12,7 @@
 
 import collections
 import copy
+from dataclasses import dataclass
 import pprint
 import time
 
@@ -39,9 +40,13 @@ def _import_agents_db():
     return importutils.import_module('neutron.db.agents_db')
 
 
-AgentConsumer = collections.namedtuple('AgentConsumer', ['agent_type',
+@dataclass(frozen=True)
-                                                         'host'])
+class AgentConsumer:
-AgentConsumer.__repr__ = lambda self: '%s@%s' % self
+    agent_type: str
+    host: str
+
+    def __repr__(self):
+        return f'{self.agent_type}@{self.host}'
 
 
 class ResourceConsumerTracker:

neutron/conf/db/migration_cli.py

@@ -11,6 +11,7 @@
 #    under the License.
 
 from importlib.metadata import entry_points
+import sys
 
 from oslo_config import cfg
 
@@ -19,13 +20,12 @@ from neutron._i18n import _
 
 MIGRATION_ENTRYPOINTS = 'neutron.db.alembic_migrations'
 
-try:
+if sys.version_info >= (3, 10):
     migration_entrypoints = {
         entrypoint.name: entrypoint
         for entrypoint in entry_points(group=MIGRATION_ENTRYPOINTS)
     }
-except TypeError:
+else:
-    # For python < 3.10
     migration_entrypoints = {
         entrypoint.name: entrypoint
         for entrypoint in entry_points()[MIGRATION_ENTRYPOINTS]

neutron/db/l3_db.py

@@ -89,7 +89,7 @@ class L3_NAT_dbonly_mixin(l3.RouterPluginBase,
                           st_attr.StandardAttrDescriptionMixin):
     """Mixin class to add L3/NAT router methods to db_base_plugin_v2."""
 
-    router_device_owners = (
+    router_device_owners: tuple[str, ...] = (
         DEVICE_OWNER_HA_REPLICATED_INT,
         DEVICE_OWNER_ROUTER_INTF,
         DEVICE_OWNER_ROUTER_GW,

neutron/db/migration/alembic_migrations/env.py

@@ -36,7 +36,7 @@ MYSQL_ENGINE = None
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
 config = context.config
-neutron_config = config.neutron_config
+neutron_config = config.neutron_config  # type:ignore[attr-defined]
 
 # set the target for 'autogenerate' support
 target_metadata = model_base.BASEV2.metadata

neutron/extensions/tagging.py

@@ -14,6 +14,7 @@
 import abc
 import copy
 import functools
+import typing
 
 from neutron_lib.api.definitions import port
 from neutron_lib.api import extensions as api_extensions
@@ -51,6 +52,8 @@ TAG_ATTRIBUTE_MAP = {
     NOT_TAGS_ANY: {'allow_post': False, 'allow_put': False,
                    'is_visible': False, 'is_filter': True},
 }
+
+TAG_ATTRIBUTE_MAP_PORTS: dict[str, typing.Any]
 TAG_ATTRIBUTE_MAP_PORTS = copy.deepcopy(TAG_ATTRIBUTE_MAP)
 TAG_ATTRIBUTE_MAP_PORTS[TAGS] = {
         'allow_post': True, 'allow_put': False,

neutron/manager.py

@@ -35,7 +35,8 @@ LOG = logging.getLogger(__name__)
 CORE_PLUGINS_NAMESPACE = 'neutron.core_plugins'
 
 
-class ManagerMeta(profiler.TracedMeta, type(periodic_task.PeriodicTasks)):
+class ManagerMeta(profiler.TracedMeta,
+                  type(periodic_task.PeriodicTasks)):  # type:ignore[misc]
     pass
 
 

neutron/objects/base.py

@@ -18,8 +18,10 @@ import functools
 import itertools
 import sys
 import traceback
+import typing
 
 from neutron_lib.db import api as db_api
+from neutron_lib.db import model_base
 from neutron_lib.db import standard_attr
 from neutron_lib import exceptions as n_exc
 from neutron_lib.objects import exceptions as o_exc
@@ -434,11 +436,11 @@ class DeclarativeObject(abc.ABCMeta):
 
 class NeutronDbObject(NeutronObject, metaclass=DeclarativeObject):
 
-    # should be overridden for all persistent objects
+    # should be set for all persistent objects
-    db_model = None
+    db_model: typing.Optional[model_base.BASEV2] = None
 
-    # should be overridden for all rbac aware objects
+    # should be set for all rbac aware objects
-    rbac_db_cls = None
+    rbac_db_cls: typing.Optional[model_base.BASEV2] = None
 
     primary_keys = ['id']
 
@@ -456,7 +458,7 @@ class NeutronDbObject(NeutronObject, metaclass=DeclarativeObject):
     # E.g. all the port extension will use 'port_id' as key.
     foreign_keys = {}
 
-    fields_no_update = []
+    fields_no_update: list[str] = []
 
     # dict with name mapping: {'field_name_in_object': 'field_name_in_db'}
     # It can be used also as DB relationship mapping to synthetic fields name.

neutron/objects/port_forwarding.py

@@ -65,9 +65,7 @@ class PortForwarding(base.NeutronDbObject):
                                   'created_at']
 
     synthetic_fields = ['floating_ip_address', 'router_id']
-    fields_no_update = {
+    fields_no_update = ['id', 'floatingip_id']
-        'id', 'floatingip_id'
-    }
 
     def __eq__(self, other):
         for attr in self.fields:

neutron/objects/ports.py

@@ -213,7 +213,7 @@ class IPAllocation(base.NeutronDbObject):
         'ip_address': obj_fields.IPAddressField(),
     }
 
-    fields_no_update = fields.keys()
+    fields_no_update = list(fields.keys())
 
     primary_keys = ['subnet_id', 'network_id', 'ip_address']
 

neutron/objects/qos/binding.py

@@ -17,6 +17,7 @@ import abc
 
 from neutron_lib.db import api as db_api
 from neutron_lib.objects import common_types
+import sqlalchemy as sa
 from sqlalchemy import and_
 from sqlalchemy import exists
 
@@ -28,7 +29,8 @@ from neutron.objects import base
 
 class _QosPolicyBindingMixin(metaclass=abc.ABCMeta):
 
-    _bound_model_id = None
+    # must be set by the subclass
+    _bound_model_id: sa.Column
 
     @classmethod
     def get_bound_ids(cls, context, policy_id):

neutron/objects/qos/rule.py

@@ -65,10 +65,9 @@ class QosRule(base.NeutronDbObject, metaclass=abc.ABCMeta):
 
     fields_no_update = ['id', 'qos_policy_id']
 
-    # should be redefined in subclasses
+    # must be redefined in subclasses
-    rule_type = None
+    rule_type: str
-
+    duplicates_compare_fields: tuple[str, ...] = ()
-    duplicates_compare_fields = ()
 
     def duplicates(self, other_rule):
         """Returns True if rules have got same values in fields defined in
@@ -139,7 +138,7 @@ class QosBandwidthLimitRule(QosRule):
             default=constants.EGRESS_DIRECTION)
     }
 
-    duplicates_compare_fields = ['direction']
+    duplicates_compare_fields = ('direction',)
 
     rule_type = qos_consts.RULE_TYPE_BANDWIDTH_LIMIT
 
@@ -166,7 +165,7 @@ class QosMinimumBandwidthRule(QosRule):
         'direction': common_types.FlowDirectionEnumField(),
     }
 
-    duplicates_compare_fields = ['direction']
+    duplicates_compare_fields = ('direction',)
 
     rule_type = qos_consts.RULE_TYPE_MINIMUM_BANDWIDTH
 
@@ -183,7 +182,7 @@ class QosPacketRateLimitRule(QosRule):
             default=constants.EGRESS_DIRECTION)
     }
 
-    duplicates_compare_fields = ['direction']
+    duplicates_compare_fields = ('direction',)
 
     rule_type = qos_consts.RULE_TYPE_PACKET_RATE_LIMIT
 
@@ -198,6 +197,6 @@ class QosMinimumPacketRateRule(QosRule):
         'direction': common_types.FlowDirectionAndAnyEnumField(),
     }
 
-    duplicates_compare_fields = ['direction']
+    duplicates_compare_fields = ('direction',)
 
     rule_type = qos_consts.RULE_TYPE_MINIMUM_PACKET_RATE

neutron/plugins/ml2/drivers/linuxbridge/agent/arp_protect.py

@@ -14,6 +14,7 @@
 #    under the License.
 
 import netaddr
+from neutron_lib import exceptions
 from neutron_lib.utils import net
 from oslo_concurrency import lockutils
 from oslo_log import log as logging
@@ -221,9 +222,16 @@ def _delete_mac_spoofing_protection(vifs, current_rules, table, chain):
 NAMESPACE = None
 
 
+def _is_retriable_failure(e):
+    if isinstance(e, exceptions.ProcessExecutionError):
+        if e.returncode in [255, 4]:
+            return True
+    return False
+
+
 @tenacity.retry(
     wait=tenacity.wait_exponential(multiplier=0.02),
-    retry=tenacity.retry_if_exception(lambda e: e.returncode in [255, 4]),
+    retry=tenacity.retry_if_exception(_is_retriable_failure),
     reraise=True
 )
 def ebtables(comm, table='nat'):

neutron/plugins/ml2/drivers/ovn/mech_driver/ovsdb/commands.py

@@ -12,6 +12,8 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import abc
+
 from oslo_utils import timeutils
 from ovsdbapp.backend.ovs_idl import command
 from ovsdbapp.backend.ovs_idl import idlutils
@@ -719,8 +721,8 @@ class SetStaticRouteCommand(command.BaseCommand):
             raise RuntimeError(msg)
 
 
-class UpdateObjectExtIdsCommand(command.BaseCommand):
+class UpdateObjectExtIdsCommand(command.BaseCommand, metaclass=abc.ABCMeta):
-    table = None
+    table: str
     field = 'name'
 
     def __init__(self, api, record, external_ids, if_exists):

neutron/plugins/ml2/drivers/ovn/mech_driver/ovsdb/ovn_client.py

@@ -95,7 +95,7 @@ OvnPortInfo = collections.namedtuple(
 )
 
 
-GW_INFO = collections.namedtuple('GatewayInfo', ['network_id', 'subnet_id',
+GW_INFO = collections.namedtuple('GW_INFO', ['network_id', 'subnet_id',
                                              'router_ip', 'gateway_ip',
                                              'ip_version', 'ip_prefix'])
 

neutron/plugins/ml2/drivers/ovn/mech_driver/ovsdb/ovsdb_monitor.py

@@ -45,8 +45,8 @@ CONF = cfg.CONF
 LOG = log.getLogger(__name__)
 
 
-class BaseEvent(row_event.RowEvent):
+class BaseEvent(row_event.RowEvent, metaclass=abc.ABCMeta):
-    table = None
+    table: str
     events = tuple()
 
     def __init__(self):

tox.ini

@@ -140,7 +140,7 @@ deps =
   bandit>=1.7.5 # Apache-2.0
   flake8-import-order>=0.18.2,<0.19.0 # LGPLv3
   pylint==3.2.0 # GPLv2
-  mypy==1.11.2
+  mypy==1.13.0
 commands=
   # If it is easier to add a check via a shell script, consider adding it in this file
   bash ./tools/misc-sanity-checks.sh