Extend ML2 core plugin

The ML2Plus core plugin extends the ML2 plugin with several driver API
features that are needed for APIC AIM support. An extended
MechanismDriver abstract base class adds an ensure_tenant() method
that is called before any transaction creating a new resource, and
(soon) adds precommit and postcommit calls for operations on
additional resources such as address scope. An extended
ExtensionDriver base class will support extending those additional
resources.

ML2 configuration is unchanged, and compatibility is maintained with
all existing ML2 drivers.

Change-Id: I4d4fcd1d368650ba5b5c1e13b973a349c0917eaf

gbpservice/neutron/plugins/ml2plus/driver_api.py

@@ -0,0 +1,47 @@
+# Copyright (c) 2016 Cisco Systems Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import abc
+import six
+
+from neutron.plugins.ml2 import driver_api
+
+
+@six.add_metaclass(abc.ABCMeta)
+class MechanismDriver(driver_api.MechanismDriver):
+
+    # REVISIT(rkukura): Is this needed for all operations, or just for
+    # create operations? If its needed for all operations, should the
+    # method be specific to the resource and operation, and include
+    # the request data (i.e. update_network_pretransaction(self,
+    # data))?
+    def ensure_tenant(self, plugin_context, tenant_id):
+        """Ensure tenant known before creating resource.
+
+        :param plugin_context: Plugin request context.
+        :param tenant_id: Tenant owning resource about to be created.
+
+        Called before the start of a transaction creating any new core
+        resource, allowing any needed tenant-specific processing to be
+        performed.
+        """
+        pass
+
+    # TODO(rkukura): Add precommit/postcommit calls for address_scope,
+    # subnet_pool, and other resources.
+
+
+# TODO(rkukura): Extend ExtensionDriver for address_scope,
+# subnet_pool, and other resources.

gbpservice/neutron/plugins/ml2plus/managers.py

@@ -0,0 +1,39 @@
+# Copyright (c) 2016 Cisco Systems Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from gbpservice.neutron.plugins.ml2plus import driver_api
+
+from neutron._i18n import _LE
+from neutron.plugins.ml2.common import exceptions as ml2_exc
+from neutron.plugins.ml2 import managers
+from oslo_log import log
+
+LOG = log.getLogger(__name__)
+
+
+class MechanismManager(managers.MechanismManager):
+
+    def __init__(self):
+        super(MechanismManager, self).__init__()
+
+    def ensure_tenant(self, plugin_context, tenant_id):
+        for driver in self.ordered_mech_drivers:
+            if isinstance(driver.obj, driver_api.MechanismDriver):
+                try:
+                    driver.obj.ensure_tenant(plugin_context, tenant_id)
+                except Exception:
+                    LOG.exception(_LE("Mechanism driver '%s' failed in "
+                                      "ensure_tenant"), driver.name)
+                    raise ml2_exc.MechanismDriverError(method="ensure_tenant")

gbpservice/neutron/plugins/ml2plus/plugin.py

@@ -0,0 +1,115 @@
+# Copyright (c) 2016 Cisco Systems Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from neutron._i18n import _LI
+from neutron.api.v2 import attributes
+from neutron.db import models_v2
+from neutron.db import securitygroups_db
+from neutron.plugins.ml2 import managers as ml2_managers
+from neutron.plugins.ml2 import plugin as ml2_plugin
+from neutron.quota import resource_registry
+from oslo_log import log
+
+from gbpservice.neutron.plugins.ml2plus import managers
+
+LOG = log.getLogger(__name__)
+
+
+class Ml2PlusPlugin(ml2_plugin.Ml2Plugin):
+
+    """Extend the ML2 core plugin with missing functionality.
+
+    The standard ML2 core plugin in Neutron is missing a few features
+    needed for optimal APIC AIM support. This class adds those
+    features, while maintaining compatibility with all standard ML2
+    drivers and configuration. The only change necessary to use
+    ML2Plus is to register the ml2plus entry point instead of the ml2
+    entry port as Neutron's core_plugin. Drivers that need these
+    features inherit from the extended MechanismDriver and
+    ExtensionDriver abstract base classes.
+    """
+
+    __native_bulk_support = True
+    __native_pagination_support = True
+    __native_sorting_support = True
+
+    # Override and bypass immediate base class's __init__ in order to
+    # instantate extended manager class(es).
+    @resource_registry.tracked_resources(
+        network=models_v2.Network,
+        port=models_v2.Port,
+        subnet=models_v2.Subnet,
+        subnetpool=models_v2.SubnetPool,
+        security_group=securitygroups_db.SecurityGroup,
+        security_group_rule=securitygroups_db.SecurityGroupRule)
+    def __init__(self):
+        LOG.info(_LI("Ml2Plus initializing"))
+        # First load drivers, then initialize DB, then initialize drivers
+        self.type_manager = ml2_managers.TypeManager()
+        self.extension_manager = ml2_managers.ExtensionManager()
+        self.mechanism_manager = managers.MechanismManager()
+        super(ml2_plugin.Ml2Plugin, self).__init__()
+        self.type_manager.initialize()
+        self.extension_manager.initialize()
+        self.mechanism_manager.initialize()
+        self._setup_dhcp()
+        self._start_rpc_notifiers()
+        self.add_agent_status_check(self.agent_health_check)
+        self._verify_service_plugins_requirements()
+        LOG.info(_LI("Modular L2 Plugin (extended) initialization complete"))
+
+    def create_network(self, context, network):
+        self._ensure_tenant(context, network[attributes.NETWORK])
+        return super(Ml2PlusPlugin, self).create_network(context, network)
+
+    def create_network_bulk(self, context, networks):
+        self._ensure_tenant_bulk(context, networks[attributes.NETWORKS],
+                                 attributes.NETWORK)
+        return super(Ml2PlusPlugin, self).create_network_bulk(context,
+                                                              networks)
+
+    def create_subnet(self, context, subnet):
+        self._ensure_tenant(context, subnet[attributes.SUBNET])
+        return super(Ml2PlusPlugin, self).create_subnet(context, subnet)
+
+    def create_subnet_bulk(self, context, subnets):
+        self._ensure_tenant_bulk(context, subnets[attributes.SUBNETS],
+                                 attributes.SUBNET)
+        return super(Ml2PlusPlugin, self).create_subnet_bulk(context,
+                                                             subnets)
+
+    def create_port(self, context, port):
+        self._ensure_tenant(context, port[attributes.PORT])
+        return super(Ml2PlusPlugin, self).create_port(context, port)
+
+    def create_port_bulk(self, context, ports):
+        self._ensure_tenant_bulk(context, ports[attributes.PORTS],
+                                 attributes.PORT)
+        return super(Ml2PlusPlugin, self).create_port_bulk(context,
+                                                           ports)
+
+    # TODO(rkukura): Override address_scope, subnet_pool, and any
+    # other needed resources to ensure tenant and invoke mechanism and
+    # extension drivers.
+
+    def _ensure_tenant(self, context, resource):
+        tenant_id = resource['tenant_id']
+        self.mechanism_manager.ensure_tenant(context, tenant_id)
+
+    def _ensure_tenant_bulk(self, context, resources, singular):
+        tenant_ids = [resource[singular]['tenant_id']
+                      for resource in resources]
+        for tenant_id in set(tenant_ids):
+            self.mechanism_manager.ensure_tenant(context, tenant_id)

gbpservice/neutron/tests/unit/plugins/ml2plus/drivers/mechanism_logger.py

@@ -0,0 +1,37 @@
+# Copyright (c) 2016 Cisco Systems Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from neutron._i18n import _LI
+from neutron.tests.unit.plugins.ml2.drivers import (
+    mechanism_logger as ml2_logger)
+from oslo_log import log
+
+from gbpservice.neutron.plugins.ml2plus import driver_api
+
+LOG = log.getLogger(__name__)
+
+
+class LoggerPlusMechanismDriver(driver_api.MechanismDriver,
+                                ml2_logger.LoggerMechanismDriver):
+    """Mechanism driver that logs all calls and parameters made.
+
+    Generally used for testing and debugging.
+    """
+
+    def initialize(self):
+        LOG.info(_LI("initialize called"))
+
+    def ensure_tenant(self, plugin_context, tenant_id):
+        LOG.info(_LI("ensure_tenant called with tenant_id %s"), tenant_id)

gbpservice/neutron/tests/unit/plugins/ml2plus/test_plugin.py

@@ -0,0 +1,167 @@
+# Copyright (c) 2016 Cisco Systems Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+
+from neutron import manager
+from neutron.plugins.ml2 import config
+from neutron.tests.unit.db import test_db_base_plugin_v2 as test_plugin
+
+from gbpservice.neutron.tests.unit.plugins.ml2plus.drivers import (
+    mechanism_logger as mech_logger)
+
+PLUGIN_NAME = 'gbpservice.neutron.plugins.ml2plus.plugin.Ml2PlusPlugin'
+
+
+# This is just a quick sanity test that basic ML2 plugin functionality
+# is preserved.
+
+class Ml2PlusPluginV2TestCase(test_plugin.NeutronDbPluginV2TestCase):
+
+    def setUp(self):
+        # Enable the test mechanism driver to ensure that
+        # we can successfully call through to all mechanism
+        # driver apis.
+        config.cfg.CONF.set_override('mechanism_drivers',
+                                     ['logger_plus', 'test'],
+                                     'ml2')
+        config.cfg.CONF.set_override('network_vlan_ranges',
+                                     ['physnet1:1000:1099'],
+                                     group='ml2_type_vlan')
+        super(Ml2PlusPluginV2TestCase, self).setUp(PLUGIN_NAME)
+        self.port_create_status = 'DOWN'
+        self.plugin = manager.NeutronManager.get_plugin()
+        self.plugin.start_rpc_listeners()
+
+
+class TestEnsureTenant(Ml2PlusPluginV2TestCase):
+    def test_network(self):
+        with mock.patch.object(mech_logger.LoggerPlusMechanismDriver,
+                               'ensure_tenant') as et:
+            self._make_network(self.fmt, 'net', True, tenant_id='t1')
+            et.assert_called_once_with(mock.ANY, 't1')
+
+    def test_network_bulk(self):
+        with mock.patch.object(mech_logger.LoggerPlusMechanismDriver,
+                               'ensure_tenant') as et:
+            networks = [{'network': {'name': 'n1',
+                                     'tenant_id': 't1'}},
+                        {'network': {'name': 'n2',
+                                     'tenant_id': 't2'}},
+                        {'network': {'name': 'n3',
+                                     'tenant_id': 't1'}}]
+            res = self._create_bulk_from_list(self.fmt, 'network', networks)
+            self.assertEqual(201, res.status_int)
+            et.assert_has_calls([mock.call(mock.ANY, 't1'),
+                                 mock.call(mock.ANY, 't2')],
+                                any_order=True)
+            self.assertEqual(2, et.call_count)
+
+    def test_subnet(self):
+        net = self._make_network(self.fmt, 'net', True)
+
+        with mock.patch.object(mech_logger.LoggerPlusMechanismDriver,
+                               'ensure_tenant') as et:
+            self._make_subnet(self.fmt, net, None, '10.0.0.0/24',
+                              tenant_id='t1')
+            et.assert_called_once_with(mock.ANY, 't1')
+
+    def test_subnet_bulk(self):
+        net = self._make_network(self.fmt, 'net', True)
+        network_id = net['network']['id']
+
+        with mock.patch.object(mech_logger.LoggerPlusMechanismDriver,
+                               'ensure_tenant') as et:
+            subnets = [{'subnet': {'name': 's1',
+                                   'network_id': network_id,
+                                   'ip_version': 4,
+                                   'cidr': '10.0.1.0/24',
+                                   'tenant_id': 't1'}},
+                       {'subnet': {'name': 's2',
+                                   'network_id': network_id,
+                                   'ip_version': 4,
+                                   'cidr': '10.0.2.0/24',
+                                   'tenant_id': 't2'}},
+                       {'subnet': {'name': 'n3',
+                                   'network_id': network_id,
+                                   'ip_version': 4,
+                                   'cidr': '10.0.3.0/24',
+                                   'tenant_id': 't1'}}]
+            res = self._create_bulk_from_list(self.fmt, 'subnet', subnets)
+            self.assertEqual(201, res.status_int)
+            et.assert_has_calls([mock.call(mock.ANY, 't1'),
+                                 mock.call(mock.ANY, 't2')],
+                                any_order=True)
+            self.assertEqual(2, et.call_count)
+
+    def test_port(self):
+        net = self._make_network(self.fmt, 'net', True)
+
+        with mock.patch.object(mech_logger.LoggerPlusMechanismDriver,
+                               'ensure_tenant') as et:
+            self._make_port(self.fmt, net['network']['id'], tenant_id='t1')
+            et.assert_called_once_with(mock.ANY, 't1')
+
+    def test_port_bulk(self):
+        net = self._make_network(self.fmt, 'net', True)
+        network_id = net['network']['id']
+
+        with mock.patch.object(mech_logger.LoggerPlusMechanismDriver,
+                               'ensure_tenant') as et:
+            ports = [{'port': {'name': 's1',
+                               'network_id': network_id,
+                               'tenant_id': 't1'}},
+                     {'port': {'name': 's2',
+                               'network_id': network_id,
+                               'tenant_id': 't2'}},
+                     {'port': {'name': 'n3',
+                               'network_id': network_id,
+                               'tenant_id': 't1'}}]
+            res = self._create_bulk_from_list(self.fmt, 'port', ports)
+            self.assertEqual(201, res.status_int)
+            et.assert_has_calls([mock.call(mock.ANY, 't1'),
+                                 mock.call(mock.ANY, 't2')],
+                                any_order=True)
+            self.assertEqual(2, et.call_count)
+
+
+class TestMl2BasicGet(test_plugin.TestBasicGet,
+                      Ml2PlusPluginV2TestCase):
+    pass
+
+
+class TestMl2V2HTTPResponse(test_plugin.TestV2HTTPResponse,
+                            Ml2PlusPluginV2TestCase):
+    pass
+
+
+class TestMl2PortsV2(test_plugin.TestPortsV2,
+                     Ml2PlusPluginV2TestCase):
+    pass
+
+
+class TestMl2NetworksV2(test_plugin.TestNetworksV2,
+                        Ml2PlusPluginV2TestCase):
+    pass
+
+
+class TestMl2SubnetsV2(test_plugin.TestSubnetsV2,
+                       Ml2PlusPluginV2TestCase):
+    pass
+
+
+class TestMl2SubnetPoolsV2(test_plugin.TestSubnetPoolsV2,
+                           Ml2PlusPluginV2TestCase):
+    pass

setup.cfg

@@ -37,6 +37,8 @@ data_files =
 [entry_points]
 console_scripts=
     gbp-db-manage = gbpservice.neutron.db.migration.cli:main
+neutron.core_plugins =
+    ml2plus = gbpservice.neutron.plugins.ml2plus.plugin:Ml2PlusPlugin
 neutron.service_plugins =
     group_policy = gbpservice.neutron.services.grouppolicy.plugin:GroupPolicyPlugin
     servicechain = gbpservice.neutron.services.servicechain.plugins.msc.plugin:ServiceChainPlugin
@@ -56,6 +58,7 @@ gbpservice.neutron.group_policy.policy_drivers =
     oneconvergence_gbp_driver = gbpservice.neutron.services.grouppolicy.drivers.oneconvergence.nvsd_gbp_driver:NvsdGbpDriver
     nuage_gbp_driver = gbpservice.neutron.services.grouppolicy.drivers.nuage.driver:NuageGBPDriver
 neutron.ml2.mechanism_drivers =
+    logger_plus = gbpservice.neutron.tests.unit.plugins.ml2plus.drivers.mechanism_logger:LoggerPlusMechanismDriver
     apic_gbp = gbpservice.neutron.plugins.ml2.drivers.grouppolicy.apic.driver:APICMechanismGBPDriver
     nuage_gbp = gbpservice.neutron.plugins.ml2.drivers.grouppolicy.nuage.driver:NuageMechanismGBPDriver
     odl_gbp = gbpservice.neutron.plugins.ml2.drivers.grouppolicy.odl.driver:OdlMechanismGBPDriver