Merge "Implement 'get-me-a-network' API building block"

etc/policy.json

@@ -45,6 +45,7 @@
     "get_network:queue_id": "rule:admin_only",
     "create_network:shared": "rule:admin_only",
     "create_network:router:external": "rule:admin_only",
+    "create_network:is_default": "rule:admin_only",
     "create_network:segments": "rule:admin_only",
     "create_network:provider:network_type": "rule:admin_only",
     "create_network:provider:physical_network": "rule:admin_only",
@@ -203,5 +204,6 @@
 
     "create_flavor_service_profile": "rule:admin_only",
     "delete_flavor_service_profile": "rule:admin_only",
-    "get_flavor_service_profile": "rule:regular_user"
+    "get_flavor_service_profile": "rule:regular_user",
+    "get_auto_allocated_topology": "rule:admin_or_owner"
 }

neutron/callbacks/resources.py

@@ -11,6 +11,7 @@
 #    under the License.
 
 # String literals representing core resources.
+EXTERNAL_NETWORK = 'external_network'
 FLOATING_IP = 'floating_ip'
 PORT = 'port'
 PROCESS = 'process'

neutron/db/external_net_db.py

@@ -19,6 +19,10 @@ from sqlalchemy.orm import exc
 from sqlalchemy.sql import expression as expr
 
 from neutron.api.v2 import attributes
+from neutron.callbacks import events
+from neutron.callbacks import exceptions as c_exc
+from neutron.callbacks import registry
+from neutron.callbacks import resources
 from neutron.common import constants as l3_constants
 from neutron.common import exceptions as n_exc
 from neutron.db import db_base_plugin_v2
@@ -36,7 +40,8 @@ class ExternalNetwork(model_base.BASEV2):
     network_id = sa.Column(sa.String(36),
                            sa.ForeignKey('networks.id', ondelete="CASCADE"),
                            primary_key=True)
-
+    # introduced by auto-allocated-topology extension
+    is_default = sa.Column(sa.Boolean(), nullable=True)
     # Add a relationship to the Network model in order to instruct
     # SQLAlchemy to eagerly load this association
     network = orm.relationship(
@@ -106,12 +111,34 @@ class External_net_db_mixin(object):
         if not external_set:
             return
 
+        # TODO(armax): these notifications should switch to *_COMMIT
+        # when the event becomes available, as this block is expected
+        # to be called within a plugin's session
         if external:
-            # expects to be called within a plugin's session
+            try:
+                registry.notify(
+                    resources.EXTERNAL_NETWORK, events.BEFORE_CREATE,
+                    self, context=context,
+                    request=req_data, network=net_data)
+            except c_exc.CallbackFailure as e:
+                # raise the underlying exception
+                raise e.errors[0].error
             context.session.add(ExternalNetwork(network_id=net_data['id']))
+            registry.notify(
+                resources.EXTERNAL_NETWORK, events.AFTER_CREATE,
+                self, context=context,
+                request=req_data, network=net_data)
         net_data[external_net.EXTERNAL] = external
 
     def _process_l3_update(self, context, net_data, req_data):
+        try:
+            registry.notify(
+                resources.EXTERNAL_NETWORK, events.BEFORE_UPDATE,
+                self, context=context,
+                request=req_data, network=net_data)
+        except c_exc.CallbackFailure as e:
+            # raise the underlying exception
+            raise e.errors[0].error
 
         new_value = req_data.get(external_net.EXTERNAL)
         net_id = net_data['id']

neutron/db/migration/alembic_migrations/versions/EXPAND_HEAD

@@ -1 +1 @@
-1df244e556f5
+19f26505c74f

neutron/db/migration/alembic_migrations/versions/mitaka/expand/19f26505c74f_auto_allocated_topology.py

@@ -0,0 +1,47 @@
+# Copyright 2015-2016 Hewlett Packard Enterprise Development Company, LP
+#
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+""" Auto Allocated Topology - aka Get-Me-A-Network
+
+Revision ID: 19f26505c74f
+Revises: 1df244e556f5
+Create Date: 2015-11-20 11:27:53.419742
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '19f26505c74f'
+down_revision = '1df244e556f5'
+
+
+def upgrade():
+
+    op.create_table(
+        'auto_allocated_topologies',
+        sa.Column('tenant_id', sa.String(length=255), primary_key=True),
+        sa.Column('network_id', sa.String(length=36), nullable=False),
+        sa.Column('router_id', sa.String(length=36), nullable=True),
+        sa.ForeignKeyConstraint(['network_id'], ['networks.id'],
+                                ondelete='CASCADE'),
+        sa.ForeignKeyConstraint(['router_id'], ['routers.id'],
+                                ondelete='SET NULL'),
+    )
+
+    op.add_column('externalnetworks',
+                  sa.Column('is_default', sa.Boolean(), nullable=True))

neutron/db/migration/models/head.py

@@ -54,6 +54,7 @@ from neutron.plugins.ml2.drivers import type_gre  # noqa
 from neutron.plugins.ml2.drivers import type_vlan  # noqa
 from neutron.plugins.ml2.drivers import type_vxlan  # noqa
 from neutron.plugins.ml2 import models  # noqa
+from neutron.services.auto_allocate import models  # noqa
 
 
 def get_metadata():

neutron/extensions/auto_allocated_topology.py

@@ -0,0 +1,80 @@
+# Copyright 2015-2016 Hewlett Packard Enterprise Development Company, LP
+#
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from neutron.api import extensions
+from neutron.api.v2 import attributes as attr
+from neutron.api.v2 import base
+from neutron.services.auto_allocate import plugin
+
+RESOURCE_NAME = "auto_allocated_topology"
+COLLECTION_NAME = "auto_allocated_topologies"
+IS_DEFAULT = "is_default"
+EXT_ALIAS = RESOURCE_NAME.replace('_', '-')
+
+RESOURCE_ATTRIBUTE_MAP = {
+    COLLECTION_NAME: {
+        'id': {'allow_post': False, 'allow_put': False,
+               'validate': {'type:uuid': None},
+               'is_visible': True},
+        'tenant_id': {'allow_post': False, 'allow_put': False,
+                      'validate': {'type:uuid': None},
+                      'is_visible': True},
+    },
+    'networks': {IS_DEFAULT: {'allow_post': True,
+                              'allow_put': True,
+                              'default': False,
+                              'is_visible': True,
+                              'convert_to': attr.convert_to_boolean,
+                              'enforce_policy': True,
+                              'required_by_policy': True}},
+}
+
+
+class Auto_allocated_topology(extensions.ExtensionDescriptor):
+
+    @classmethod
+    def get_name(cls):
+        return "Auto Allocated Topology Services"
+
+    @classmethod
+    def get_alias(cls):
+        return EXT_ALIAS
+
+    @classmethod
+    def get_description(cls):
+        return "Auto Allocated Topology Services."
+
+    @classmethod
+    def get_updated(cls):
+        return "2016-01-01T00:00:00-00:00"
+
+    @classmethod
+    def get_resources(cls):
+        params = RESOURCE_ATTRIBUTE_MAP.get(COLLECTION_NAME, dict())
+        controller = base.create_resource(COLLECTION_NAME,
+                                          EXT_ALIAS,
+                                          plugin.Plugin.get_instance(),
+                                          params, allow_bulk=False)
+        return [extensions.ResourceExtension(EXT_ALIAS, controller)]
+
+    def get_required_extensions(self):
+        return ["subnet_allocation", "external-net", "router"]
+
+    def get_extended_resources(self, version):
+        if version == "2.0":
+            return RESOURCE_ATTRIBUTE_MAP
+        else:
+            return {}

neutron/services/auto_allocate/db.py

@@ -0,0 +1,288 @@
+# Copyright 2015-2016 Hewlett Packard Enterprise Development Company, LP
+#
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from sqlalchemy import sql
+
+from oslo_db import exception as db_exc
+from oslo_log import log as logging
+
+from neutron._i18n import _, _LE
+from neutron.api.v2 import attributes
+from neutron.callbacks import events
+from neutron.callbacks import registry
+from neutron.callbacks import resources
+from neutron.common import exceptions as n_exc
+from neutron.db import common_db_mixin
+from neutron.db import db_base_plugin_v2
+from neutron.db import external_net_db
+from neutron.db import model_base
+from neutron.db import models_v2
+from neutron.extensions import l3
+from neutron import manager
+from neutron.plugins.common import constants
+from neutron.plugins.common import utils as p_utils
+from neutron.services.auto_allocate import exceptions
+from neutron.services.auto_allocate import models
+
+LOG = logging.getLogger(__name__)
+IS_DEFAULT = 'is_default'
+
+
+def _extend_external_network_default(self, net_res, net_db):
+    """Add is_default field to 'show' response."""
+    if net_db.external is not None:
+        net_res[IS_DEFAULT] = net_db.external.is_default
+    return net_res
+
+
+def _ensure_external_network_default_value_callback(
+    resource, event, trigger, context, request, network):
+    """Ensure the is_default db field matches the create/update request."""
+    is_default = request.get(IS_DEFAULT)
+    if event in (events.BEFORE_CREATE, events.BEFORE_UPDATE) and is_default:
+        # ensure there is only one default external network at any given time
+        obj = (context.session.query(external_net_db.ExternalNetwork).
+            filter_by(is_default=True)).first()
+        if obj and network['id'] != obj.network_id:
+            raise exceptions.DefaultExternalNetworkExists(
+                net_id=obj.network_id)
+
+    # Reflect the status of the is_default on the create/update request
+    obj = (context.session.query(external_net_db.ExternalNetwork).
+        filter_by(network_id=network['id']))
+    obj.update({IS_DEFAULT: is_default})
+
+
+class AutoAllocatedTopologyMixin(common_db_mixin.CommonDbMixin):
+
+    db_base_plugin_v2.NeutronDbPluginV2.register_dict_extend_funcs(
+        attributes.NETWORKS, [_extend_external_network_default])
+    registry.subscribe(_ensure_external_network_default_value_callback,
+        resources.EXTERNAL_NETWORK, events.BEFORE_CREATE)
+    registry.subscribe(_ensure_external_network_default_value_callback,
+        resources.EXTERNAL_NETWORK, events.AFTER_CREATE)
+    registry.subscribe(_ensure_external_network_default_value_callback,
+        resources.EXTERNAL_NETWORK, events.BEFORE_UPDATE)
+    # TODO(armax): if a tenant modifies auto allocated resources under
+    # the hood the behavior of the get_auto_allocated_topology API is
+    # undetermined. Consider adding callbacks to deal with the following
+    # situations:
+    # - insert subnet -> plug router interface
+    # - delete router -> remove the entire topology
+    # - update subnet -> prevent operation
+    # - update router gateway -> prevent operation
+    # - ...
+
+    def get_auto_allocated_topology(self, context, tenant_id, fields=None):
+        """Return tenant's network associated to auto-allocated topology.
+
+        The topology will be provisioned upon return, if network is missing.
+        """
+        tenant_id = self._validate(context, tenant_id)
+        # Check for an existent topology
+        network_id = self._get_auto_allocated_network(context, tenant_id)
+        if network_id:
+            return self._response(network_id, tenant_id, fields=fields)
+        # See if we indeed have an external network to connect to, otherwise
+        # we will fail fast
+        default_external_network = self._get_default_external_network(
+            context)
+
+        # If we reach this point, then we got some work to do!
+        subnets = self._provision_tenant_private_network(context, tenant_id)
+        network_id = subnets[0]['network_id']
+        router = self._provision_external_connectivity(
+            context, default_external_network, subnets, tenant_id)
+        network_id = self._save(
+            context, tenant_id, network_id, router['id'], subnets)
+        return self._response(network_id, tenant_id, fields=fields)
+
+    @property
+    def core_plugin(self):
+        if not getattr(self, '_core_plugin', None):
+            self._core_plugin = manager.NeutronManager.get_plugin()
+        return self._core_plugin
+
+    @property
+    def l3_plugin(self):
+        if not getattr(self, '_l3_plugin', None):
+            self._l3_plugin = manager.NeutronManager.get_service_plugins().get(
+                constants.L3_ROUTER_NAT)
+        return self._l3_plugin
+
+    def _validate(self, context, tenant_id):
+        """Validate and return the tenant to be associated to the topology."""
+        if tenant_id == 'None':
+            # NOTE(HenryG): the client might be sending us astray by
+            # passing no tenant; this is really meant to be the tenant
+            # issuing the request, therefore let's get it from the context
+            tenant_id = context.tenant_id
+
+        if not context.is_admin and tenant_id != context.tenant_id:
+            raise n_exc.NotAuthorized()
+
+        return tenant_id
+
+    def _get_auto_allocated_network(self, context, tenant_id):
+        """Get the auto allocated network for the tenant."""
+        with context.session.begin(subtransactions=True):
+            network = (context.session.query(models.AutoAllocatedTopology).
+                filter_by(tenant_id=tenant_id).first())
+
+        if network:
+            return network['network_id']
+
+    def _response(self, network_id, tenant_id, fields=None):
+        """Build response for auto-allocated network."""
+        res = {
+            'id': network_id,
+            'tenant_id': tenant_id
+        }
+        return self._fields(res, fields)
+
+    def _get_default_external_network(self, context):
+        """Get the default external network for the deployment."""
+        with context.session.begin(subtransactions=True):
+            default_external_networks = (context.session.query(
+                external_net_db.ExternalNetwork).
+                filter_by(is_default=sql.true()).
+                join(models_v2.Network).
+                join(model_base.StandardAttribute).
+                order_by(model_base.StandardAttribute.id).all())
+
+        if not default_external_networks:
+            LOG.error(_LE("Unable to find default external network "
+                          "for deployment, please create/assign one to "
+                          "allow auto-allocation to work correctly."))
+            raise exceptions.AutoAllocationFailure(
+                reason=_("No default router:external network"))
+        if len(default_external_networks) > 1:
+            LOG.error(_LE("Multiple external default networks detected. "
+                          "Network %s is true 'default'."),
+                      default_external_networks[0]['network_id'])
+        return default_external_networks[0]
+
+    def _get_supported_versions(self, context):
+        """Return the IP versions of default subnet pools available."""
+        default_subnet_pools = [
+            self.core_plugin.get_default_subnetpool(
+                context, ver) for ver in (4, 6)
+        ]
+        ip_versions = [
+            s['ip_version'] for s in default_subnet_pools if s
+        ]
+        if not ip_versions:
+            LOG.error(_LE("No default pools available"))
+            raise n_exc.NotFound()
+
+        return ip_versions
+
+    def _provision_tenant_private_network(self, context, tenant_id):
+        """Create a tenant private network/subnets."""
+        network = None
+        try:
+            network_args = {
+                'name': 'auto_allocated_network',
+                'admin_state_up': True,
+                'tenant_id': tenant_id,
+                'shared': False
+            }
+            network = p_utils.create_network(
+                self.core_plugin, context, {'network': network_args})
+            subnets = []
+            for ip_version in self._get_supported_versions(context):
+                subnet_args = {
+                    'name': 'auto_allocated_subnet_v%s' % ip_version,
+                    'network_id': network['id'],
+                    'tenant_id': tenant_id,
+                    'ip_version': ip_version,
+                }
+                subnets.append(p_utils.create_subnet(
+                    self.core_plugin, context, {'subnet': subnet_args}))
+            return subnets
+        except (ValueError, n_exc.BadRequest, n_exc.NotFound):
+            LOG.error(_LE("Unable to auto allocate topology for tenant "
+                          "%s due to missing requirements, e.g. default "
+                          "or shared subnetpools"), tenant_id)
+            if network:
+                self._cleanup(context, network['id'])
+            raise exceptions.AutoAllocationFailure(
+                reason=_("Unable to provide tenant private network"))
+
+    def _provision_external_connectivity(
+        self, context, default_external_network, subnets, tenant_id):
+        """Uplink tenant subnet(s) to external network."""
+        router_args = {
+            'name': 'auto_allocated_router',
+            l3.EXTERNAL_GW_INFO: default_external_network,
+            'tenant_id': tenant_id,
+            'admin_state_up': True
+        }
+        router = None
+        try:
+            router = self.l3_plugin.create_router(
+                context, {'router': router_args})
+            for subnet in subnets:
+                self.l3_plugin.add_router_interface(
+                    context, router['id'], {'subnet_id': subnet['id']})
+            return router
+        except n_exc.BadRequest:
+            LOG.error(_LE("Unable to auto allocate topology for tenant "
+                          "%s because of router errors."), tenant_id)
+            if router:
+                self._cleanup(context,
+                    network_id=subnets[0]['network_id'],
+                    router_id=router['id'], subnets=subnets)
+            raise exceptions.AutoAllocationFailure(
+                reason=_("Unable to provide external connectivity"))
+
+    def _save(self, context, tenant_id, network_id, router_id, subnets):
+        """Save auto-allocated topology, or revert in case of DB errors."""
+        try:
+            # NOTE(armax): saving the auto allocated topology in a
+            # separate transaction will keep the Neutron DB and the
+            # Neutron plugin backend in sync, thus allowing for a
+            # more bullet proof cleanup.
+            with context.session.begin(subtransactions=True):
+                context.session.add(
+                    models.AutoAllocatedTopology(
+                        tenant_id=tenant_id,
+                        network_id=network_id,
+                        router_id=router_id))
+        except db_exc.DBDuplicateEntry:
+            LOG.error(_LE("Multiple auto-allocated networks detected for "
+                          "tenant %(tenant)s. Attempting clean up for "
+                          "network %(network)s and router %(router)s"),
+                      {'tenant': tenant_id,
+                       'network': network_id,
+                       'router': router_id})
+            self._cleanup(
+                context, network_id=network_id,
+                router_id=router_id, subnets=subnets)
+            network_id = self._get_auto_allocated_network(
+                context, tenant_id)
+        return network_id
+
+    def _cleanup(self, context, network_id=None, router_id=None, subnets=None):
+        """Clean up auto allocated resources."""
+        if router_id:
+            for subnet in subnets or []:
+                self.l3_plugin.remove_router_interface(
+                    context, router_id, {'subnet_id': subnet['id']})
+            self.l3_plugin.delete_router(context, router_id)
+
+        if network_id:
+            self.core_plugin.delete_network(context, network_id)

neutron/services/auto_allocate/exceptions.py

@@ -0,0 +1,26 @@
+# Copyright 2015-2016 Hewlett Packard Enterprise Development Company, LP
+#
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from neutron._i18n import _
+from neutron.common import exceptions as n_exc
+
+
+class AutoAllocationFailure(n_exc.Conflict):
+    message = _("Deployment error: %(reason)s.")
+
+
+class DefaultExternalNetworkExists(n_exc.Conflict):
+    message = _("A default external network already exists: %(net_id)s.")

neutron/services/auto_allocate/models.py

@@ -0,0 +1,34 @@
+# Copyright (c) 2015-2016 Hewlett Packard Enterprise Development Company LP
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import sqlalchemy as sa
+
+from neutron.db import model_base
+
+
+class AutoAllocatedTopology(model_base.BASEV2):
+
+    __tablename__ = 'auto_allocated_topologies'
+
+    tenant_id = sa.Column(sa.String(255), primary_key=True)
+
+    network_id = sa.Column(sa.String(36),
+                           sa.ForeignKey('networks.id',
+                                         ondelete='CASCADE'),
+                           nullable=False)
+    router_id = sa.Column(sa.String(36),
+                          sa.ForeignKey('routers.id',
+                                        ondelete='SET NULL'),
+                          nullable=True)

neutron/services/auto_allocate/plugin.py

@@ -0,0 +1,37 @@
+# Copyright 2015-2016 Hewlett Packard Enterprise Development Company, LP
+#
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+from neutron.services.auto_allocate import db
+
+
+class Plugin(db.AutoAllocatedTopologyMixin):
+
+    _instance = None
+
+    supported_extension_aliases = ["auto-allocated-topology"]
+
+    @classmethod
+    def get_instance(cls):
+        if cls._instance is None:
+            cls._instance = cls()
+        return cls._instance
+
+    def get_plugin_description(self):
+        return "Auto Allocated Topology - aka get me a network."
+
+    def get_plugin_type(self):
+        return "auto-allocated-topology"

neutron/tests/etc/policy.json

@@ -45,6 +45,7 @@
     "get_network:queue_id": "rule:admin_only",
     "create_network:shared": "rule:admin_only",
     "create_network:router:external": "rule:admin_only",
+    "create_network:is_default": "rule:admin_only",
     "create_network:segments": "rule:admin_only",
     "create_network:provider:network_type": "rule:admin_only",
     "create_network:provider:physical_network": "rule:admin_only",
@@ -203,5 +204,6 @@
 
     "create_flavor_service_profile": "rule:admin_only",
     "delete_flavor_service_profile": "rule:admin_only",
-    "get_flavor_service_profile": "rule:regular_user"
+    "get_flavor_service_profile": "rule:regular_user",
+    "get_auto_allocated_topology": "rule:admin_or_owner"
 }