From c5a979864e01eee1b553d6e1a8ccb196275282e1 Mon Sep 17 00:00:00 2001 From: Hongbin Lu Date: Wed, 5 Apr 2017 00:42:04 +0000 Subject: [PATCH] Implement kuryr integration After this patch, all containers created by native docker driver will connect to neutron. Right now, users don't have a choice of networks the container will be created from (which is a future work). Instead, Zun will select a neutron network for each container by looking up all the networks under user's tenant. This is the first step for enabling Kuryr integration. The following work needs to be done later: * Expose network feature via REST API. Allow users to create, inspect, delete container network, and connect/disconnect container from/to a network. * Add support for associate/dissociate floating IPs to containers. * Add support for tuning security groups of the containers. Implements: blueprint kuryr-integration Change-Id: I2701eb9a82a74aedf00c1a2af29850d4bd0e8f7a --- requirements.txt | 1 + setup.cfg | 3 + zun/common/clients.py | 14 ++ zun/conf/__init__.py | 4 + zun/conf/docker.py | 2 +- zun/conf/network.py | 34 ++++ zun/conf/neutron_client.py | 51 ++++++ zun/container/docker/driver.py | 74 ++++++-- zun/network/__init__.py | 0 zun/network/kuryr_network.py | 162 ++++++++++++++++++ zun/network/network.py | 59 +++++++ .../container/docker/test_docker_driver.py | 33 ++-- 12 files changed, 409 insertions(+), 28 deletions(-) create mode 100644 zun/conf/network.py create mode 100644 zun/conf/neutron_client.py create mode 100644 zun/network/__init__.py create mode 100644 zun/network/kuryr_network.py create mode 100644 zun/network/network.py diff --git a/requirements.txt b/requirements.txt index 347864b1d..99ae6be7a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,6 +11,7 @@ pbr!=2.1.0,>=2.0.0 # Apache-2.0 pecan!=1.0.2,!=1.0.3,!=1.0.4,!=1.2,>=1.0.0 # BSD python-etcd>=0.4.3 # MIT License python-glanceclient>=2.5.0 # Apache-2.0 +python-neutronclient>=5.1.0 # Apache-2.0 python-novaclient>=7.1.0 # Apache-2.0 oslo.i18n>=2.1.0 # Apache-2.0 oslo.log>=3.22.0 # Apache-2.0 diff --git a/setup.cfg b/setup.cfg index 9521b0d71..cb2ca60f6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -69,6 +69,9 @@ zun.image.driver = glance = zun.image.glance.driver:GlanceDriver docker = zun.image.docker.driver:DockerDriver +zun.network.driver = + kuryr = zun.network.kuryr_network:KuryrNetwork + tempest.test_plugins = zun_tests = zun.tests.tempest.plugin:ZunTempestPlugin diff --git a/zun/common/clients.py b/zun/common/clients.py index 50e82cc2e..d9bba0fd5 100644 --- a/zun/common/clients.py +++ b/zun/common/clients.py @@ -13,6 +13,7 @@ # under the License. from glanceclient import client as glanceclient +from neutronclient.v2_0 import client as neutronclient from novaclient import client as novaclient from oslo_log import log as logging @@ -31,6 +32,7 @@ class OpenStackClients(object): self._keystone = None self._glance = None self._nova = None + self._neutron = None def url_for(self, **kwargs): return self.keystone().session.get_endpoint(**kwargs) @@ -96,3 +98,15 @@ class OpenStackClients(object): self._nova = novaclient.Client(nova_api_version, session=session) return self._nova + + @exception.wrap_keystone_exception + def neutron(self): + if self._neutron: + return self._neutron + + session = self.keystone().session + endpoint_type = self._get_client_option('neutron', 'endpoint_type') + self._neutron = neutronclient.Client(session=session, + endpoint_type=endpoint_type) + + return self._neutron diff --git a/zun/conf/__init__.py b/zun/conf/__init__.py index 3cc9d9bcc..0ad038da2 100644 --- a/zun/conf/__init__.py +++ b/zun/conf/__init__.py @@ -21,6 +21,8 @@ from zun.conf import database from zun.conf import docker from zun.conf import glance_client from zun.conf import image_driver +from zun.conf import network +from zun.conf import neutron_client from zun.conf import nova_client from zun.conf import path from zun.conf import profiler @@ -45,3 +47,5 @@ services.register_opts(CONF) zun_client.register_opts(CONF) ssl.register_opts(CONF) profiler.register_opts(CONF) +neutron_client.register_opts(CONF) +network.register_opts(CONF) diff --git a/zun/conf/docker.py b/zun/conf/docker.py index 578777830..de4ffc903 100644 --- a/zun/conf/docker.py +++ b/zun/conf/docker.py @@ -20,7 +20,7 @@ docker_group = cfg.OptGroup(name='docker', docker_opts = [ cfg.StrOpt('docker_remote_api_version', - default='1.22', + default='1.23', help='Docker remote api version. Override it according to ' 'specific docker api version in your environment.'), cfg.IntOpt('default_timeout', diff --git a/zun/conf/network.py b/zun/conf/network.py new file mode 100644 index 000000000..2c936f04e --- /dev/null +++ b/zun/conf/network.py @@ -0,0 +1,34 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + + +network_group = cfg.OptGroup(name='network', + title='Options for the container network') + +network_opts = [ + cfg.StrOpt('driver', + default='kuryr', + help='Defines which driver to use for container network.'), +] + +ALL_OPTS = (network_opts) + + +def register_opts(conf): + conf.register_group(network_group) + conf.register_opts(ALL_OPTS, group=network_group) + + +def list_opts(): + return {network_group: ALL_OPTS} diff --git a/zun/conf/neutron_client.py b/zun/conf/neutron_client.py new file mode 100644 index 000000000..babb9ad31 --- /dev/null +++ b/zun/conf/neutron_client.py @@ -0,0 +1,51 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +from zun.common.i18n import _ + + +neutron_group = cfg.OptGroup(name='neutron_client', + title='Options for the Neutron client') + +common_security_opts = [ + cfg.StrOpt('ca_file', + help=_('Optional CA cert file to use in SSL connections.')), + cfg.StrOpt('cert_file', + help=_('Optional PEM-formatted certificate chain file.')), + cfg.StrOpt('key_file', + help=_('Optional PEM-formatted file that contains the ' + 'private key.')), + cfg.BoolOpt('insecure', + default=False, + help=_("If set, then the server's certificate will not " + "be verified."))] + +neutron_client_opts = [ + cfg.StrOpt('endpoint_type', + default='publicURL', + help=_( + 'Type of endpoint in Identity service catalog to use ' + 'for communication with the OpenStack service.'))] + + +ALL_OPTS = (neutron_client_opts + common_security_opts) + + +def register_opts(conf): + conf.register_group(neutron_group) + conf.register_opts(ALL_OPTS, group=neutron_group) + + +def list_opts(): + return {neutron_group: ALL_OPTS} diff --git a/zun/container/docker/driver.py b/zun/container/docker/driver.py index 8ca6c4d4a..5419ab623 100644 --- a/zun/container/docker/driver.py +++ b/zun/container/docker/driver.py @@ -18,6 +18,7 @@ from docker import errors from oslo_log import log as logging from oslo_utils import timeutils +from zun.common import clients from zun.common import consts from zun.common import exception from zun.common.i18n import _ @@ -27,8 +28,10 @@ from zun.common.utils import check_container_id import zun.conf from zun.container.docker import utils as docker_utils from zun.container import driver +from zun.network import network as zun_network from zun import objects + CONF = zun.conf.CONF LOG = logging.getLogger(__name__) ATTACH_FLAG = "/attach/ws?logs=0&stream=1&stdin=1&stdout=1&stderr=1" @@ -470,17 +473,60 @@ class DockerDriver(driver.ContainerDriver): value = unicode(value) return value.encode('utf-8') - def create_sandbox(self, context, container, image='kubernetes/pause'): + def create_sandbox(self, context, container, image='kubernetes/pause', + networks=None): with docker_utils.docker_client() as docker: + network_api = zun_network.api(context=context, docker_api=docker) + if networks is None: + # Find an available neutron net and create docker network by + # wrapping the neutron net. + neutron_net = self._get_available_network(context) + network = self._get_or_create_docker_network( + context, network_api, neutron_net['id']) + networks = [network['Name']] + name = self.get_sandbox_name(container) - response = docker.create_container(image, name=name, - hostname=name[:63]) - sandbox_id = response['Id'] - docker.start(sandbox_id) - return sandbox_id + sandbox = docker.create_container(image, name=name, + hostname=name[:63]) + # Container connects to the bridge network by default so disconnect + # the container from it before connecting it to neutron network. + # This avoids potential conflict between these two networks. + network_api.disconnect_container_from_network(sandbox, 'bridge') + for network in networks: + network_api.connect_container_to_network(sandbox, network) + docker.start(sandbox['Id']) + return sandbox['Id'] + + def _get_available_network(self, context): + neutron = clients.OpenStackClients(context).neutron() + search_opts = {'tenant_id': context.project_id, 'shared': False} + nets = neutron.list_networks(**search_opts).get('networks', []) + if not nets: + raise exception.ZunException(_( + "There is no neutron network available")) + nets.sort(key=lambda x: x['created_at']) + return nets[0] + + def _get_or_create_docker_network(self, context, network_api, + neutron_net_id): + # Append project_id to the network name to avoid name collision + # across projects. + docker_net_name = neutron_net_id + '-' + context.project_id + docker_networks = network_api.list_networks(names=[docker_net_name]) + if not docker_networks: + network_api.create_network(neutron_net_id=neutron_net_id, + name=docker_net_name) + docker_networks = network_api.list_networks( + names=[docker_net_name]) + + return docker_networks[0] def delete_sandbox(self, context, sandbox_id): with docker_utils.docker_client() as docker: + network_api = zun_network.api(context=context, docker_api=docker) + sandbox = docker.inspect_container(sandbox_id) + for network in sandbox["NetworkSettings"]["Networks"]: + network_api.disconnect_container_from_network(sandbox, network) try: docker.remove_container(sandbox_id, force=True) except errors.APIError as api_error: @@ -514,15 +560,15 @@ class DockerDriver(driver.ContainerDriver): def get_addresses(self, context, container): sandbox_id = self.get_sandbox_id(container) with docker_utils.docker_client() as docker: + addresses = {} response = docker.inspect_container(sandbox_id) - addr = response["NetworkSettings"]["IPAddress"] - addresses = { - 'default': [ - { - 'addr': addr, - }, - ], - } + networks = response["NetworkSettings"]["Networks"] + for name, network in networks.items(): + addresses[name] = [ + {'addr': network["IPAddress"]}, + {'addr': network["GlobalIPv6Address"]}, + ] + return addresses diff --git a/zun/network/__init__.py b/zun/network/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/zun/network/kuryr_network.py b/zun/network/kuryr_network.py new file mode 100644 index 000000000..26614164d --- /dev/null +++ b/zun/network/kuryr_network.py @@ -0,0 +1,162 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import ipaddress +import six + +from oslo_log import log as logging + +from zun.common import clients +from zun.common import exception +from zun.common.i18n import _ +import zun.conf +from zun.network import network + + +LOG = logging.getLogger(__name__) +CONF = zun.conf.CONF + + +class KuryrNetwork(network.Network): + + def init(self, context, docker_api): + self.docker = docker_api + self.neutron = clients.OpenStackClients(context).neutron() + + def create_network(self, name, neutron_net_id): + """Create a docker network with Kuryr driver. + + The docker network to be created will be based on the specified + neutron net. It is assumed that the neutron net will have one + or two subnets. If there are two subnets, it must be a ipv4 + subnet and a ipv6 subnet and containers created from this network + will have both ipv4 and ipv6 addresses. + + What this method does is finding the subnets under the specified + neutron net, retrieving the cidr, gateway, subnetpool of each + subnet, and compile the list of parameters for docker.create_network. + """ + # find a v4 and/or v6 subnet of the network + subnets = self.neutron.list_subnets(network_id=neutron_net_id) + subnets = subnets.get('subnets', []) + v4_subnet = self._get_subnet(subnets, ip_version=4) + v6_subnet = self._get_subnet(subnets, ip_version=6) + if not v4_subnet and not v6_subnet: + raise exception.ZunException(_( + "The Neutron network %s has no subnet") % neutron_net_id) + + ipam_options = { + "Driver": "kuryr", + "Options": {}, + "Config": [] + } + if v4_subnet: + ipam_options["Options"]['neutron.pool.uuid'] = ( + v4_subnet.get('subnetpool_id')) + ipam_options["Config"].append({ + "Subnet": v4_subnet['cidr'], + "Gateway": v4_subnet['gateway_ip'] + }) + if v6_subnet: + ipam_options["Options"]['neutron.pool.v6.uuid'] = ( + v6_subnet.get('subnetpool_id')) + ipam_options["Config"].append({ + "Subnet": v6_subnet['cidr'], + "Gateway": v6_subnet['gateway_ip'] + }) + + options = { + 'neutron.net.uuid': neutron_net_id + } + if v4_subnet: + options['neutron.pool.uuid'] = v4_subnet.get('subnetpool_id') + if v6_subnet: + options['neutron.pool.v6.uuid'] = v6_subnet.get('subnetpool_id') + + docker_network = self.docker.create_network( + name=name, + driver='kuryr', + enable_ipv6=True if v6_subnet else False, + options=options, + ipam=ipam_options) + + return docker_network + + def _get_subnet(self, subnets, ip_version): + subnets = [s for s in subnets if s['ip_version'] == ip_version] + if len(subnets) == 0: + return None + elif len(subnets) == 1: + return subnets[0] + else: + raise exception.ZunException(_( + "Multiple Neutron subnets exist with ip version %s") % + ip_version) + + def delete_network(self, network_name): + self.docker.delete_network(network_name) + + def inspect_network(self, network_name): + return self.docker.inspect_network(network_name) + + def list_networks(self, **kwargs): + return self.docker.networks(**kwargs) + + def connect_container_to_network(self, container, network_name): + """Connect container to the network + + This method will create a neutron port, retrieve the ip address(es) + of the port, and pass them to docker.connect_container_to_network. + """ + network = self.inspect_network(network_name) + neutron_net_id = network['Options']['neutron.net.uuid'] + neutron_port = self.neutron.create_port({'port': { + 'network_id': neutron_net_id, + }}) + + ipv4_address = None + ipv6_address = None + for fixed_ip in neutron_port['port']['fixed_ips']: + ip_address = fixed_ip['ip_address'] + ip = ipaddress.ip_address(six.text_type(ip_address)) + if ip.version == 4: + ipv4_address = ip_address + else: + ipv6_address = ip_address + + kwargs = {} + if ipv4_address: + kwargs['ipv4_address'] = ipv4_address + if ipv6_address: + kwargs['ipv6_address'] = ipv6_address + self.docker.connect_container_to_network( + container['Id'], network_name, **kwargs) + + def disconnect_container_from_network(self, container, network_name): + container_id = container['Id'] + neutron_ports = None + # TODO(hongbin): Use objects instead of an ad hoc dict. + if "NetworkSettings" in container: + network = container["NetworkSettings"]["Networks"][network_name] + endpoint_id = network["EndpointID"] + # Kuryr set the port's device_id as endpoint_id so we leverge it + neutron_ports = self.neutron.list_ports(device_id=endpoint_id) + neutron_ports = neutron_ports.get('ports', []) + if not neutron_ports: + LOG.warning("Cannot find the neutron port that bind container " + "%s to network %s", container_id, network_name) + + self.docker.disconnect_container_from_network(container_id, + network_name) + if neutron_ports: + port_id = neutron_ports[0]['id'] + self.neutron.delete_port(port_id) diff --git a/zun/network/network.py b/zun/network/network.py new file mode 100644 index 000000000..583ab8412 --- /dev/null +++ b/zun/network/network.py @@ -0,0 +1,59 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import abc +import six + +from stevedore import driver + +import zun.conf + + +CONF = zun.conf.CONF + + +def api(*args, **kwargs): + network_driver = CONF.network.driver + network_api = driver.DriverManager( + "zun.network.driver", + network_driver, + invoke_on_load=True).driver + + network_api.init(*args, **kwargs) + return network_api + + +@six.add_metaclass(abc.ABCMeta) +class Network(object): + """The base class that all Network classes should inherit from.""" + + def init(self, context, *args, **kwargs): + raise NotImplementedError() + + def create_network(self, *args, **kwargs): + raise NotImplementedError() + + def delete_network(self, network_name, **kwargs): + raise NotImplementedError() + + def inspect_network(self, network_name, **kwargs): + raise NotImplementedError() + + def list_networks(self, **kwargs): + raise NotImplementedError() + + def connect_container_to_network(self, container, network_name, **kwargs): + raise NotImplementedError() + + def disconnect_container_from_network(self, container, network_name, + **kwargs): + raise NotImplementedError() diff --git a/zun/tests/unit/container/docker/test_docker_driver.py b/zun/tests/unit/container/docker/test_docker_driver.py index a1d577135..9cb6c4d20 100644 --- a/zun/tests/unit/container/docker/test_docker_driver.py +++ b/zun/tests/unit/container/docker/test_docker_driver.py @@ -335,32 +335,35 @@ class TestDockerDriver(base.DriverTestCase): self.mock_docker.resize.assert_called_once_with( mock_container.container_id, 100, 100) + @mock.patch('zun.network.kuryr_network.KuryrNetwork' + '.connect_container_to_network') @mock.patch('zun.container.docker.driver.DockerDriver.get_sandbox_name') - def test_create_sandbox(self, mock_get_sandbox_name): + def test_create_sandbox(self, mock_get_sandbox_name, mock_connect): sandbox_name = 'my_test_sandbox' mock_get_sandbox_name.return_value = sandbox_name self.mock_docker.create_container = mock.Mock( return_value={'Id': 'val1', 'key1': 'val2'}) - self.mock_docker.start() mock_container = mock.MagicMock() - result_sandbox_id = self.driver.create_sandbox(self.context, - mock_container, - 'kubernetes/pause') + with mock.patch.object(self.driver, '_get_available_network'): + result_sandbox_id = self.driver.create_sandbox( + self.context, mock_container, 'kubernetes/pause') self.mock_docker.create_container.assert_called_once_with( 'kubernetes/pause', name=sandbox_name, hostname=sandbox_name) self.assertEqual(result_sandbox_id, 'val1') + @mock.patch('zun.network.kuryr_network.KuryrNetwork' + '.connect_container_to_network') @mock.patch('zun.container.docker.driver.DockerDriver.get_sandbox_name') - def test_create_sandbox_with_long_name(self, mock_get_sandbox_name): + def test_create_sandbox_with_long_name(self, mock_get_sandbox_name, + mock_connect): sandbox_name = 'x' * 100 mock_get_sandbox_name.return_value = sandbox_name self.mock_docker.create_container = mock.Mock( return_value={'Id': 'val1', 'key1': 'val2'}) - self.mock_docker.start() mock_container = mock.MagicMock() - result_sandbox_id = self.driver.create_sandbox(self.context, - mock_container, - 'kubernetes/pause') + with mock.patch.object(self.driver, '_get_available_network'): + result_sandbox_id = self.driver.create_sandbox( + self.context, mock_container, 'kubernetes/pause') self.mock_docker.create_container.assert_called_once_with( 'kubernetes/pause', name=sandbox_name, hostname=sandbox_name[:63]) self.assertEqual(result_sandbox_id, 'val1') @@ -415,14 +418,18 @@ class TestDockerDriver(base.DriverTestCase): def test_get_addresses(self, mock_get_sandbox_id): mock_get_sandbox_id.return_value = 'test_sandbox_id' self.mock_docker.inspect_container = mock.Mock( - return_value={'NetworkSettings': {'IPAddress': '127.0.0.1'}}) + return_value={'NetworkSettings': {'Networks': {'default': { + 'IPAddress': '127.0.0.1', + 'GlobalIPv6Address': 'fe80::4', + }}}}) mock_container = mock.MagicMock() result_addresses = self.driver.get_addresses(self.context, mock_container) self.mock_docker.inspect_container.assert_called_once_with( 'test_sandbox_id') - self.assertEqual(result_addresses, - {'default': [{'addr': '127.0.0.1', }, ], }) + expected_addresses = {'default': [ + {'addr': '127.0.0.1'}, {'addr': 'fe80::4'}]} + self.assertEqual(expected_addresses, result_addresses) def test_execute_resize(self): self.mock_docker.exec_resize = mock.Mock()