diff --git a/requirements.txt b/requirements.txt index 347864b1d..99ae6be7a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,6 +11,7 @@ pbr!=2.1.0,>=2.0.0 # Apache-2.0 pecan!=1.0.2,!=1.0.3,!=1.0.4,!=1.2,>=1.0.0 # BSD python-etcd>=0.4.3 # MIT License python-glanceclient>=2.5.0 # Apache-2.0 +python-neutronclient>=5.1.0 # Apache-2.0 python-novaclient>=7.1.0 # Apache-2.0 oslo.i18n>=2.1.0 # Apache-2.0 oslo.log>=3.22.0 # Apache-2.0 diff --git a/setup.cfg b/setup.cfg index 9521b0d71..cb2ca60f6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -69,6 +69,9 @@ zun.image.driver = glance = zun.image.glance.driver:GlanceDriver docker = zun.image.docker.driver:DockerDriver +zun.network.driver = + kuryr = zun.network.kuryr_network:KuryrNetwork + tempest.test_plugins = zun_tests = zun.tests.tempest.plugin:ZunTempestPlugin diff --git a/zun/common/clients.py b/zun/common/clients.py index 4c18c8de9..edf4d1301 100644 --- a/zun/common/clients.py +++ b/zun/common/clients.py @@ -13,6 +13,7 @@ # under the License. from glanceclient import client as glanceclient +from neutronclient.v2_0 import client as neutronclient from novaclient import client as novaclient from zun.common import exception @@ -28,6 +29,7 @@ class OpenStackClients(object): self._keystone = None self._glance = None self._nova = None + self._neutron = None def url_for(self, **kwargs): return self.keystone().session.get_endpoint(**kwargs) @@ -93,3 +95,15 @@ class OpenStackClients(object): self._nova = novaclient.Client(nova_api_version, session=session) return self._nova + + @exception.wrap_keystone_exception + def neutron(self): + if self._neutron: + return self._neutron + + session = self.keystone().session + endpoint_type = self._get_client_option('neutron', 'endpoint_type') + self._neutron = neutronclient.Client(session=session, + endpoint_type=endpoint_type) + + return self._neutron diff --git a/zun/conf/__init__.py b/zun/conf/__init__.py index 3cc9d9bcc..0ad038da2 100644 --- a/zun/conf/__init__.py +++ b/zun/conf/__init__.py @@ -21,6 +21,8 @@ from zun.conf import database from zun.conf import docker from zun.conf import glance_client from zun.conf import image_driver +from zun.conf import network +from zun.conf import neutron_client from zun.conf import nova_client from zun.conf import path from zun.conf import profiler @@ -45,3 +47,5 @@ services.register_opts(CONF) zun_client.register_opts(CONF) ssl.register_opts(CONF) profiler.register_opts(CONF) +neutron_client.register_opts(CONF) +network.register_opts(CONF) diff --git a/zun/conf/docker.py b/zun/conf/docker.py index 578777830..de4ffc903 100644 --- a/zun/conf/docker.py +++ b/zun/conf/docker.py @@ -20,7 +20,7 @@ docker_group = cfg.OptGroup(name='docker', docker_opts = [ cfg.StrOpt('docker_remote_api_version', - default='1.22', + default='1.23', help='Docker remote api version. Override it according to ' 'specific docker api version in your environment.'), cfg.IntOpt('default_timeout', diff --git a/zun/conf/network.py b/zun/conf/network.py new file mode 100644 index 000000000..2c936f04e --- /dev/null +++ b/zun/conf/network.py @@ -0,0 +1,34 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + + +network_group = cfg.OptGroup(name='network', + title='Options for the container network') + +network_opts = [ + cfg.StrOpt('driver', + default='kuryr', + help='Defines which driver to use for container network.'), +] + +ALL_OPTS = (network_opts) + + +def register_opts(conf): + conf.register_group(network_group) + conf.register_opts(ALL_OPTS, group=network_group) + + +def list_opts(): + return {network_group: ALL_OPTS} diff --git a/zun/conf/neutron_client.py b/zun/conf/neutron_client.py new file mode 100644 index 000000000..babb9ad31 --- /dev/null +++ b/zun/conf/neutron_client.py @@ -0,0 +1,51 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +from zun.common.i18n import _ + + +neutron_group = cfg.OptGroup(name='neutron_client', + title='Options for the Neutron client') + +common_security_opts = [ + cfg.StrOpt('ca_file', + help=_('Optional CA cert file to use in SSL connections.')), + cfg.StrOpt('cert_file', + help=_('Optional PEM-formatted certificate chain file.')), + cfg.StrOpt('key_file', + help=_('Optional PEM-formatted file that contains the ' + 'private key.')), + cfg.BoolOpt('insecure', + default=False, + help=_("If set, then the server's certificate will not " + "be verified."))] + +neutron_client_opts = [ + cfg.StrOpt('endpoint_type', + default='publicURL', + help=_( + 'Type of endpoint in Identity service catalog to use ' + 'for communication with the OpenStack service.'))] + + +ALL_OPTS = (neutron_client_opts + common_security_opts) + + +def register_opts(conf): + conf.register_group(neutron_group) + conf.register_opts(ALL_OPTS, group=neutron_group) + + +def list_opts(): + return {neutron_group: ALL_OPTS} diff --git a/zun/container/docker/driver.py b/zun/container/docker/driver.py index ff85f7c81..81931ddad 100644 --- a/zun/container/docker/driver.py +++ b/zun/container/docker/driver.py @@ -18,6 +18,7 @@ from docker import errors from oslo_log import log as logging from oslo_utils import timeutils +from zun.common import clients from zun.common import consts from zun.common import exception from zun.common.i18n import _ @@ -27,8 +28,10 @@ from zun.common.utils import check_container_id import zun.conf from zun.container.docker import utils as docker_utils from zun.container import driver +from zun.network import network as zun_network from zun import objects + CONF = zun.conf.CONF LOG = logging.getLogger(__name__) ATTACH_FLAG = "/attach/ws?logs=0&stream=1&stdin=1&stdout=1&stderr=1" @@ -457,17 +460,60 @@ class DockerDriver(driver.ContainerDriver): value = unicode(value) return value.encode('utf-8') - def create_sandbox(self, context, container, image='kubernetes/pause'): + def create_sandbox(self, context, container, image='kubernetes/pause', + networks=None): with docker_utils.docker_client() as docker: + network_api = zun_network.api(context=context, docker_api=docker) + if networks is None: + # Find an available neutron net and create docker network by + # wrapping the neutron net. + neutron_net = self._get_available_network(context) + network = self._get_or_create_docker_network( + context, network_api, neutron_net['id']) + networks = [network['Name']] + name = self.get_sandbox_name(container) - response = docker.create_container(image, name=name, - hostname=name[:63]) - sandbox_id = response['Id'] - docker.start(sandbox_id) - return sandbox_id + sandbox = docker.create_container(image, name=name, + hostname=name[:63]) + # Container connects to the bridge network by default so disconnect + # the container from it before connecting it to neutron network. + # This avoids potential conflict between these two networks. + network_api.disconnect_container_from_network(sandbox, 'bridge') + for network in networks: + network_api.connect_container_to_network(sandbox, network) + docker.start(sandbox['Id']) + return sandbox['Id'] + + def _get_available_network(self, context): + neutron = clients.OpenStackClients(context).neutron() + search_opts = {'tenant_id': context.project_id, 'shared': False} + nets = neutron.list_networks(**search_opts).get('networks', []) + if not nets: + raise exception.ZunException(_( + "There is no neutron network available")) + nets.sort(key=lambda x: x['created_at']) + return nets[0] + + def _get_or_create_docker_network(self, context, network_api, + neutron_net_id): + # Append project_id to the network name to avoid name collision + # across projects. + docker_net_name = neutron_net_id + '-' + context.project_id + docker_networks = network_api.list_networks(names=[docker_net_name]) + if not docker_networks: + network_api.create_network(neutron_net_id=neutron_net_id, + name=docker_net_name) + docker_networks = network_api.list_networks( + names=[docker_net_name]) + + return docker_networks[0] def delete_sandbox(self, context, sandbox_id): with docker_utils.docker_client() as docker: + network_api = zun_network.api(context=context, docker_api=docker) + sandbox = docker.inspect_container(sandbox_id) + for network in sandbox["NetworkSettings"]["Networks"]: + network_api.disconnect_container_from_network(sandbox, network) try: docker.remove_container(sandbox_id, force=True) except errors.APIError as api_error: @@ -501,15 +547,15 @@ class DockerDriver(driver.ContainerDriver): def get_addresses(self, context, container): sandbox_id = self.get_sandbox_id(container) with docker_utils.docker_client() as docker: + addresses = {} response = docker.inspect_container(sandbox_id) - addr = response["NetworkSettings"]["IPAddress"] - addresses = { - 'default': [ - { - 'addr': addr, - }, - ], - } + networks = response["NetworkSettings"]["Networks"] + for name, network in networks.items(): + addresses[name] = [ + {'addr': network["IPAddress"]}, + {'addr': network["GlobalIPv6Address"]}, + ] + return addresses def get_container_numbers(self): diff --git a/zun/network/__init__.py b/zun/network/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/zun/network/kuryr_network.py b/zun/network/kuryr_network.py new file mode 100644 index 000000000..26614164d --- /dev/null +++ b/zun/network/kuryr_network.py @@ -0,0 +1,162 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import ipaddress +import six + +from oslo_log import log as logging + +from zun.common import clients +from zun.common import exception +from zun.common.i18n import _ +import zun.conf +from zun.network import network + + +LOG = logging.getLogger(__name__) +CONF = zun.conf.CONF + + +class KuryrNetwork(network.Network): + + def init(self, context, docker_api): + self.docker = docker_api + self.neutron = clients.OpenStackClients(context).neutron() + + def create_network(self, name, neutron_net_id): + """Create a docker network with Kuryr driver. + + The docker network to be created will be based on the specified + neutron net. It is assumed that the neutron net will have one + or two subnets. If there are two subnets, it must be a ipv4 + subnet and a ipv6 subnet and containers created from this network + will have both ipv4 and ipv6 addresses. + + What this method does is finding the subnets under the specified + neutron net, retrieving the cidr, gateway, subnetpool of each + subnet, and compile the list of parameters for docker.create_network. + """ + # find a v4 and/or v6 subnet of the network + subnets = self.neutron.list_subnets(network_id=neutron_net_id) + subnets = subnets.get('subnets', []) + v4_subnet = self._get_subnet(subnets, ip_version=4) + v6_subnet = self._get_subnet(subnets, ip_version=6) + if not v4_subnet and not v6_subnet: + raise exception.ZunException(_( + "The Neutron network %s has no subnet") % neutron_net_id) + + ipam_options = { + "Driver": "kuryr", + "Options": {}, + "Config": [] + } + if v4_subnet: + ipam_options["Options"]['neutron.pool.uuid'] = ( + v4_subnet.get('subnetpool_id')) + ipam_options["Config"].append({ + "Subnet": v4_subnet['cidr'], + "Gateway": v4_subnet['gateway_ip'] + }) + if v6_subnet: + ipam_options["Options"]['neutron.pool.v6.uuid'] = ( + v6_subnet.get('subnetpool_id')) + ipam_options["Config"].append({ + "Subnet": v6_subnet['cidr'], + "Gateway": v6_subnet['gateway_ip'] + }) + + options = { + 'neutron.net.uuid': neutron_net_id + } + if v4_subnet: + options['neutron.pool.uuid'] = v4_subnet.get('subnetpool_id') + if v6_subnet: + options['neutron.pool.v6.uuid'] = v6_subnet.get('subnetpool_id') + + docker_network = self.docker.create_network( + name=name, + driver='kuryr', + enable_ipv6=True if v6_subnet else False, + options=options, + ipam=ipam_options) + + return docker_network + + def _get_subnet(self, subnets, ip_version): + subnets = [s for s in subnets if s['ip_version'] == ip_version] + if len(subnets) == 0: + return None + elif len(subnets) == 1: + return subnets[0] + else: + raise exception.ZunException(_( + "Multiple Neutron subnets exist with ip version %s") % + ip_version) + + def delete_network(self, network_name): + self.docker.delete_network(network_name) + + def inspect_network(self, network_name): + return self.docker.inspect_network(network_name) + + def list_networks(self, **kwargs): + return self.docker.networks(**kwargs) + + def connect_container_to_network(self, container, network_name): + """Connect container to the network + + This method will create a neutron port, retrieve the ip address(es) + of the port, and pass them to docker.connect_container_to_network. + """ + network = self.inspect_network(network_name) + neutron_net_id = network['Options']['neutron.net.uuid'] + neutron_port = self.neutron.create_port({'port': { + 'network_id': neutron_net_id, + }}) + + ipv4_address = None + ipv6_address = None + for fixed_ip in neutron_port['port']['fixed_ips']: + ip_address = fixed_ip['ip_address'] + ip = ipaddress.ip_address(six.text_type(ip_address)) + if ip.version == 4: + ipv4_address = ip_address + else: + ipv6_address = ip_address + + kwargs = {} + if ipv4_address: + kwargs['ipv4_address'] = ipv4_address + if ipv6_address: + kwargs['ipv6_address'] = ipv6_address + self.docker.connect_container_to_network( + container['Id'], network_name, **kwargs) + + def disconnect_container_from_network(self, container, network_name): + container_id = container['Id'] + neutron_ports = None + # TODO(hongbin): Use objects instead of an ad hoc dict. + if "NetworkSettings" in container: + network = container["NetworkSettings"]["Networks"][network_name] + endpoint_id = network["EndpointID"] + # Kuryr set the port's device_id as endpoint_id so we leverge it + neutron_ports = self.neutron.list_ports(device_id=endpoint_id) + neutron_ports = neutron_ports.get('ports', []) + if not neutron_ports: + LOG.warning("Cannot find the neutron port that bind container " + "%s to network %s", container_id, network_name) + + self.docker.disconnect_container_from_network(container_id, + network_name) + if neutron_ports: + port_id = neutron_ports[0]['id'] + self.neutron.delete_port(port_id) diff --git a/zun/network/network.py b/zun/network/network.py new file mode 100644 index 000000000..583ab8412 --- /dev/null +++ b/zun/network/network.py @@ -0,0 +1,59 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import abc +import six + +from stevedore import driver + +import zun.conf + + +CONF = zun.conf.CONF + + +def api(*args, **kwargs): + network_driver = CONF.network.driver + network_api = driver.DriverManager( + "zun.network.driver", + network_driver, + invoke_on_load=True).driver + + network_api.init(*args, **kwargs) + return network_api + + +@six.add_metaclass(abc.ABCMeta) +class Network(object): + """The base class that all Network classes should inherit from.""" + + def init(self, context, *args, **kwargs): + raise NotImplementedError() + + def create_network(self, *args, **kwargs): + raise NotImplementedError() + + def delete_network(self, network_name, **kwargs): + raise NotImplementedError() + + def inspect_network(self, network_name, **kwargs): + raise NotImplementedError() + + def list_networks(self, **kwargs): + raise NotImplementedError() + + def connect_container_to_network(self, container, network_name, **kwargs): + raise NotImplementedError() + + def disconnect_container_from_network(self, container, network_name, + **kwargs): + raise NotImplementedError() diff --git a/zun/tests/unit/container/docker/test_docker_driver.py b/zun/tests/unit/container/docker/test_docker_driver.py index 9bc8af666..0d1185966 100644 --- a/zun/tests/unit/container/docker/test_docker_driver.py +++ b/zun/tests/unit/container/docker/test_docker_driver.py @@ -337,32 +337,35 @@ class TestDockerDriver(base.DriverTestCase): self.mock_docker.resize.assert_called_once_with( mock_container.container_id, 100, 100) + @mock.patch('zun.network.kuryr_network.KuryrNetwork' + '.connect_container_to_network') @mock.patch('zun.container.docker.driver.DockerDriver.get_sandbox_name') - def test_create_sandbox(self, mock_get_sandbox_name): + def test_create_sandbox(self, mock_get_sandbox_name, mock_connect): sandbox_name = 'my_test_sandbox' mock_get_sandbox_name.return_value = sandbox_name self.mock_docker.create_container = mock.Mock( return_value={'Id': 'val1', 'key1': 'val2'}) - self.mock_docker.start() mock_container = mock.MagicMock() - result_sandbox_id = self.driver.create_sandbox(self.context, - mock_container, - 'kubernetes/pause') + with mock.patch.object(self.driver, '_get_available_network'): + result_sandbox_id = self.driver.create_sandbox( + self.context, mock_container, 'kubernetes/pause') self.mock_docker.create_container.assert_called_once_with( 'kubernetes/pause', name=sandbox_name, hostname=sandbox_name) self.assertEqual(result_sandbox_id, 'val1') + @mock.patch('zun.network.kuryr_network.KuryrNetwork' + '.connect_container_to_network') @mock.patch('zun.container.docker.driver.DockerDriver.get_sandbox_name') - def test_create_sandbox_with_long_name(self, mock_get_sandbox_name): + def test_create_sandbox_with_long_name(self, mock_get_sandbox_name, + mock_connect): sandbox_name = 'x' * 100 mock_get_sandbox_name.return_value = sandbox_name self.mock_docker.create_container = mock.Mock( return_value={'Id': 'val1', 'key1': 'val2'}) - self.mock_docker.start() mock_container = mock.MagicMock() - result_sandbox_id = self.driver.create_sandbox(self.context, - mock_container, - 'kubernetes/pause') + with mock.patch.object(self.driver, '_get_available_network'): + result_sandbox_id = self.driver.create_sandbox( + self.context, mock_container, 'kubernetes/pause') self.mock_docker.create_container.assert_called_once_with( 'kubernetes/pause', name=sandbox_name, hostname=sandbox_name[:63]) self.assertEqual(result_sandbox_id, 'val1') @@ -417,14 +420,18 @@ class TestDockerDriver(base.DriverTestCase): def test_get_addresses(self, mock_get_sandbox_id): mock_get_sandbox_id.return_value = 'test_sandbox_id' self.mock_docker.inspect_container = mock.Mock( - return_value={'NetworkSettings': {'IPAddress': '127.0.0.1'}}) + return_value={'NetworkSettings': {'Networks': {'default': { + 'IPAddress': '127.0.0.1', + 'GlobalIPv6Address': 'fe80::4', + }}}}) mock_container = mock.MagicMock() result_addresses = self.driver.get_addresses(self.context, mock_container) self.mock_docker.inspect_container.assert_called_once_with( 'test_sandbox_id') - self.assertEqual(result_addresses, - {'default': [{'addr': '127.0.0.1', }, ], }) + expected_addresses = {'default': [ + {'addr': '127.0.0.1'}, {'addr': 'fe80::4'}]} + self.assertEqual(expected_addresses, result_addresses) def test_execute_resize(self): self.mock_docker.exec_resize = mock.Mock()