openstack/zun
Code Issues Proposed changes

Merge "Support to provide 'image_driver' during container create"

Browse Source
This commit is contained in:
Jenkins 2017-01-30 13:56:57 +00:00 committed by Gerrit Code Review
commit 3b6070dc3f
23 changed files with 213 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
setup.cfg
View File

@ -64,5 +64,9 @@ zun.scheduler.driver =
chance_scheduler = zun.scheduler.chance_scheduler:ChanceScheduler
fake_scheduler = zun.tests.unit.scheduler.fake_scheduler:FakeScheduler
zun.image.driver =
glance = zun.image.glance.driver:GlanceDriver
docker = zun.image.docker.driver:DockerDriver
tempest.test_plugins =
zun_tests = zun.tests.tempest.plugin:ZunTempestPlugin

8
zun/api/controllers/v1/containers.py
View File

@ -202,13 +202,19 @@ class ContainersController(rest.RestController):
'"false", True, False, "True" and "False"')
raise exception.InvalidValue(msg)
# Valiadtion accepts 'None' so need to convert it to None
if container_dict.get('image_driver'):
container_dict['image_driver'] = api_utils.string_or_none(
container_dict.get('image_driver'))
# NOTE(mkrai): Intent here is to check the existence of image
# before proceeding to create container. If image is not found,
# container create will fail with 400 status.
images = compute_api.image_search(context, container_dict['image'],
container_dict.get('image_driver'),
True)
if not images:
raise exception.ImageNotFound(container_dict['image'])
raise exception.ImageNotFound(image=container_dict['image'])
container_dict['project_id'] = context.project_id
container_dict['user_id'] = context.user_id
name = container_dict.get('name') or \

8
zun/api/controllers/v1/images.py
View File

@ -117,7 +117,8 @@ class ImagesController(rest.RestController):
@pecan.expose('json')
@exception.wrap_pecan_controller_exception
def search(self, image, exact_match=False):
@validation.validate_query_param(pecan.request, schema.query_param_search)
def search(self, image, image_driver=None, exact_match=False):
context = pecan.request.context
policy.enforce(context, "image:search",
action="image:search")
@ -129,5 +130,10 @@ class ImagesController(rest.RestController):
msg = _("Valid exact_match values are true,"
" false, 0, 1, yes and no")
raise exception.InvalidValue(msg)
# Valiadtion accepts 'None' so need to convert it to None
if image_driver:
image_driver = api_utils.string_or_none(image_driver)
return pecan.request.compute_api.image_search(context, image,
image_driver,
exact_match)

1
zun/api/controllers/v1/schemas/containers.py
View File

@ -27,6 +27,7 @@ _container_properties = {
'restart_policy': parameter_types.restart_policy,
'tty': parameter_types.boolean,
'stdin_open': parameter_types.boolean,
'image_driver': parameter_types.image_driver
}
container_create = {

9
zun/api/controllers/v1/schemas/images.py
View File

@ -25,3 +25,12 @@ image_create = {
'required': ['repo'],
'additionalProperties': False
}
query_param_search = {
'type': 'object',
'properties': {
'image_driver': parameter_types.image_driver,
'exact_match': parameter_types.boolean
},
'additionalProperties': False
}

3
zun/api/controllers/v1/views/containers_view.py
View File

@ -37,7 +37,8 @@ _basic_keys = (
'restart_policy',
'status_detail',
'tty',
'stdin_open'
'stdin_open',
'image_driver'
)

7
zun/api/utils.py
View File

@ -33,6 +33,13 @@ JSONPATCH_EXCEPTIONS = (jsonpatch.JsonPatchException,
DOCKER_MINIMUM_MEMORY = 4 * 1024 * 1024
def string_or_none(value):
if value in [None, 'None']:
return None
else:
return value
def validate_limit(limit):
try:
if limit is not None and int(limit) <= 0:

11
zun/common/validation/parameter_types.py
View File

@ -11,7 +11,13 @@
# under the License.
import copy
import zun.conf
CONF = zun.conf.CONF
image_driver_list = [driver for driver in CONF.image_driver_list]
image_driver_list_with_none = image_driver_list + [None, 'None']
non_negative_integer = {
'type': ['integer', 'string'],
@ -31,6 +37,11 @@ boolean = {
'enum': [True, 'True', 'true', False, 'False', 'false'],
}
image_driver = {
'type': ['string', 'null'],
'enum': image_driver_list_with_none
}
container_name = {
'type': ['string', 'null'],
'minLength': 2,

4
zun/compute/api.py
View File

@ -90,5 +90,5 @@ class API(object):
def image_pull(self, context, image, *args):
return self.rpcapi.image_pull(context, image, *args)
def image_search(self, context, image, *args):
return self.rpcapi.image_search(context, image, *args)
def image_search(self, context, image, image_driver, *args):
return self.rpcapi.image_search(context, image, image_driver, *args)

23
zun/compute/manager.py
View File

@ -21,11 +21,12 @@ from zun.common import exception
from zun.common.i18n import _LE
from zun.common import utils
from zun.common.utils import translate_exception
import zun.conf
from zun.container import driver
from zun.image import driver as image_driver
from zun.objects import fields
CONF = zun.conf.CONF
LOG = logging.getLogger(__name__)
@ -67,10 +68,14 @@ class Manager(object):
container.task_state = fields.TaskState.SANDBOX_CREATING
container.save(context)
sandbox_id = None
sandbox_image = 'kubernetes/pause'
sandbox_image = CONF.sandbox_image
sandbox_image_driver = CONF.sandbox_image_driver
sandbox_image_pull_policy = CONF.sandbox_image_pull_policy
repo, tag = utils.parse_image_name(sandbox_image)
try:
image = image_driver.pull_image(context, repo, tag, 'ifnotpresent')
image = image_driver.pull_image(context, repo, tag,
sandbox_image_pull_policy,
sandbox_image_driver)
sandbox_id = self.driver.create_sandbox(context, container,
image=sandbox_image)
except Exception as e:
@ -86,9 +91,11 @@ class Manager(object):
repo, tag = utils.parse_image_name(container.image)
image_pull_policy = utils.get_image_pull_policy(
container.image_pull_policy, tag)
image_driver_name = container.image_driver
try:
image = image_driver.pull_image(context, repo,
tag, image_pull_policy)
image = image_driver.pull_image(context, repo, tag,
image_pull_policy,
image_driver_name)
except exception.ImageNotFound as e:
with excutils.save_and_reraise_exception(reraise=reraise):
LOG.error(six.text_type(e))
@ -112,6 +119,7 @@ class Manager(object):
return
container.task_state = fields.TaskState.CONTAINER_CREATING
container.image_driver = image.get('driver')
container.save(context)
try:
container = self.driver.create(context, container,
@ -372,10 +380,11 @@ class Manager(object):
raise
@translate_exception
def image_search(self, context, image, exact_match):
def image_search(self, context, image, image_driver_name, exact_match):
LOG.debug('Searching image...', image=image)
try:
return image_driver.search_image(context, image, exact_match)
return image_driver.search_image(context, image,
image_driver_name, exact_match)
except Exception as e:
LOG.exception(_LE("Unexpected exception while searching "
"image: %s"), six.text_type(e))

3
zun/compute/rpcapi.py
View File

@ -89,10 +89,11 @@ class API(rpc_service.API):
host = None
self._cast(host, 'image_pull', image=image)
def image_search(self, context, image, exact_match):
def image_search(self, context, image, image_driver, exact_match):
# NOTE(hongbin): Image API doesn't support multiple compute nodes
# scenario yet, so we temporarily set host to None and rpc will
# choose an arbitrary host.
host = None
return self._call(host, 'image_search', image=image,
image_driver_name=image_driver,
exact_match=exact_match)

24
zun/conf/image_driver.py
View File

@ -18,11 +18,11 @@ from zun.conf import path
image_driver_opts = [
cfg.ListOpt(
'image_driver_list',
default=['glance.driver.GlanceDriver', 'docker.driver.DockerDriver'],
default=['glance', 'docker'],
help="""Defines the list of image driver to use for downloading image.
Possible values:
* ``docker.driver.DockerDriver``
* ``glance.driver.GlanceDriver``
* ``docker``
* ``glance``
Services which consume this:
* ``zun-compute``
Interdependencies to other options:
@ -30,6 +30,21 @@ Interdependencies to other options:
""")
]
sandbox_opts = [
cfg.StrOpt(
'sandbox_image',
default='kubernetes/pause',
help='Container image for sandbox container.'),
cfg.StrOpt(
'sandbox_image_driver',
default='docker',
help='Image driver for sandbox container.'),
cfg.StrOpt(
'sandbox_image_pull_policy',
default='ifnotpresent',
help='Image pull policy for sandbox image.'),
]
glance_driver_opts = [
cfg.StrOpt(
'images_directory',
@ -42,13 +57,14 @@ glance_driver_opts = [
glance_opt_group = cfg.OptGroup(name='glance',
title='Glance options for image management')
ALL_OPTS = (glance_driver_opts + image_driver_opts)
ALL_OPTS = (glance_driver_opts + image_driver_opts + sandbox_opts)
def register_opts(conf):
conf.register_group(glance_opt_group)
conf.register_opts(glance_driver_opts, group=glance_opt_group)
conf.register_opts(image_driver_opts)
conf.register_opts(sandbox_opts)
def list_opts():

35
zun/db/sqlalchemy/alembic/versions/5458f8394206_add_image_driver_field.py Normal file
View File

@ -0,0 +1,35 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""add image driver field
Revision ID: 5458f8394206
Revises: d1ef05fd92c8
Create Date: 2017-01-25 19:01:46.033461
"""
# revision identifiers, used by Alembic.
revision = '5458f8394206'
down_revision = 'd1ef05fd92c8'
branch_labels = None
depends_on = None
from alembic import op
import sqlalchemy as sa
def upgrade():
op.add_column('container',
sa.Column('image_driver', sa.Text(),
nullable=True))

1
zun/db/sqlalchemy/models.py
View File

@ -150,6 +150,7 @@ class Container(Base):
status_detail = Column(String(50))
tty = Column(Boolean, default=False)
stdin_open = Column(Boolean, default=False)
image_driver = Column(String(255))
class Image(Base):

28
zun/image/driver.py
View File

@ -16,7 +16,7 @@ import six
import sys
from oslo_log import log as logging
from oslo_utils import importutils
import stevedore
from zun.common import exception
from zun.common.i18n import _
@ -45,26 +45,34 @@ def load_image_driver(image_driver=None):
LOG.info(_LI("Loading container image driver '%s'"), image_driver)
try:
driver = importutils.import_object(
'zun.image.%s' % image_driver)
driver = stevedore.driver.DriverManager(
"zun.image.driver",
image_driver,
invoke_on_load=True).driver
if not isinstance(driver, ContainerImageDriver):
raise Exception(_('Expected driver of type: %s') %
str(ContainerImageDriver))
return driver
except ImportError:
except Exception:
LOG.exception(_LE("Unable to load the container image driver"))
sys.exit(1)
def pull_image(context, repo, tag, image_pull_policy):
image_driver_list = CONF.image_driver_list
def pull_image(context, repo, tag, image_pull_policy, image_driver):
if image_driver:
image_driver_list = [image_driver.lower()]
else:
image_driver_list = CONF.image_driver_list
for driver in image_driver_list:
try:
image_driver = load_image_driver(driver)
image = image_driver.pull_image(context, repo,
tag, image_pull_policy)
if image:
image['driver'] = driver.split('.')[0]
break
except exception.ImageNotFound:
image = None
@ -77,10 +85,14 @@ def pull_image(context, repo, tag, image_pull_policy):
return image
def search_image(context, image_name, exact_match):
def search_image(context, image_name, image_driver, exact_match):
images = []
repo, tag = parse_image_name(image_name)
for driver in CONF.image_driver_list:
if image_driver:
image_driver_list = [image_driver.lower()]
else:
image_driver_list = CONF.image_driver_list
for driver in image_driver_list:
try:
image_driver = load_image_driver(driver)
imgs = image_driver.search_image(context, repo, tag,

1
zun/image/glance/driver.py
View File

@ -98,7 +98,6 @@ class GlanceDriver(driver.ContainerImageDriver):
try:
# TODO(hongbin): find image by both repo and tag
images = utils.find_images(context, repo, exact_match)
LOG.debug('Image %s was found in glance' % repo)
return images
except Exception as e:
raise exception.ZunException(six.text_type(e))

4
zun/objects/container.py
View File

@ -30,7 +30,8 @@ class Container(base.ZunPersistentObject, base.ZunObject):
# Version 1.8: Add restart_policy
# Version 1.9: Add status_detail column
# Version 1.10: Add tty, stdin_open
VERSION = '1.10'
# Version 1.11: Add image_driver
VERSION = '1.11'
fields = {
'id': fields.IntegerField(),
@ -59,6 +60,7 @@ class Container(base.ZunPersistentObject, base.ZunObject):
'status_detail': fields.StringField(nullable=True),
'tty': fields.BooleanField(nullable=True),
'stdin_open': fields.BooleanField(nullable=True),
'image_driver': fields.StringField(nullable=True)
}
@staticmethod

3
zun/tests/tempest/api/common/datagen.py
View File

@ -53,7 +53,8 @@ def container_data(**kwargs):
'image': 'cirros:latest',
'command': 'sleep 10000',
'memory': '100',
'environment': {}
'environment': {},
'image_driver': 'docker'
}
data.update(kwargs)

17
zun/tests/unit/api/controllers/v1/test_containers.py
View File

@ -1101,6 +1101,23 @@ class TestContainerController(api_base.FunctionalTest):
'/v1/containers/%s/%s/' % (container_uuid, 'kill'))
self.assertTrue(mock_container_kill.called)
@patch('zun.compute.api.API.container_create')
@patch('zun.compute.api.API.image_search')
def test_create_container_resp_has_image_driver(self, mock_search,
mock_container_create):
mock_container_create.side_effect = lambda x, y: y
# Create a container with a command
params = ('{"name": "MyDocker", "image": "ubuntu",'
'"command": "env", "memory": "512",'
'"environment": {"key1": "val1", "key2": "val2"},'
'"image_driver": "glance"}')
response = self.app.post('/v1/containers/',
params=params,
content_type='application/json')
self.assertEqual(202, response.status_int)
self.assertIn('image_driver', response.json.keys())
self.assertEqual('glance', response.json.get('image_driver'))
class TestContainerEnforcement(api_base.FunctionalTest):

29
zun/tests/unit/api/controllers/v1/test_images.py
View File

@ -138,7 +138,7 @@ class TestImageController(api_base.FunctionalTest):
response = self.app.get('/v1/images/redis/search/')
self.assertEqual(200, response.status_int)
mock_image_search.assert_called_once_with(
mock.ANY, 'redis', False)
mock.ANY, 'redis', None, False)
@patch('zun.compute.api.API.image_search')
def test_search_image_with_tag(self, mock_image_search):
@ -146,37 +146,46 @@ class TestImageController(api_base.FunctionalTest):
response = self.app.get('/v1/images/redis:test/search/')
self.assertEqual(200, response.status_int)
mock_image_search.assert_called_once_with(
mock.ANY, 'redis:test', False)
mock.ANY, 'redis:test', None, False)
@patch('zun.compute.api.API.image_search')
def test_search_image_not_found(self, mock_image_search):
mock_image_search.side_effect = exception.ImageNotFound
self.assertRaises(AppError, self.app.get, '/v1/images/redis/search/')
mock_image_search.assert_called_once_with(
mock.ANY, 'redis', False)
mock.ANY, 'redis', None, False)
@patch('zun.compute.rpcapi.API.image_search')
def test_search_image_with_exact_match_true(self, mock_image_search):
mock_image_search.return_value = {'name': 'redis', 'stars': 2000}
response = self.app.get('/v1/images/redis/search?exact_match=true')
response = self.app.get(
'/v1/images/redis/search?exact_match=true&image_driver=docker')
self.assertEqual(200, response.status_int)
mock_image_search.assert_called_once_with(
mock.ANY, 'redis', True)
mock.ANY, 'redis', 'docker', True)
@patch('zun.compute.rpcapi.API.image_search')
def test_search_image_with_exact_match_false(self, mock_image_search):
mock_image_search.return_value = {'name': 'redis', 'stars': 2000}
response = self.app.get('/v1/images/redis/search?exact_match=false')
response = self.app.get(
'/v1/images/redis/search?exact_match=false&image_driver=glance')
self.assertEqual(200, response.status_int)
mock_image_search.assert_called_once_with(
mock.ANY, 'redis', False)
mock.ANY, 'redis', 'glance', False)
@patch('zun.compute.api.API.image_search')
def test_search_image_with_exact_match_wrong(self, mock_image_search):
mock_image_search.side_effect = exception.InvalidValue
self.assertRaises(AppError, self.app.get,
'/v1/images/redis/search?exact_match=wrong')
self.assertTrue(mock_image_search.not_called)
with self.assertRaisesRegexp(AppError,
"Invalid input for query parameters"):
self.app.get('/v1/images/redis/search?exact_match=wrong')
@patch('zun.compute.api.API.image_search')
def test_search_image_with_image_driver_wrong(self, mock_image_search):
mock_image_search.side_effect = exception.InvalidValue
with self.assertRaisesRegexp(AppError,
"Invalid input for query parameters"):
self.app.get('/v1/images/redis/search?image_driver=wrong')
class TestImageEnforcement(api_base.FunctionalTest):

16
zun/tests/unit/common/test_validations.py
View File

@ -28,7 +28,8 @@ CONTAINER_CREATE = {
'image_pull_policy': parameter_types.image_pull_policy,
'labels': parameter_types.labels,
'environment': parameter_types.environment,
'restart_policy': parameter_types.restart_policy
'restart_policy': parameter_types.restart_policy,
'image_driver': parameter_types.image_driver
},
'required': ['image'],
'additionalProperties': False,
@ -48,7 +49,8 @@ class TestSchemaValidations(base.BaseTestCase):
'labels': {'abc': 12, 'bcd': 'xyz'},
'environment': {'xyz': 'pqr', 'pqr': 2},
'restart_policy': {'Name': 'no',
'MaximumRetryCount': '0'}}
'MaximumRetryCount': '0'},
'image_driver': 'docker'}
self.schema_validator.validate(request_to_validate)
def test_create_schema_with_all_parameters_none(self):
@ -58,7 +60,8 @@ class TestSchemaValidations(base.BaseTestCase):
'image_pull_policy': None,
'labels': None,
'environment': None,
'restart_policy': None
'restart_policy': None,
'image_driver': None
}
self.schema_validator.validate(request_to_validate)
@ -154,3 +157,10 @@ class TestSchemaValidations(base.BaseTestCase):
with self.assertRaisesRegexp(exception.SchemaValidationError,
"'Name' is a required property"):
self.schema_validator.validate(request_to_validate)
def test_create_schema_wrong_image_driver(self):
request_to_validate = {'image_driver': 'xyz', 'image': 'nginx'}
with self.assertRaisesRegexp(exception.SchemaValidationError,
"Invalid input for field"
" 'image_driver'"):
self.schema_validator.validate(request_to_validate)

24
zun/tests/unit/compute/test_compute_manager.py
View File

@ -50,14 +50,15 @@ class TestManager(base.TestCase):
def test_container_create(self, mock_create_sandbox, mock_create,
mock_pull, mock_save):
container = Container(self.context, **utils.get_test_container())
mock_pull.return_value = 'fake_path'
image = {'image': 'repo', 'path': 'out_path', 'driver': 'glance'}
mock_pull.return_value = image
mock_create_sandbox.return_value = 'fake_id'
self.compute_manager._do_container_create(self.context, container)
mock_save.assert_called_with(self.context)
mock_pull.assert_any_call(self.context, container.image, 'latest',
'always')
'always', 'glance')
mock_create.assert_called_once_with(self.context, container,
'fake_id', 'fake_path')
'fake_id', image)
@mock.patch.object(Container, 'save')
@mock.patch.object(fake_driver, 'create_sandbox')
@ -109,6 +110,8 @@ class TestManager(base.TestCase):
mock_create, mock_pull,
mock_save):
container = Container(self.context, **utils.get_test_container())
image = {'image': 'repo', 'path': 'out_path', 'driver': 'glance'}
mock_pull.return_value = image
mock_create.side_effect = exception.DockerError("Creation Failed")
mock_create_sandbox.return_value = mock.MagicMock()
self.compute_manager._do_container_create(self.context, container)
@ -122,15 +125,16 @@ class TestManager(base.TestCase):
def test_container_run(self, mock_start,
mock_create, mock_pull, mock_save):
container = Container(self.context, **utils.get_test_container())
mock_pull.return_value = 'fake_path'
image = {'image': 'repo', 'path': 'out_path', 'driver': 'glance'}
mock_create.return_value = container
mock_pull.return_value = image
container.status = 'Stopped'
self.compute_manager._do_container_run(self.context, container)
mock_save.assert_called_with(self.context)
mock_pull.assert_any_call(self.context, container.image, 'latest',
'always')
'always', 'glance')
mock_create.assert_called_once_with(self.context, container,
None, 'fake_path')
None, image)
mock_start.assert_called_once_with(container)
@mock.patch.object(Container, 'save')
@ -147,7 +151,7 @@ class TestManager(base.TestCase):
mock_fail.assert_called_with(self.context,
container, 'Image Not Found')
mock_pull.assert_called_once_with(self.context, 'kubernetes/pause',
'latest', 'ifnotpresent')
'latest', 'ifnotpresent', 'docker')
@mock.patch.object(Container, 'save')
@mock.patch('zun.image.driver.pull_image')
@ -163,7 +167,7 @@ class TestManager(base.TestCase):
mock_fail.assert_called_with(self.context,
container, 'Image Not Found')
mock_pull.assert_called_once_with(self.context, 'kubernetes/pause',
'latest', 'ifnotpresent')
'latest', 'ifnotpresent', 'docker')
@mock.patch.object(Container, 'save')
@mock.patch('zun.image.driver.pull_image')
@ -179,7 +183,7 @@ class TestManager(base.TestCase):
mock_fail.assert_called_with(self.context,
container, 'Docker Error occurred')
mock_pull.assert_called_once_with(self.context, 'kubernetes/pause',
'latest', 'ifnotpresent')
'latest', 'ifnotpresent', 'docker')
@mock.patch.object(Container, 'save')
@mock.patch('zun.image.driver.pull_image')
@ -198,7 +202,7 @@ class TestManager(base.TestCase):
mock_fail.assert_called_with(self.context,
container, 'Docker Error occurred')
mock_pull.assert_any_call(self.context, container.image, 'latest',
'always')
'always', 'glance')
mock_create.assert_called_once_with(self.context, container, None,
{'name': 'nginx', 'path': None})

1
zun/tests/unit/db/utils.py
View File

@ -62,6 +62,7 @@ def get_test_container(**kw):
'status_detail': kw.get('status_detail', 'up from 5 hours'),
'tty': kw.get('tty', True),
'stdin_open': kw.get('stdin_open', True),
'image_driver': 'glance'
}