openstack/glance
Code Issues Proposed changes

Merge "Adds 'web-download' import method"

Browse Source
This commit is contained in:
Zuul 2018-02-07 06:03:47 +00:00 committed by Gerrit Code Review
commit abfa272c1f
8 changed files with 197 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
glance/api/v2/discovery.py
View File

@ -32,13 +32,10 @@ class InfoController(object):
raise webob.exc.HTTPNotFound(explanation=msg)
# TODO(jokke): All the rest of the boundaries should be implemented.
# TODO(jokke): Once we have the rest of the methods implemented
# the value should be inherited from the CONF rather than hard-
# coded.
import_methods = {
'description': 'Import methods available.',
'type': 'array',
'value': ['glance-direct']
'value': CONF.get('enabled_import_methods')
}
return {

5
glance/api/v2/images.py
View File

@ -94,7 +94,8 @@ class ImagesController(object):
executor_factory = self.gateway.get_task_executor_factory(req.context)
task_repo = self.gateway.get_task_repo(req.context)
task_input = {'image_id': image_id}
task_input = {'image_id': image_id,
'import_req': body}
try:
import_task = task_factory.new_task(task_type='api_image_import',
@ -799,7 +800,7 @@ class RequestDeserializer(wsgi.JSONRequestDeserializer):
except KeyError:
msg = _("Import request requires a 'name' field.")
raise webob.exc.HTTPBadRequest(explanation=msg)
if method_name != 'glance-direct':
if method_name not in ['glance-direct', 'web-download']:
msg = _("Unknown import method name '%s'.") % method_name
raise webob.exc.HTTPBadRequest(explanation=msg)

37
glance/async/flows/_internal_plugins/__init__.py Normal file
View File

@ -0,0 +1,37 @@
# Copyright 2018 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_config import cfg
from stevedore import named
CONF = cfg.CONF
def get_import_plugin(**kwargs):
method_list = CONF.enabled_import_methods
import_method = kwargs.get('import_req')['method']['name'].replace("-",
"_")
if import_method in method_list:
task_list = [import_method]
# TODO(jokke): Implement error handling of non-listed methods.
extensions = named.NamedExtensionManager(
'glance.image_import.internal_plugins',
names=task_list,
name_order=True,
invoke_on_load=True,
invoke_kwds=kwargs)
for extension in extensions.extensions:
return extension.obj

127
glance/async/flows/_internal_plugins/web_download.py Normal file
View File

@ -0,0 +1,127 @@
# Copyright 2018 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from glance_store import backend
from oslo_config import cfg
from oslo_log import log as logging
from taskflow.patterns import linear_flow as lf
from taskflow import task
from taskflow.types import failure
from glance.common import exception
from glance.common.scripts import utils as script_utils
from glance.i18n import _, _LE
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
class _WebDownload(task.Task):
default_provides = 'file_uri'
def __init__(self, task_id, task_type, task_repo, image_id, uri):
self.task_id = task_id
self.task_type = task_type
self.task_repo = task_repo
self.image_id = image_id
self.uri = uri
super(_WebDownload, self).__init__(
name='%s-WebDownload-%s' % (task_type, task_id))
if CONF.node_staging_uri is None:
msg = (_("%(task_id)s of %(task_type)s not configured "
"properly. Missing node_staging_uri: %(work_dir)s") %
{'task_id': self.task_id,
'task_type': self.task_type,
'work_dir': CONF.node_staging_uri})
raise exception.BadTaskConfiguration(msg)
self.store = self._build_store()
def _build_store(self):
# NOTE(flaper87): Due to the nice glance_store api (#sarcasm), we're
# forced to build our own config object, register the required options
# (and by required I mean *ALL* of them, even the ones we don't want),
# and create our own store instance by calling a private function.
# This is certainly unfortunate but it's the best we can do until the
# glance_store refactor is done. A good thing is that glance_store is
# under our team's management and it gates on Glance so changes to
# this API will (should?) break task's tests.
conf = cfg.ConfigOpts()
backend.register_opts(conf)
conf.set_override('filesystem_store_datadir',
CONF.node_staging_uri[7:],
group='glance_store')
# NOTE(flaper87): Do not even try to judge me for this... :(
# With the glance_store refactor, this code will change, until
# that happens, we don't have a better option and this is the
# least worst one, IMHO.
store = backend._load_store(conf, 'file')
if store is None:
msg = (_("%(task_id)s of %(task_type)s not configured "
"properly. Could not load the filesystem store") %
{'task_id': self.task_id, 'task_type': self.task_type})
raise exception.BadTaskConfiguration(msg)
store.configure()
return store
def execute(self):
"""Create temp file into store and return path to it
:param image_id: Glance Image ID
"""
# NOTE(jokke): We've decided to use staging area for this task as
# a way to expect users to configure a local store for pre-import
# works on the image to happen.
#
# While using any path should be "technically" fine, it's not what
# we recommend as the best solution. For more details on this, please
# refer to the comment in the `_ImportToStore.execute` method.
data = script_utils.get_image_data_iter(self.uri)
path = self.store.add(self.image_id, data, 0)[0]
return path
def revert(self, result, **kwargs):
if isinstance(result, failure.Failure):
LOG.exception(_LE('Task: %(task_id)s failed to import image '
'%(image_id)s to the filesystem.'),
{'task_id': self.task_id,
'image_id': self.image_id})
def get_flow(**kwargs):
"""Return task flow for web-download.
:param task_id: Task ID.
:param task_type: Type of the task.
:param image_repo: Image repository used.
:param uri: URI the image data is downloaded from.
"""
task_id = kwargs.get('task_id')
task_type = kwargs.get('task_type')
image_repo = kwargs.get('image_repo')
image_id = kwargs.get('image_id')
uri = kwargs.get('import_req')['method'].get('uri')
return lf.Flow(task_type).add(
_WebDownload(task_id, task_type, image_repo, image_id, uri),
)

16
glance/async/flows/api_image_import.py
View File

@ -23,6 +23,7 @@ from taskflow.patterns import linear_flow as lf
from taskflow import retry
from taskflow import task
import glance.async.flows._internal_plugins as internal_plugins
import glance.async.flows.plugins as import_plugins
from glance.common import exception
from glance.common.scripts.image_import import main as image_import
@ -359,16 +360,23 @@ def get_flow(**kwargs):
task_repo = kwargs.get('task_repo')
image_repo = kwargs.get('image_repo')
image_id = kwargs.get('image_id')
uri = kwargs.get('uri')
import_method = kwargs.get('import_req')['method']['name']
uri = kwargs.get('import_req')['method'].get('uri')
if not uri:
if not uri and import_method == 'glance-direct':
separator = ''
if not CONF.node_staging_uri.endswith('/'):
separator = '/'
uri = separator.join((CONF.node_staging_uri, str(image_id)))
flow = lf.Flow(task_type, retry=retry.AlwaysRevert())
flow.add(_VerifyStaging(task_id, task_type, task_repo, uri))
if uri.startswith("http") and import_method == 'web-download':
downloadToStaging = internal_plugins.get_import_plugin(**kwargs)
flow.add(downloadToStaging)
else:
file_uri = uri
flow.add(_VerifyStaging(task_id, task_type, task_repo, file_uri))
for plugin in import_plugins.get_import_plugins(**kwargs):
flow.add(plugin)
@ -376,7 +384,7 @@ def get_flow(**kwargs):
import_to_store = _ImportToStore(task_id,
task_type,
image_repo,
uri,
file_uri,
image_id)
flow.add(import_to_store)

12
glance/common/config.py
View File

@ -726,6 +726,18 @@ to Image Import Refactoring work.
Related options:
* [DEFAULT]/node_staging_uri""")),
cfg.ListOpt('enabled_import_methods',
item_type=cfg.types.String(quotes=True),
bounds=True,
default=['glance-direct', 'web-download'],
help=_("""
List of enabled Image Import Methods
Both 'glance-direct' and 'web-download' are enabled by default.
Related options:
* [DEFAULT]/node_staging_uri
* [DEFAULT]/enable_image_import""")),
]
CONF = cfg.CONF

4
glance/tests/unit/v2/test_discovery_image_import.py
View File

@ -37,10 +37,10 @@ class TestInfoControllers(test_utils.BaseTestCase):
def test_get_import_info(self):
# TODO(rosmaita): change this when import methods are
# listed in the config file
import_method = 'glance-direct'
import_methods = ['glance-direct', 'web-download']
self.config(enable_image_import=True)
req = unit_test_utils.get_fake_request()
output = self.controller.get_image_import(req)
self.assertIn('import-methods', output)
self.assertEqual([import_method], output['import-methods']['value'])
self.assertEqual(import_methods, output['import-methods']['value'])

3
setup.cfg
View File

@ -76,6 +76,9 @@ glance.image_import.plugins =
no_op = glance.async.flows.plugins.no_op:get_flow
inject_image_metadata=glance.async.flows.plugins.inject_image_metadata:get_flow
glance.image_import.internal_plugins =
web_download = glance.async.flows._internal_plugins.web_download:get_flow
[build_sphinx]
builder = html man
all_files = 1