Merge "Add image conversion plugin"

doc/source/admin/interoperable-image-import.rst

@@ -377,6 +377,87 @@ required.
    See the :ref:`property-protections` section of this Guide for more
    information.
 
+The Image Conversion
+--------------------
+.. list-table::
+
+   * - release introduced
+     - Rocky (Glance 17.0.0)
+   * - configuration file
+     - ``glance-image-import.conf``
+   * - configuration file section
+     - ``[image_conversion]``
+
+This plugin implements automated image conversion for Interoperable Image
+Import. One use case for this plugin would be environments where Ceph is used
+as image back-end and operators want to optimize the back-end capabilities by
+ensuring that all images will be in raw format while not putting the burden of
+converting the images to their end users.
+
+.. note::
+
+   This plugin may only be used as part of the interoperable image import
+   workflow (``POST v2/images/{image_id}/import``).  *It has no effect on the
+   image data upload call* (``PUT v2/images/{image_id}/file``).
+
+   You can guarantee that your end users must use interoperable image import by
+   restricting the ``upload_image`` policy appropriately in the Glance
+   ``policy.json`` file.  By default, this policy is unrestricted (that is,
+   any authorized user may make the image upload call).
+
+   For example, to allow only admin or service users to make the image upload
+   call, the policy could be restricted as follows:
+
+   .. code-block:: text
+
+      "upload_image": "role:admin or (service_user_id:<uuid of nova user>) or
+         (service_roles:<service user role>)"
+
+   where "service_role" is the role which is created for the service user
+   and assigned to trusted services.
+
+To use the Image Conversion Plugin, the following configuration is
+required.
+
+You will need to configure 'glance-image-import.conf' file as shown below:
+
+   .. code-block:: ini
+
+       [image_import_opts]
+       image_import_plugins = ['image_conversion']
+
+       [image_conversion]
+       output_format = raw
+
+.. note::
+
+  The default output format is raw in which case there is no need to have
+  'image_conversion' section and its 'output_format' defined in the config
+  file.
+
+  The input format needs to be one of the `qemu-img supported ones`_ for this
+  feature to work. In case of qemu-img call failing on the source image the
+  import process will fail if 'image_conversion' plugin is enabled.
+
+.. note::
+
+  ``image_import_plugins`` config option is a list and multiple plugins can be
+  enabled for the import flow. The plugins are not run in parallel. One can
+  enable multiple plugins by configuring them in the
+  ``glance-image-import.conf`` for example as following:
+
+  .. code-block:: ini
+
+       [image_import_opts]
+       image_import_plugins = ['inject_image_metadata', 'image_conversion']
+
+       [inject_metadata_properties]
+       ignore_user_roles = admin,...
+       inject = "property1":"value1","property2":"value2",...
+
+       [image_conversion]
+       output_format = raw
+
 .. _glance-api.conf: http://git.openstack.org/cgit/openstack/glance/tree/etc/glance-api.conf
 .. _glance-image-import.conf.sample: http://git.openstack.org/cgit/openstack/glance/tree/etc/glance-image-import.conf.sample
 .. _`Image Import Refactor`: https://specs.openstack.org/openstack/glance-specs/specs/mitaka/approved/image-import/image-import-refactor.html
@@ -387,4 +468,4 @@ required.
 .. _`Stevedore`: https://docs.openstack.org/stevedore
 .. _`Taskflow`: https://docs.openstack.org/taskflow
 .. _`Taskflow "Task" object`: https://docs.openstack.org/taskflow/latest/user/atoms.html#task
-
+.. _`qemu-img supported ones`: https://github.com/qemu/qemu/blob/master/qemu-img.texi#L599-L725

etc/glance-image-import.conf.sample

@@ -1,6 +1,34 @@
 [DEFAULT]
 
 
+[image_conversion]
+
+#
+# From glance
+#
+
+#
+# Desired output format for image conversion plugin.
+#
+# Provide a valid image format to which the conversion plugin
+# will convert the image before storing it to the back-end.
+#
+# Note, if the Image Conversion plugin for image import is defined, users
+# should only upload disk formats that are supported by `quemu-img` otherwise
+# the conversion and import will fail.
+#
+# Possible values:
+#     * qcow2
+#     * raw
+#     * vdmk
+#
+# Related Options:
+#     * disk_formats
+#  (string value)
+# Allowed values: qcow2, raw, vdmk
+#output_format = raw
+
+
 [image_import_opts]
 
 #

glance/async/flows/plugins/image_conversion.py

@@ -0,0 +1,165 @@
+# Copyright 2018 Red Hat, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import json
+import os
+
+from oslo_concurrency import processutils as putils
+from oslo_config import cfg
+from oslo_log import log as logging
+from oslo_utils import encodeutils
+from oslo_utils import excutils
+from taskflow.patterns import linear_flow as lf
+from taskflow import task
+
+from glance.async import utils
+from glance.i18n import _
+
+LOG = logging.getLogger(__name__)
+
+conversion_plugin_opts = [
+    cfg.StrOpt('output_format',
+               default='raw',
+               choices=('qcow2', 'raw', 'vdmk'),
+               help=_("""
+Desired output format for image conversion plugin.
+
+Provide a valid image format to which the conversion plugin
+will convert the image before storing it to the back-end.
+
+Note, if the Image Conversion plugin for image import is defined, users
+should only upload disk formats that are supported by `quemu-img` otherwise
+the conversion and import will fail.
+
+Possible values:
+    * qcow2
+    * raw
+    * vdmk
+
+Related Options:
+    * disk_formats
+""")),
+]
+
+CONF = cfg.CONF
+
+CONF.register_opts(conversion_plugin_opts, group='image_conversion')
+
+
+class _ConvertImage(task.Task):
+
+    default_provides = 'file_path'
+
+    def __init__(self, context, task_id, task_type,
+                 image_repo, image_id):
+        self.context = context
+        self.task_id = task_id
+        self.task_type = task_type
+        self.image_repo = image_repo
+        self.image_id = image_id
+        self.dest_path = ""
+        super(_ConvertImage, self).__init__(
+            name='%s-Convert_Image-%s' % (task_type, task_id))
+
+    def execute(self, file_path, **kwargs):
+
+        target_format = CONF.conversion_plugin_options.output_format
+        # TODO(jokke): Once we support other schemas we need to take them into
+        # account and handle the paths here.
+        src_path = file_path.split('file://')[-1]
+        dest_path = "%(path)s.%(target)s" % {'path': src_path,
+                                             'target': target_format}
+        self.dest_path = dest_path
+
+        try:
+            stdout, stderr = putils.trycmd("qemu-img", "info",
+                                           "--output=json",
+                                           src_path,
+                                           prlimit=utils.QEMU_IMG_PROC_LIMITS,
+                                           log_errors=putils.LOG_ALL_ERRORS,)
+        except OSError as exc:
+            with excutils.save_and_reraise_exception():
+                exc_message = encodeutils.exception_to_unicode(exc)
+                msg = ("Failed to do introspection as part of image "
+                       "conversion for %(iid)s: %(err)s")
+                LOG.error(msg, {'iid': self.image_id, 'err': exc_message})
+
+        if stderr:
+            raise RuntimeError(stderr)
+
+        metadata = json.loads(stdout)
+        source_format = metadata.get('format')
+        virtual_size = metadata.get('virtual-size', 0)
+        image = self.image_repo.get(self.image_id)
+        image.virtual_size = virtual_size
+
+        if source_format == target_format:
+            LOG.debug("Source is already in target format, "
+                      "not doing conversion for %s", self.image_id)
+            self.image_repo.save(image)
+            return file_path
+
+        try:
+            stdout, stderr = putils.trycmd('qemu-img', 'convert',
+                                           '-f', source_format,
+                                           '-O', target_format,
+                                           src_path, dest_path,
+                                           log_errors=putils.LOG_ALL_ERRORS)
+        except OSError as exc:
+            with excutils.save_and_reraise_exception():
+                exc_message = encodeutils.exception_to_unicode(exc)
+                msg = "Failed to do image conversion for %(iid)s: %(err)s"
+                LOG.error(msg, {'iid': self.image_id, 'err': exc_message})
+
+        if stderr:
+            raise RuntimeError(stderr)
+
+        image.disk_format = target_format
+        image.container_format = 'bare'
+        self.image_repo.save(image)
+
+        os.remove(src_path)
+
+        return "file://%s" % dest_path
+
+    def revert(self, result=None, **kwargs):
+        # NOTE(flaper87): If result is None, it probably
+        # means this task failed. Otherwise, we would have
+        # a result from its execution.
+        if result is not None:
+            LOG.debug("Image conversion failed.")
+            if os.path.exists(self.dest_path):
+                os.remove(self.dest_path)
+
+
+def get_flow(**kwargs):
+    """Return task flow for no-op.
+
+    :param context: request context
+    :param task_id: Task ID.
+    :param task_type: Type of the task.
+    :param image_repo: Image repository used.
+    :param image_id: Image ID
+    """
+    context = kwargs.get('context')
+    task_id = kwargs.get('task_id')
+    task_type = kwargs.get('task_type')
+    image_repo = kwargs.get('image_repo')
+    image_id = kwargs.get('image_id')
+
+    return lf.Flow(task_type).add(
+        _ConvertImage(context, task_id, task_type,
+                      image_repo, image_id),
+    )

glance/async/flows/plugins/plugin_opts.py

@@ -14,6 +14,7 @@
 # under the License.
 
 
+import glance.async.flows.plugins.image_conversion
 import glance.async.flows.plugins.inject_image_metadata
 
 
@@ -27,6 +28,8 @@ import glance.async.flows.plugins.inject_image_metadata
 PLUGIN_OPTS = [
     ('inject_metadata_properties',
      glance.async.flows.plugins.inject_image_metadata.inject_metadata_opts),
+    ('image_conversion',
+     glance.async.flows.plugins.image_conversion.conversion_plugin_opts),
 ]
 
 

glance/domain/__init__.py

@@ -194,7 +194,8 @@ class Image(object):
 
     @container_format.setter
     def container_format(self, value):
-        if hasattr(self, '_container_format') and self.status != 'queued':
+        if (hasattr(self, '_container_format') and
+                self.status not in ('queued', 'importing')):
             msg = _("Attribute container_format can be only replaced "
                     "for a queued image.")
             raise exception.Forbidden(message=msg)
@@ -206,7 +207,8 @@ class Image(object):
 
     @disk_format.setter
     def disk_format(self, value):
-        if hasattr(self, '_disk_format') and self.status != 'queued':
+        if (hasattr(self, '_disk_format') and
+                self.status not in ('queued', 'importing')):
             msg = _("Attribute disk_format can be only replaced "
                     "for a queued image.")
             raise exception.Forbidden(message=msg)

releasenotes/notes/image-conversion-plugin-5aee45e1a1a5bb2b.yaml

@@ -0,0 +1,14 @@
+---
+prelude: >
+  Automatic image conversion plugin for Interoperable Image Import. This
+  release introduces a new plugin that can be used to convert images to
+  specific format automatically upon image import.
+features:
+  - |
+    Automatic image conversion plugin for Interoperable Image Import. With
+    this release operators can specify target image format and get all images
+    created via the Image Import methods introduced in the Images API v2.6
+    converted automatically to that format. The feautre uses qemu-img under
+    the hood which limits the source image formats that users can upload. Any
+    image that fails the conversion when this plugin is enabled will fail the
+    image creation.

setup.cfg

@@ -75,6 +75,7 @@ glance.flows.import =
 glance.image_import.plugins =
     no_op = glance.async.flows.plugins.no_op:get_flow
     inject_image_metadata=glance.async.flows.plugins.inject_image_metadata:get_flow
+    image_conversion=glance.async.flows.plugins.image_conversion:get_flow
 
 glance.image_import.internal_plugins =
     web_download = glance.async.flows._internal_plugins.web_download:get_flow