Merge "Use format_inspector from oslo"

glance/async_/flows/plugins/image_conversion.py

@@ -21,11 +21,11 @@ from oslo_config import cfg
 from oslo_log import log as logging
 from oslo_utils import encodeutils
 from oslo_utils import excutils
+from oslo_utils.imageutils import format_inspector
 from taskflow.patterns import linear_flow as lf
 from taskflow import task
 
 from glance.async_ import utils
-from glance.common import format_inspector
 from glance.i18n import _, _LI
 
 LOG = logging.getLogger(__name__)
@@ -99,12 +99,11 @@ class _ConvertImage(task.Task):
         # See https://bugs.launchpad.net/nova/+bug/2059809 for details.
         try:
             inspector = format_inspector.detect_file_format(self.src_path)
-            if not inspector.safety_check():
+            inspector.safety_check()
+        except format_inspector.SafetyCheckFailed:
             LOG.error('Image failed %s safety check; aborting conversion',
                       source_format)
             raise RuntimeError('Image has disallowed configuration')
-        except RuntimeError:
-            raise
         except format_inspector.ImageFormatError as e:
             LOG.error('Image claimed to be %s format failed format '
                       'inspection: %s', source_format, e)
@@ -113,7 +112,12 @@ class _ConvertImage(task.Task):
             LOG.exception('Unknown error inspecting image format: %s', e)
             raise RuntimeError('Unable to inspect image')
 
-        if str(inspector) == 'iso':
+        detected_format = str(inspector)
+        if detected_format == 'gpt':
+            # FIXME(danms): We need to consider GPT to be raw for compatibility
+            detected_format = 'raw'
+
+        if detected_format == 'iso':
             if source_format == 'iso':
                 # NOTE(abhishekk): Excluding conversion and preserving image
                 # disk_format as `iso` only
@@ -126,7 +130,7 @@ class _ConvertImage(task.Task):
             LOG.error('Image claimed to be %s format but format '
                       'inspection found: ISO', source_format)
             raise RuntimeError("Image has disallowed configuration")
-        elif str(inspector) != source_format:
+        elif detected_format != source_format:
             LOG.error('Image claimed to be %s format failed format '
                       'inspection', source_format)
             raise RuntimeError('Image format mismatch')

glance/location.py

@@ -25,9 +25,9 @@ from oslo_config import cfg
 from oslo_log import log as logging
 from oslo_utils import encodeutils
 from oslo_utils import excutils
+from oslo_utils.imageutils import format_inspector
 
 from glance.common import exception
-from glance.common import format_inspector
 from glance.common import store_utils
 from glance.common import utils
 import glance.domain.proxy
@@ -580,37 +580,55 @@ class ImageProxy(glance.domain.proxy.Image):
                 img_signature_key_type=key_type
             )
 
-        if not self.image.virtual_size:
+        if self.image.container_format == 'bare':
-            inspector = format_inspector.get_inspector(self.image.disk_format)
+            # FIXME(danms): We do not pass an expected_format here because
-        else:
+            # we do not (currently) want to interrupt the data pipeline if
-            # No need to do this again
+            # the format does not match.
-            inspector = None
+            data = format_inspector.InspectWrapper(data)
-
+            LOG.debug('Enabling in-flight format inspection for %s',
-        if inspector and self.image.container_format == 'bare':
+                      self.image.disk_format)
-            fmt = inspector()
-            data = format_inspector.InfoWrapper(data, fmt)
-            LOG.debug('Enabling in-flight format inspection for %s', fmt)
-        else:
-            fmt = None
 
         self._upload_to_store(data, verifier, backend, size)
 
-        virtual_size = 0
-        if fmt and fmt.format_match:
         try:
-                virtual_size = fmt.virtual_size
+            data.close()
+        except AttributeError:
+            # We did not get a closeable or file-like object as data and/or
+            # did not wrap it because of container_format
+            pass
+
+        virtual_size = 0
+        try:
+            inspector = data.format
+            format = str(inspector)
+            # format_inspector detects GPT, which we need to treat as raw for
+            # compatibility reasons
+            if format == 'gpt':
+                format = 'raw'
+            if format == self.image.disk_format:
+                virtual_size = inspector.virtual_size
                 LOG.info('Image format matched and virtual size computed: %i',
                          virtual_size)
-            except Exception as e:
+            else:
-                LOG.error(_LE('Unable to determine virtual_size because: %s'),
+                LOG.info('Image declared as %s but detected as %s, '
-                          e)
+                         'not updating virtual_size',
-        elif fmt:
+                         self.image.disk_format, inspector)
+        except AttributeError:
+            # We must not have wrapped this for inspection because of
+            # container_format
+            pass
+        except format_inspector.ImageFormatError:
+            # No format matched!
+            # FIXME(danms): This should be an error in the future for most
+            # cases
             LOG.warning('Image format %s did not match; '
                         'unable to calculate virtual size',
                         self.image.disk_format)
+        except Exception as e:
+            LOG.error(_LE('Unable to determine virtual_size because: %s'), e)
 
         if virtual_size:
-            self.image.virtual_size = fmt.virtual_size
+            self.image.virtual_size = virtual_size
 
         if set_active and self.image.status != 'active':
             self.image.status = 'active'

glance/tests/unit/async_/flows/plugins/test_image_conversion.py

@@ -21,11 +21,11 @@ from unittest import mock
 import glance_store
 from oslo_concurrency import processutils
 from oslo_config import cfg
+from oslo_utils.imageutils import format_inspector
 
 import glance.async_.flows.api_image_import as import_flow
 import glance.async_.flows.plugins.image_conversion as image_conversion
 from glance.async_ import utils as async_utils
-from glance.common import format_inspector
 from glance.common import utils
 from glance import domain
 from glance import gateway
@@ -97,8 +97,8 @@ class TestConvertImageTask(test_utils.BaseTestCase):
                                                        self.task.task_id)
 
         self.detect_file_format_mock = mock.MagicMock()
-        self.useFixture(fixtures.MockPatch('glance.common.format_inspector.'
+        self.useFixture(fixtures.MockPatch(
-                                           'detect_file_format',
+            'oslo_utils.imageutils.format_inspector.detect_file_format',
             self.detect_file_format_mock))
 
     @mock.patch.object(os, 'stat')
@@ -126,7 +126,6 @@ class TestConvertImageTask(test_utils.BaseTestCase):
                                             'virtual-size': 456}
                 inspector = self.detect_file_format_mock.return_value
                 inspector.__str__.return_value = 'raw'
-                inspector.safety_check.return_value = True
                 image_convert.execute('file:///test/path.raw')
 
                 # NOTE(hemanthm): Asserting that the source format is passed
@@ -164,7 +163,6 @@ class TestConvertImageTask(test_utils.BaseTestCase):
         self.img_repo.get.return_value = image
         inspector = self.detect_file_format_mock.return_value
         inspector.__str__.return_value = 'iso'
-        inspector.safety_check.return_value = True
 
         image_convert.execute('file:///test/path.iso')
         self.assertEqual(fmt, image.disk_format)
@@ -206,7 +204,6 @@ class TestConvertImageTask(test_utils.BaseTestCase):
         with mock.patch.object(processutils, 'execute') as exc_mock:
             inspector = self.detect_file_format_mock.return_value
             inspector.__str__.return_value = 'qcow2'
-            inspector.safety_check.return_value = True
             exc_mock.side_effect = OSError('fail')
             self.assertRaises(OSError,
                               convert.execute, 'file:///test/path.raw')
@@ -227,7 +224,6 @@ class TestConvertImageTask(test_utils.BaseTestCase):
             exc_mock.return_value = '', 'some error'
             inspector = self.detect_file_format_mock.return_value
             inspector.__str__.return_value = 'qcow2'
-            inspector.safety_check.return_value = True
             self.assertRaises(RuntimeError,
                               convert.execute, 'file:///test/path.raw')
             exc_mock.assert_called_once_with(
@@ -249,7 +245,6 @@ class TestConvertImageTask(test_utils.BaseTestCase):
         with mock.patch.object(processutils, 'execute') as exc_mock:
             inspector = self.detect_file_format_mock.return_value
             inspector.__str__.return_value = 'qcow2'
-            inspector.safety_check.return_value = True
             exc_mock.return_value = json.dumps(data), ''
             e = self.assertRaises(RuntimeError,
                                   convert.execute, 'file:///test/path.qcow')
@@ -269,7 +264,6 @@ class TestConvertImageTask(test_utils.BaseTestCase):
             exc_mock.return_value = json.dumps(data), ''
             inspector = self.detect_file_format_mock.return_value
             inspector.__str__.return_value = 'qcow2'
-            inspector.safety_check.return_value = True
             e = self.assertRaises(RuntimeError,
                                   convert.execute, 'file:///test/path.qcow')
             self.assertEqual('QCOW images with data-file set are not allowed',
@@ -284,7 +278,8 @@ class TestConvertImageTask(test_utils.BaseTestCase):
     def test_image_convert_fails_inspection_safety_check(self):
         convert = self._setup_image_convert_info_fail()
         inspector = self.detect_file_format_mock.return_value
-        inspector.safety_check.return_value = False
+        inspector.safety_check.side_effect = (
+            format_inspector.SafetyCheckFailed({}))
         self.assertRaisesRegex(RuntimeError,
                                'Image has disallowed configuration',
                                convert.execute, 'file:///test/path.qcow')
@@ -315,7 +310,6 @@ class TestConvertImageTask(test_utils.BaseTestCase):
         with mock.patch.object(processutils, 'execute') as exc_mock:
             inspector = self.detect_file_format_mock.return_value
             inspector.__str__.return_value = 'vmdk'
-            inspector.safety_check.return_value = True
             exc_mock.return_value = json.dumps(data), ''
             convert.execute('file:///test/path.vmdk')
 
@@ -347,7 +341,6 @@ class TestConvertImageTask(test_utils.BaseTestCase):
         with mock.patch.object(processutils, 'execute') as exc_mock:
             inspector = self.detect_file_format_mock.return_value
             inspector.__str__.return_value = 'raw'
-            inspector.safety_check.return_value = True
             exc_mock.side_effect = [('{"format":"raw"}', ''),
                                     OSError('convert_fail')]
             self.assertRaises(OSError,
@@ -373,7 +366,6 @@ class TestConvertImageTask(test_utils.BaseTestCase):
                                     ('', 'some error')]
             inspector = self.detect_file_format_mock.return_value
             inspector.__str__.return_value = 'raw'
-            inspector.safety_check.return_value = True
             self.assertRaises(RuntimeError,
                               convert.execute, 'file:///test/path.raw')
             exc_mock.assert_has_calls(
@@ -396,7 +388,6 @@ class TestConvertImageTask(test_utils.BaseTestCase):
             exc_mock.return_value = ('{}', '')
             inspector = self.detect_file_format_mock.return_value
             inspector.__str__.return_value = 'qcow2'
-            inspector.safety_check.return_value = True
             exc = self.assertRaises(RuntimeError,
                                     convert.execute, 'file:///test/path.raw')
             self.assertIn('Image metadata disagrees about format', str(exc))
@@ -429,7 +420,6 @@ class TestConvertImageTask(test_utils.BaseTestCase):
                 '{"format": "qcow2", "virtual-size": 123}', '')
             inspector = self.detect_file_format_mock.return_value
             inspector.__str__.return_value = 'qcow2'
-            inspector.safety_check.return_value = True
             convert.execute('file:///test/path.qcow')
             # Make sure we only called qemu-img for inspection, not conversion
             exc_mock.assert_called_once_with(

glance/tests/unit/common/test_format_inspector.py

@@ -1,648 +0,0 @@
-# Copyright 2020 Red Hat, Inc
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-import io
-import os
-import re
-import struct
-import subprocess
-import tempfile
-from unittest import mock
-
-from oslo_utils import units
-
-from glance.common import format_inspector
-from glance.tests import utils as test_utils
-
-
-TEST_IMAGE_PREFIX = 'glance-unittest-formatinspector-'
-
-
-def get_size_from_qemu_img(filename):
-    output = subprocess.check_output('qemu-img info "%s"' % filename,
-                                     shell=True)
-    for line in output.split(b'\n'):
-        m = re.search(b'^virtual size: .* .([0-9]+) bytes', line.strip())
-        if m:
-            return int(m.group(1))
-
-    raise Exception('Could not find virtual size with qemu-img')
-
-
-class TestFormatInspectors(test_utils.BaseTestCase):
-    def setUp(self):
-        super(TestFormatInspectors, self).setUp()
-        self._created_files = []
-
-    def tearDown(self):
-        super(TestFormatInspectors, self).tearDown()
-        for fn in self._created_files:
-            try:
-                os.remove(fn)
-            except Exception:
-                pass
-
-    def _create_iso(self, image_size, subformat='9660'):
-        """Create an ISO file of the given size.
-        :param image_size: The size of the image to create in bytes
-        :param subformat: The subformat to use, if any
-        """
-        # these tests depend on mkisofs
-        # being installed and in the path,
-        # if it is not installed, skip
-        try:
-            subprocess.check_output('mkisofs --version', shell=True)
-        except Exception:
-            self.skipTest('mkisofs not installed')
-        size = image_size // units.Mi
-        base_cmd = "mkisofs"
-        if subformat == 'udf':
-            # depending on the distribution mkisofs may not support udf
-            # and may be provided by genisoimage instead. As a result we
-            # need to check if the command supports udf via help
-            # instead of checking the installed version.
-            # mkisofs --help outputs to stderr so we need to
-            # redirect it to stdout to use grep.
-            try:
-                subprocess.check_output(
-                    'mkisofs --help 2>&1 | grep udf', shell=True)
-            except Exception:
-                self.skipTest('mkisofs does not support udf format')
-            base_cmd += " -udf"
-        prefix = TEST_IMAGE_PREFIX
-        prefix += '-%s-' % subformat
-        fn = tempfile.mktemp(prefix=prefix, suffix='.iso')
-        self._created_files.append(fn)
-        subprocess.check_output(
-            'dd if=/dev/zero of=%s bs=1M count=%i' % (fn, size),
-            shell=True)
-        subprocess.check_output(
-            '%s -V "TEST" -o %s  %s' % (base_cmd, fn, fn),
-            shell=True)
-        return fn
-
-    def _generate_bad_iso(self):
-        # we want to emulate a malicious user who uploads a an
-        # ISO file has a qcow2 header in the system area
-        # of the ISO file
-        # we will create a qcow2 image and an ISO file
-        # and then copy the qcow2 header to the ISO file
-        # e.g.
-        #   mkisofs -o orig.iso /etc/resolv.conf
-        #   qemu-img create orig.qcow2 -f qcow2 64M
-        #   dd if=orig.qcow2 of=outcome bs=32K count=1
-        #   dd if=orig.iso of=outcome bs=32K skip=1 seek=1
-        qcow = self._create_img('qcow2', 10 * units.Mi)
-        iso = self._create_iso(64 * units.Mi, subformat='9660')
-        # first ensure the files are valid
-        iso_fmt = self._test_format_at_block_size('iso', iso, 4 * units.Ki)
-        self.assertTrue(iso_fmt.format_match)
-        qcow_fmt = self._test_format_at_block_size('qcow2', qcow, 4 * units.Ki)
-        self.assertTrue(qcow_fmt.format_match)
-        # now copy the qcow2 header to an ISO file
-        prefix = TEST_IMAGE_PREFIX
-        prefix += '-bad-'
-        fn = tempfile.mktemp(prefix=prefix, suffix='.iso')
-        self._created_files.append(fn)
-        subprocess.check_output(
-            'dd if=%s of=%s bs=32K count=1' % (qcow, fn),
-            shell=True)
-        subprocess.check_output(
-            'dd if=%s of=%s bs=32K skip=1 seek=1' % (iso, fn),
-            shell=True)
-        return qcow, iso, fn
-
-    def _create_img(self, fmt, size, subformat=None, options=None,
-                    backing_file=None):
-        """Create an image file of the given format and size.
-        :param fmt: The format to create
-        :param size: The size of the image to create in bytes
-        :param subformat: The subformat to use, if any
-        :param options: A dictionary of options to pass to the format
-        :param backing_file: The backing file to use, if any
-        """
-
-        if fmt == 'iso':
-            return self._create_iso(size, subformat)
-        # these tests depend on qemu-img
-        # being installed and in the path,
-        # if it is not installed, skip
-        try:
-            subprocess.check_output('qemu-img --version', shell=True)
-        except Exception:
-            self.skipTest('qemu-img not installed')
-
-        if fmt == 'vhd':
-            # QEMU calls the vhd format vpc
-            fmt = 'vpc'
-
-        if options is None:
-            options = {}
-        opt = ''
-        prefix = TEST_IMAGE_PREFIX
-
-        if subformat:
-            options['subformat'] = subformat
-            prefix += subformat + '-'
-
-        if options:
-            opt += '-o ' + ','.join('%s=%s' % (k, v)
-                                    for k, v in options.items())
-
-        if backing_file is not None:
-            opt += ' -b %s -F raw' % backing_file
-
-        fn = tempfile.mktemp(prefix=prefix,
-                             suffix='.%s' % fmt)
-        self._created_files.append(fn)
-        subprocess.check_output(
-            'qemu-img create -f %s %s %s %i' % (fmt, opt, fn, size),
-            shell=True)
-        return fn
-
-    def _create_allocated_vmdk(self, size_mb, subformat=None):
-        # We need a "big" VMDK file to exercise some parts of the code of the
-        # format_inspector. A way to create one is to first create an empty
-        # file, and then to convert it with the -S 0 option.
-
-        if subformat is None:
-            # Matches qemu-img default, see `qemu-img convert -O vmdk -o help`
-            subformat = 'monolithicSparse'
-
-        prefix = TEST_IMAGE_PREFIX
-        prefix += '-%s-' % subformat
-        fn = tempfile.mktemp(prefix=prefix, suffix='.vmdk')
-        self._created_files.append(fn)
-        raw = tempfile.mktemp(prefix=prefix, suffix='.raw')
-        self._created_files.append(raw)
-
-        # Create a file with pseudo-random data, otherwise it will get
-        # compressed in the streamOptimized format
-        subprocess.check_output(
-            'dd if=/dev/urandom of=%s bs=1M count=%i' % (raw, size_mb),
-            shell=True)
-
-        # Convert it to VMDK
-        subprocess.check_output(
-            'qemu-img convert -f raw -O vmdk -o subformat=%s -S 0 %s %s' % (
-                subformat, raw, fn),
-            shell=True)
-        return fn
-
-    def _test_format_at_block_size(self, format_name, img, block_size):
-        fmt = format_inspector.get_inspector(format_name)()
-        self.assertIsNotNone(fmt,
-                             'Did not get format inspector for %s' % (
-                                 format_name))
-        wrapper = format_inspector.InfoWrapper(open(img, 'rb'), fmt)
-
-        while True:
-            chunk = wrapper.read(block_size)
-            if not chunk:
-                break
-
-        wrapper.close()
-        return fmt
-
-    def _test_format_at_image_size(self, format_name, image_size,
-                                   subformat=None):
-        """Test the format inspector for the given format at the
-        given image size.
-        :param format_name: The format to test
-        :param image_size: The size of the image to create in bytes
-        :param subformat: The subformat to use, if any
-        """
-        img = self._create_img(format_name, image_size, subformat=subformat)
-
-        # Some formats have internal alignment restrictions making this not
-        # always exactly like image_size, so get the real value for comparison
-        virtual_size = get_size_from_qemu_img(img)
-
-        # Read the format in various sizes, some of which will read whole
-        # sections in a single read, others will be completely unaligned, etc.
-        block_sizes = [64 * units.Ki, 1 * units.Mi]
-        # ISO images have a 32KB system area at the beginning of the image
-        # as a result reading that in 17 or 512 byte blocks takes too long,
-        # causing the test to fail. The 64KiB block size is enough to read
-        # the system area and header in a single read. the 1MiB block size
-        # adds very little time to the test so we include it.
-        if format_name != 'iso':
-            block_sizes.extend([17, 512])
-        for block_size in block_sizes:
-            fmt = self._test_format_at_block_size(format_name, img, block_size)
-            self.assertTrue(fmt.format_match,
-                            'Failed to match %s at size %i block %i' % (
-                                format_name, image_size, block_size))
-            self.assertEqual(virtual_size, fmt.virtual_size,
-                             ('Failed to calculate size for %s at size %i '
-                              'block %i') % (format_name, image_size,
-                                             block_size))
-            memory = sum(fmt.context_info.values())
-            self.assertLess(memory, 512 * units.Ki,
-                            'Format used more than 512KiB of memory: %s' % (
-                                fmt.context_info))
-
-    def _test_format(self, format_name, subformat=None):
-        # Try a few different image sizes, including some odd and very small
-        # sizes
-        for image_size in (512, 513, 2057, 7):
-            self._test_format_at_image_size(format_name, image_size * units.Mi,
-                                            subformat=subformat)
-
-    def test_qcow2(self):
-        self._test_format('qcow2')
-
-    def test_iso_9660(self):
-        self._test_format('iso', subformat='9660')
-
-    def test_udf(self):
-        self._test_format('iso', subformat='udf')
-
-    def test_bad_iso_qcow2(self):
-        _, _, fn = self._generate_bad_iso()
-        iso_check = self._test_format_at_block_size('iso', fn, 4 * units.Ki)
-        qcow_check = self._test_format_at_block_size('qcow2', fn, 4 * units.Ki)
-        # this system area of the ISO file is not considered part of the format
-        # the qcow2 header is in the system area of the ISO file
-        # so the ISO file is still valid
-        self.assertTrue(iso_check.format_match)
-        # the qcow2 header is in the system area of the ISO file
-        # but that will be parsed by the qcow2 format inspector
-        # and it will match
-        self.assertTrue(qcow_check.format_match)
-        # if we call format_inspector.detect_file_format it should detect
-        # and raise an exception because both match internally.
-        e = self.assertRaises(
-            format_inspector.ImageFormatError,
-            format_inspector.detect_file_format, fn)
-        self.assertIn('Multiple formats detected', str(e))
-
-    def test_vhd(self):
-        self._test_format('vhd')
-
-    def test_vhdx(self):
-        self._test_format('vhdx')
-
-    def test_vmdk(self):
-        self._test_format('vmdk')
-
-    def test_vmdk_stream_optimized(self):
-        self._test_format('vmdk', 'streamOptimized')
-
-    def test_from_file_reads_minimum(self):
-        img = self._create_img('qcow2', 10 * units.Mi)
-        file_size = os.stat(img).st_size
-        fmt = format_inspector.QcowInspector.from_file(img)
-        # We know everything we need from the first 512 bytes of a QCOW image,
-        # so make sure that we did not read the whole thing when we inspect
-        # a local file.
-        self.assertLess(fmt.actual_size, file_size)
-
-    def qed_supported(self):
-        output = subprocess.check_output('qemu-img create --help', shell=True)
-        return b' qed ' in output
-
-    def test_qed_always_unsafe(self):
-        if not self.qed_supported():
-            raise self.skipException('qed format not supported by qemu-img')
-
-        img = self._create_img('qed', 10 * units.Mi)
-        fmt = format_inspector.get_inspector('qed').from_file(img)
-        self.assertTrue(fmt.format_match)
-        self.assertFalse(fmt.safety_check())
-
-    def _test_vmdk_bad_descriptor_offset(self, subformat=None):
-        format_name = 'vmdk'
-        image_size = 10 * units.Mi
-        descriptorOffsetAddr = 0x1c
-        BAD_ADDRESS = 0x400
-        img = self._create_img(format_name, image_size, subformat=subformat)
-
-        # Corrupt the header
-        fd = open(img, 'r+b')
-        fd.seek(descriptorOffsetAddr)
-        fd.write(struct.pack('<Q', BAD_ADDRESS // 512))
-        fd.close()
-
-        # Read the format in various sizes, some of which will read whole
-        # sections in a single read, others will be completely unaligned, etc.
-        for block_size in (64 * units.Ki, 512, 17, 1 * units.Mi):
-            fmt = self._test_format_at_block_size(format_name, img, block_size)
-            self.assertTrue(fmt.format_match,
-                            'Failed to match %s at size %i block %i' % (
-                                format_name, image_size, block_size))
-            self.assertEqual(0, fmt.virtual_size,
-                             ('Calculated a virtual size for a corrupt %s at '
-                              'size %i block %i') % (format_name, image_size,
-                                                     block_size))
-
-    def test_vmdk_bad_descriptor_offset(self):
-        self._test_vmdk_bad_descriptor_offset()
-
-    def test_vmdk_bad_descriptor_offset_stream_optimized(self):
-        self._test_vmdk_bad_descriptor_offset(subformat='streamOptimized')
-
-    def _test_vmdk_bad_descriptor_mem_limit(self, subformat=None):
-        format_name = 'vmdk'
-        image_size = 5 * units.Mi
-        virtual_size = 5 * units.Mi
-        descriptorOffsetAddr = 0x1c
-        descriptorSizeAddr = descriptorOffsetAddr + 8
-        twoMBInSectors = (2 << 20) // 512
-        # We need a big VMDK because otherwise we will not have enough data to
-        # fill-up the CaptureRegion.
-        img = self._create_allocated_vmdk(image_size // units.Mi,
-                                          subformat=subformat)
-
-        # Corrupt the end of descriptor address so it "ends" at 2MB
-        fd = open(img, 'r+b')
-        fd.seek(descriptorSizeAddr)
-        fd.write(struct.pack('<Q', twoMBInSectors))
-        fd.close()
-
-        # Read the format in various sizes, some of which will read whole
-        # sections in a single read, others will be completely unaligned, etc.
-        for block_size in (64 * units.Ki, 512, 17, 1 * units.Mi):
-            fmt = self._test_format_at_block_size(format_name, img, block_size)
-            self.assertTrue(fmt.format_match,
-                            'Failed to match %s at size %i block %i' % (
-                                format_name, image_size, block_size))
-            self.assertEqual(virtual_size, fmt.virtual_size,
-                             ('Failed to calculate size for %s at size %i '
-                              'block %i') % (format_name, image_size,
-                                             block_size))
-            memory = sum(fmt.context_info.values())
-            self.assertLess(memory, 1.5 * units.Mi,
-                            'Format used more than 1.5MiB of memory: %s' % (
-                                fmt.context_info))
-
-    def test_vmdk_bad_descriptor_mem_limit(self):
-        self._test_vmdk_bad_descriptor_mem_limit()
-
-    def test_vmdk_bad_descriptor_mem_limit_stream_optimized(self):
-        self._test_vmdk_bad_descriptor_mem_limit(subformat='streamOptimized')
-
-    def test_qcow2_safety_checks(self):
-        # Create backing and data-file names (and initialize the backing file)
-        backing_fn = tempfile.mktemp(prefix='backing')
-        self._created_files.append(backing_fn)
-        with open(backing_fn, 'w') as f:
-            f.write('foobar')
-        data_fn = tempfile.mktemp(prefix='data')
-        self._created_files.append(data_fn)
-
-        # A qcow with no backing or data file is safe
-        fn = self._create_img('qcow2', 5 * units.Mi, None)
-        inspector = format_inspector.QcowInspector.from_file(fn)
-        self.assertTrue(inspector.safety_check())
-
-        # A backing file makes it unsafe
-        fn = self._create_img('qcow2', 5 * units.Mi, None,
-                              backing_file=backing_fn)
-        inspector = format_inspector.QcowInspector.from_file(fn)
-        self.assertFalse(inspector.safety_check())
-
-        # A data-file makes it unsafe
-        fn = self._create_img('qcow2', 5 * units.Mi,
-                              options={'data_file': data_fn,
-                                       'data_file_raw': 'on'})
-        inspector = format_inspector.QcowInspector.from_file(fn)
-        self.assertFalse(inspector.safety_check())
-
-        # Trying to load a non-QCOW file is an error
-        self.assertRaises(format_inspector.ImageFormatError,
-                          format_inspector.QcowInspector.from_file,
-                          backing_fn)
-
-    def test_qcow2_feature_flag_checks(self):
-        data = bytearray(512)
-        data[0:4] = b'QFI\xFB'
-        inspector = format_inspector.QcowInspector()
-        inspector.region('header').data = data
-
-        # All zeros, no feature flags - all good
-        self.assertFalse(inspector.has_unknown_features)
-
-        # A feature flag set in the first byte (highest-order) is not
-        # something we know about, so fail.
-        data[0x48] = 0x01
-        self.assertTrue(inspector.has_unknown_features)
-
-        # The first bit in the last byte (lowest-order) is known (the dirty
-        # bit) so that should pass
-        data[0x48] = 0x00
-        data[0x4F] = 0x01
-        self.assertFalse(inspector.has_unknown_features)
-
-        # Currently (as of 2024), the high-order feature flag bit in the low-
-        # order byte is not assigned, so make sure we reject it.
-        data[0x4F] = 0x80
-        self.assertTrue(inspector.has_unknown_features)
-
-    def test_vdi(self):
-        self._test_format('vdi')
-
-    def _test_format_with_invalid_data(self, format_name):
-        fmt = format_inspector.get_inspector(format_name)()
-        wrapper = format_inspector.InfoWrapper(open(__file__, 'rb'), fmt)
-        while True:
-            chunk = wrapper.read(32)
-            if not chunk:
-                break
-
-        wrapper.close()
-        self.assertFalse(fmt.format_match)
-        self.assertEqual(0, fmt.virtual_size)
-        memory = sum(fmt.context_info.values())
-        self.assertLess(memory, 512 * units.Ki,
-                        'Format used more than 512KiB of memory: %s' % (
-                            fmt.context_info))
-
-    def test_qcow2_invalid(self):
-        self._test_format_with_invalid_data('qcow2')
-
-    def test_vhd_invalid(self):
-        self._test_format_with_invalid_data('vhd')
-
-    def test_vhdx_invalid(self):
-        self._test_format_with_invalid_data('vhdx')
-
-    def test_vmdk_invalid(self):
-        self._test_format_with_invalid_data('vmdk')
-
-    def test_vdi_invalid(self):
-        self._test_format_with_invalid_data('vdi')
-
-    def test_vmdk_invalid_type(self):
-        fmt = format_inspector.get_inspector('vmdk')()
-        wrapper = format_inspector.InfoWrapper(open(__file__, 'rb'), fmt)
-        while True:
-            chunk = wrapper.read(32)
-            if not chunk:
-                break
-
-        wrapper.close()
-
-        fake_rgn = mock.MagicMock()
-        fake_rgn.complete = True
-        fake_rgn.data = b'foocreateType="someunknownformat"bar'
-
-        with mock.patch.object(fmt, 'has_region', return_value=True):
-            with mock.patch.object(fmt, 'region', return_value=fake_rgn):
-                self.assertEqual(0, fmt.virtual_size)
-
-
-class TestFormatInspectorInfra(test_utils.BaseTestCase):
-    def _test_capture_region_bs(self, bs):
-        data = b''.join(chr(x).encode() for x in range(ord('A'), ord('z')))
-
-        regions = [
-            format_inspector.CaptureRegion(3, 9),
-            format_inspector.CaptureRegion(0, 256),
-            format_inspector.CaptureRegion(32, 8),
-        ]
-
-        for region in regions:
-            # None of them should be complete yet
-            self.assertFalse(region.complete)
-
-        pos = 0
-        for i in range(0, len(data), bs):
-            chunk = data[i:i + bs]
-            pos += len(chunk)
-            for region in regions:
-                region.capture(chunk, pos)
-
-        self.assertEqual(data[3:12], regions[0].data)
-        self.assertEqual(data[0:256], regions[1].data)
-        self.assertEqual(data[32:40], regions[2].data)
-
-        # The small regions should be complete
-        self.assertTrue(regions[0].complete)
-        self.assertTrue(regions[2].complete)
-
-        # This region extended past the available data, so not complete
-        self.assertFalse(regions[1].complete)
-
-    def test_capture_region(self):
-        for block_size in (1, 3, 7, 13, 32, 64):
-            self._test_capture_region_bs(block_size)
-
-    def _get_wrapper(self, data):
-        source = io.BytesIO(data)
-        fake_fmt = mock.create_autospec(format_inspector.get_inspector('raw'))
-        return format_inspector.InfoWrapper(source, fake_fmt)
-
-    def test_info_wrapper_file_like(self):
-        data = b''.join(chr(x).encode() for x in range(ord('A'), ord('z')))
-        wrapper = self._get_wrapper(data)
-
-        read_data = b''
-        while True:
-            chunk = wrapper.read(8)
-            if not chunk:
-                break
-            read_data += chunk
-
-        self.assertEqual(data, read_data)
-
-    def test_info_wrapper_iter_like(self):
-        data = b''.join(chr(x).encode() for x in range(ord('A'), ord('z')))
-        wrapper = self._get_wrapper(data)
-
-        read_data = b''
-        for chunk in wrapper:
-            read_data += chunk
-
-        self.assertEqual(data, read_data)
-
-    def test_info_wrapper_file_like_eats_error(self):
-        wrapper = self._get_wrapper(b'123456')
-        wrapper._format.eat_chunk.side_effect = Exception('fail')
-
-        data = b''
-        while True:
-            chunk = wrapper.read(3)
-            if not chunk:
-                break
-            data += chunk
-
-        # Make sure we got all the data despite the error
-        self.assertEqual(b'123456', data)
-
-        # Make sure we only called this once and never again after
-        # the error was raised
-        wrapper._format.eat_chunk.assert_called_once_with(b'123')
-
-    def test_info_wrapper_iter_like_eats_error(self):
-        fake_fmt = mock.create_autospec(format_inspector.get_inspector('raw'))
-        wrapper = format_inspector.InfoWrapper(iter([b'123', b'456']),
-                                               fake_fmt)
-        fake_fmt.eat_chunk.side_effect = Exception('fail')
-
-        data = b''
-        for chunk in wrapper:
-            data += chunk
-
-        # Make sure we got all the data despite the error
-        self.assertEqual(b'123456', data)
-
-        # Make sure we only called this once and never again after
-        # the error was raised
-        fake_fmt.eat_chunk.assert_called_once_with(b'123')
-
-    def test_get_inspector(self):
-        self.assertEqual(format_inspector.QcowInspector,
-                         format_inspector.get_inspector('qcow2'))
-        self.assertIsNone(format_inspector.get_inspector('foo'))
-
-
-class TestFormatInspectorsTargeted(test_utils.BaseTestCase):
-    def _make_vhd_meta(self, guid_raw, item_length):
-        # Meta region header, padded to 32 bytes
-        data = struct.pack('<8sHH', b'metadata', 0, 1)
-        data += b'0' * 20
-
-        # Metadata table entry, 16-byte GUID, 12-byte information,
-        # padded to 32-bytes
-        data += guid_raw
-        data += struct.pack('<III', 256, item_length, 0)
-        data += b'0' * 6
-
-        return data
-
-    def test_vhd_table_over_limit(self):
-        ins = format_inspector.VHDXInspector()
-        meta = format_inspector.CaptureRegion(0, 0)
-        desired = b'012345678ABCDEF0'
-        # This is a poorly-crafted image that specifies a larger table size
-        # than is allowed
-        meta.data = self._make_vhd_meta(desired, 33 * 2048)
-        ins.new_region('metadata', meta)
-        new_region = ins._find_meta_entry(ins._guid(desired))
-        # Make sure we clamp to our limit of 32 * 2048
-        self.assertEqual(
-            format_inspector.VHDXInspector.VHDX_METADATA_TABLE_MAX_SIZE,
-            new_region.length)
-
-    def test_vhd_table_under_limit(self):
-        ins = format_inspector.VHDXInspector()
-        meta = format_inspector.CaptureRegion(0, 0)
-        desired = b'012345678ABCDEF0'
-        meta.data = self._make_vhd_meta(desired, 16 * 2048)
-        ins.new_region('metadata', meta)
-        new_region = ins._find_meta_entry(ins._guid(desired))
-        # Table size was under the limit, make sure we get it back
-        self.assertEqual(16 * 2048, new_region.length)

glance/tests/unit/test_store_image.py

@@ -12,6 +12,8 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
+import io
+
 from cryptography import exceptions as crypto_exception
 from cursive import exception as cursive_exception
 from cursive import signature_utils
@@ -117,14 +119,23 @@ class TestStoreMultiBackends(utils.BaseTestCase):
         image_stub.disk_format = 'iso'
         image = glance.location.ImageProxy(image_stub, context,
                                            self.store_api, self.store_utils)
+        found_data = []
+
+        def consumer(data, _v, _b, _s):
+            found_data.append(data.read(4))
+
         with mock.patch.object(image, "_upload_to_store") as mloc:
-            image.set_data('YYYY', 4, backend='ceph1')
+            mloc.side_effect = consumer
+            src_data = io.BytesIO(b'YYYY')
+            image.set_data(src_data, 4, backend='ceph1')
             msig.assert_called_once_with(context=context,
                                          img_signature_certificate_uuid='UUID',
                                          img_signature_hash_method='METHOD',
                                          img_signature='VALID',
                                          img_signature_key_type='TYPE')
-            mloc.assert_called_once_with('YYYY', msig.return_value, 'ceph1', 4)
+            mloc.assert_called_once_with(mock.ANY, msig.return_value, 'ceph1',
+                                         4)
+            self.assertEqual([b'YYYY'], found_data)
 
         self.assertEqual('active', image.status)
 
@@ -284,9 +295,11 @@ class TestStoreImage(utils.BaseTestCase):
         self.assertEqual('active', image.status)
         self.assertEqual(0, image.virtual_size)
 
-    @mock.patch('glance.common.format_inspector.QcowInspector.virtual_size',
+    @mock.patch('oslo_utils.imageutils.format_inspector.QcowInspector.'
+                'virtual_size',
                 new_callable=mock.PropertyMock)
-    @mock.patch('glance.common.format_inspector.QcowInspector.format_match',
+    @mock.patch('oslo_utils.imageutils.format_inspector.QcowInspector.'
+                'format_match',
                 new_callable=mock.PropertyMock)
     def test_image_set_data_inspector_virtual_size_failure(self, mock_fm,
                                                            mock_vs):
@@ -315,11 +328,12 @@ class TestStoreImage(utils.BaseTestCase):
         self.assertEqual('active', image.status)
         self.assertEqual(0, image.virtual_size)
 
-    @mock.patch('glance.common.format_inspector.get_inspector')
+    @mock.patch('oslo_utils.imageutils.format_inspector.InspectWrapper')
-    def test_image_set_data_inspector_not_needed(self, mock_gi):
+    def test_image_set_data_inspector_not_needed(self, mock_inspect):
         context = glance.context.RequestContext(user=USER1)
         image_stub = ImageStub(UUID2, status='queued', locations=[])
         image_stub.virtual_size = 123
+        image_stub.container_format = 'gzip'
         image_stub.disk_format = 'qcow2'
         # We are going to pass an iterable data source, so use the
         # FakeStoreAPIReader that actually reads from that data
@@ -335,7 +349,7 @@ class TestStoreImage(utils.BaseTestCase):
         self.assertEqual(123, image.virtual_size)
         # If the image already had virtual_size set (i.e. we're setting
         # a new location), we should not re-calculate the value.
-        mock_gi.assert_not_called()
+        mock_inspect.assert_not_called()
 
     def test_image_set_data_location_metadata(self):
         context = glance.context.RequestContext(user=USER1)

requirements.txt

@@ -13,7 +13,7 @@ oslo.config>=8.1.0 # Apache-2.0
 oslo.concurrency>=4.5.1 # Apache-2.0
 oslo.context>=2.22.0 # Apache-2.0
 oslo.upgradecheck>=1.3.0 # Apache-2.0
-oslo.utils>=4.7.0 # Apache-2.0
+oslo.utils>=7.3.0 # Apache-2.0
 stevedore!=3.0.0,>=1.20.0 # Apache-2.0
 futurist>=1.2.0 # Apache-2.0
 taskflow>=4.0.0 # Apache-2.0