Allow streaming raw partition images

Currently we support streaming raw whole disk images, but not
partition ones. This change enables it.

Change-Id: Ie95102aa3f2054a6b429f3d3e0926e90923c5faf
Story: #2003809
Task: #26558

ironic_python_agent/extensions/standby.py

@@ -57,6 +57,7 @@ def _write_partition_image(image, image_info, device):
     """Call disk_util to create partition and write the partition image.
 
     :param image: Local path to image file to be written to the partition.
+        If ``None``, the image is not populated.
     :param image_info: Image information dictionary.
     :param device: The device name, as a string, on which to store the image.
                    Example: '/dev/sda'
@@ -71,16 +72,18 @@ def _write_partition_image(image, image_info, device):
     boot_option = image_info.get('boot_option', 'netboot')
     boot_mode = image_info.get('deploy_boot_mode', 'bios')
     disk_label = image_info.get('disk_label', 'msdos')
-    image_mb = disk_utils.get_image_mb(image)
     root_mb = image_info['root_mb']
 
     cpu_arch = hardware.dispatch_to_managers('get_cpus').architecture
 
-    if image_mb > int(root_mb):
+    if image is not None:
-        msg = ('Root partition is too small for requested image. Image '
+        image_mb = disk_utils.get_image_mb(image)
-               'virtual size: {} MB, Root size: {} MB').format(image_mb,
+        if image_mb > int(root_mb):
-                                                               root_mb)
+            msg = ('Root partition is too small for requested image. Image '
-        raise errors.InvalidCommandParamsError(msg)
+                   'virtual size: {} MB, Root size: {} MB').format(image_mb,
+                                                                   root_mb)
+            raise errors.InvalidCommandParamsError(msg)
+
     try:
         return disk_utils.work_on_disk(device, root_mb,
                                        image_info['swap_mb'],
@@ -465,9 +468,16 @@ class StandbyExtension(base.BaseAgentExtension):
                 LOG.debug('Already had %s cached, overwriting',
                           self.cached_image_id)
 
-            if (stream_raw_images and disk_format == 'raw'
+            if stream_raw_images and disk_format == 'raw':
-                    and image_info.get('image_type') != 'partition'):
+                if image_info.get('image_type') == 'partition':
-                self._stream_raw_image_onto_device(image_info, device)
+                    self.partition_uuids = _write_partition_image(None,
+                                                                  image_info,
+                                                                  device)
+                    stream_to = self.partition_uuids['partitions']['root']
+                else:
+                    stream_to = device
+
+                self._stream_raw_image_onto_device(image_info, stream_to)
             else:
                 self._cache_and_write_image(image_info, device)
 

ironic_python_agent/tests/unit/extensions/test_standby.py

@@ -649,6 +649,7 @@ class TestStandbyExtension(base.IronicAgentTest):
 
     @mock.patch('ironic_lib.disk_utils.get_disk_identifier',
                 lambda dev: 'ROOT')
+    @mock.patch('ironic_lib.disk_utils.work_on_disk', autospec=True)
     @mock.patch('ironic_lib.disk_utils.create_config_drive_partition',
                 autospec=True)
     @mock.patch('ironic_python_agent.hardware.dispatch_to_managers',
@@ -659,9 +660,21 @@ class TestStandbyExtension(base.IronicAgentTest):
                 '._stream_raw_image_onto_device', autospec=True)
     def _test_prepare_image_raw(self, image_info, stream_mock,
                                 cache_write_mock, dispatch_mock,
-                                configdrive_copy_mock):
+                                configdrive_copy_mock, work_on_disk_mock,
-        dispatch_mock.return_value = '/dev/foo'
+                                partition=False):
+        # Calls get_cpus().architecture with partition images
+        dispatch_mock.side_effect = ['/dev/foo', self.fake_cpu]
         configdrive_copy_mock.return_value = None
+        work_on_disk_mock.return_value = {
+            'root uuid': 'a318821b-2a60-40e5-a011-7ac07fce342b',
+            'partitions': {
+                'root': '/dev/foo-part1',
+            }
+        }
+        if partition:
+            expected_device = '/dev/foo-part1'
+        else:
+            expected_device = '/dev/foo'
 
         async_result = self.agent_extension.prepare_image(
             image_info=image_info,
@@ -669,14 +682,15 @@ class TestStandbyExtension(base.IronicAgentTest):
         )
         async_result.join()
 
-        dispatch_mock.assert_called_once_with('get_os_install_device')
+        dispatch_mock.assert_any_call('get_os_install_device')
         self.assertFalse(configdrive_copy_mock.called)
 
         # Assert we've streamed the image or not
         if image_info['stream_raw_images']:
             stream_mock.assert_called_once_with(mock.ANY, image_info,
-                                                '/dev/foo')
+                                                expected_device)
             self.assertFalse(cache_write_mock.called)
+            self.assertIs(partition, work_on_disk_mock.called)
         else:
             cache_write_mock.assert_called_once_with(mock.ANY, image_info,
                                                      '/dev/foo')
@@ -694,6 +708,18 @@ class TestStandbyExtension(base.IronicAgentTest):
         image_info['stream_raw_images'] = False
         self._test_prepare_image_raw(image_info)
 
+    def test_prepare_partition_image_raw_stream_true(self):
+        image_info = _build_fake_partition_image_info()
+        image_info['disk_format'] = 'raw'
+        image_info['stream_raw_images'] = True
+        self._test_prepare_image_raw(image_info, partition=True)
+
+    def test_prepare_partition_image_raw_and_stream_false(self):
+        image_info = _build_fake_partition_image_info()
+        image_info['disk_format'] = 'raw'
+        image_info['stream_raw_images'] = False
+        self._test_prepare_image_raw(image_info, partition=True)
+
     @mock.patch('ironic_python_agent.utils.execute', autospec=True)
     def test_run_shutdown_command_invalid(self, execute_mock):
         self.assertRaises(errors.InvalidCommandParamsError,

lower-constraints.txt

@@ -24,7 +24,7 @@ greenlet==0.4.13
 hacking==1.0.0
 idna==2.6
 imagesize==1.0.0
-ironic-lib==2.14.0
+ironic-lib==2.16.0
 iso8601==0.1.11
 Jinja2==2.10
 keystoneauth1==3.4.0

releasenotes/notes/streaming-partition-images-cdeb260ef8f90012.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Adds support for streaming raw partition images onto target partition
+    without caching them in memory.

requirements.txt

@@ -21,4 +21,4 @@ rtslib-fb>=2.1.65 # Apache-2.0
 six>=1.10.0 # MIT
 stevedore>=1.20.0 # Apache-2.0
 WSME>=0.8.0 # MIT
-ironic-lib>=2.14.0 # Apache-2.0
+ironic-lib>=2.16.0 # Apache-2.0