New deploy step for injecting arbitrary files

This change adds a deploy step inject_files that adds a flexible
way to inject files into the instance.

Change-Id: I0e70a2cbc13744195c9493a48662e465ec010dbe
Story: #2008611
Task: #41794

doc/source/admin/hardware_managers.rst

@@ -25,6 +25,52 @@ Deploy steps
     and must be used through the :ironic-doc:`ironic RAID feature
     <admin/raid.html>`.
 
+Injecting files
+~~~~~~~~~~~~~~~
+
+``deploy.inject_files(node, ports, files, verify_ca=True)``
+
+This optional deploy step (introduced in the Wallaby release series) allows
+injecting arbitrary files into the node. The list of files is built from the
+optional ``inject_files`` property of the node concatenated with the explicit
+``files`` argument. Each item in the list is a dictionary with the following
+fields:
+
+``path`` (required)
+    An absolute path to the file on the target partition. All missing
+    directories will be created.
+``partition``
+    Specifies the target partition in one of 3 ways:
+
+    * A number is treated as a partition index (starting with 1) on the root
+      device.
+    * A path is treated as a block device path (e.g. ``/dev/sda1`` or
+      ``/dev/disk/by-partlabel/<something>``.
+    * If missing, the agent will try to find a partition containing the first
+      component of the ``path`` on the root device. E.g. for
+      ``/etc/sysctl.d/my.conf``, look for a partition containing ``/etc``.
+``deleted``
+    If ``True``, the file is deleted, not created.
+    Incompatible with ``content``.
+``content``
+    Data to write. Incompatible with ``deleted``. Can take two forms:
+
+    * A URL of the content. Can use Python-style formatting to build a node
+      specific URL, e.g. ``http://server/{node[uuid]}/{ports[0][address]}``.
+    * Base64 encoded binary contents.
+``mode``, ``owner``, ``group``
+    Numeric mode, owner ID and group ID of the file.
+``dirmode``
+    Numeric mode of the leaf directory if it has to be created.
+
+This deploy step is disabled by default and can be enabled via a deploy
+template or via the ``ipa-inject-files-priority`` kernel parameter.
+
+Known limitations:
+
+* Names are not supported for ``owner`` and ``group``.
+* LVM is not supported.
+
 Clean steps
 -----------
 

ironic_python_agent/config.py

@@ -316,6 +316,11 @@ cli_opts = [
                      'used if you are absolutely sure of what you are '
                      'doing and that your hardware supports '
                      'such functionality. Hint: Most hardware does not.'),
+    cfg.IntOpt('inject_files_priority',
+               default=APARAMS.get('ipa-inject-files-priority', 0),
+               min=0, max=99,  # 100 is when IPA is booted
+               help='Priority of the inject_files deploy step (disabled '
+                    'by default), an integer between 1 and .'),
 ]
 
 CONF.register_cli_opts(cli_opts)

ironic_python_agent/hardware.py

@@ -39,6 +39,7 @@ import yaml
 from ironic_python_agent import encoding
 from ironic_python_agent import errors
 from ironic_python_agent.extensions import base as ext_base
+from ironic_python_agent import inject_files
 from ironic_python_agent import netutils
 from ironic_python_agent import raid_utils
 from ironic_python_agent import tls_utils
@@ -1794,6 +1795,13 @@ class GenericHardwareManager(HardwareManager):
                 'interface': 'deploy',
                 'reboot_requested': False,
             },
+            {
+                'step': 'inject_files',
+                'priority': CONF.inject_files_priority,
+                'interface': 'deploy',
+                'reboot_requested': False,
+                'argsinfo': inject_files.ARGSINFO,
+            },
         ]
 
     def apply_configuration(self, node, ports, raid_config,
@@ -2266,6 +2274,17 @@ class GenericHardwareManager(HardwareManager):
         """Generate a TLS certificate for the IP address."""
         return tls_utils.generate_tls_certificate(ip_address)
 
+    def inject_files(self, node, ports, files=None, verify_ca=True):
+        """A deploy step to inject arbitrary files.
+
+        :param node: A dictionary of the node object
+        :param ports: A list of dictionaries containing information
+                      of ports for the node (unused)
+        :param files: See :py:mod:`inject_files`
+        :param verify_ca: Whether to verify TLS certificate.
+        """
+        return inject_files.inject_files(node, ports, files, verify_ca)
+
 
 def _compare_extensions(ext1, ext2):
     mgr1 = ext1.obj

ironic_python_agent/inject_files.py

@@ -0,0 +1,256 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Implementation of the inject_files deploy step."""
+
+import base64
+import contextlib
+import os
+
+from ironic_lib import disk_utils
+from ironic_lib import utils as ironic_utils
+from oslo_concurrency import processutils
+from oslo_config import cfg
+from oslo_log import log
+
+from ironic_python_agent import errors
+from ironic_python_agent import hardware
+from ironic_python_agent import utils
+
+
+CONF = cfg.CONF
+LOG = log.getLogger(__name__)
+
+
+ARGSINFO = {
+    "files": {
+        "description": (
+            "Files to inject, a list of file structures with keys: 'path' "
+            "(path to the file), 'partition' (partition specifier), "
+            "'content' (base64 encoded string), 'mode' (new file mode) and "
+            "'dirmode' (mode for the leaf directory, if created). "
+            "Merged with the values from node.properties[inject_files]."
+        ),
+        "required": False,
+    },
+    "verify_ca": {
+        "description": (
+            "Whether to verify TLS certificates. Global agent options "
+            "are used by default."
+        ),
+        "required": False,
+    }
+}
+
+
+def inject_files(node, ports, files, verify_ca=True):
+    """A deploy step to inject arbitrary files.
+
+    :param node: A dictionary of the node object
+    :param ports: A list of dictionaries containing information
+                  of ports for the node
+    :param files: See ARGSINFO.
+    :param verify_ca: Whether to verify TLS certificate.
+    :raises: InvalidCommandParamsError
+    """
+    files = _validate_files(
+        node['properties'].get('inject_files') or [],
+        files or [])
+    if not files:
+        LOG.info('No files to inject')
+        return
+
+    http_get = utils.StreamingClient(verify_ca)
+    root_dev = hardware.dispatch_to_managers('get_os_install_device')
+
+    for fl in files:
+        _inject_one(node, ports, fl, root_dev, http_get)
+
+
+def _inject_one(node, ports, fl, root_dev, http_get):
+    """Inject one file.
+
+    :param node: A dictionary of the node object
+    :param ports: A list of dictionaries containing information
+                  of ports for the node
+    :param fl: File information.
+    :param root_dev: Root device used for the current node.
+    :param http_get: Context manager to get HTTP URLs.
+    """
+    with _find_and_mount_path(fl['path'], fl.get('partition'),
+                              root_dev) as path:
+        if fl.get('deleted'):
+            ironic_utils.unlink_without_raise(path)
+            return
+
+        try:
+            dirpath = os.path.dirname(path)
+            try:
+                os.makedirs(dirpath)
+            except FileExistsError:
+                pass
+            else:
+                # Use chmod here and below to avoid relying on umask
+                if fl.get('dirmode'):
+                    os.chmod(dirpath, fl['dirmode'])
+
+            content = fl['content']
+            with open(path, 'wb') as fp:
+                if '://' in content:
+                    # Allow node-specific URLs to be used in a deploy template
+                    url = content.format(node=node, ports=ports)
+                    with http_get(url) as resp:
+                        for chunk in resp:
+                            fp.write(chunk)
+                else:
+                    fp.write(base64.b64decode(content))
+
+            if fl.get('mode'):
+                os.chmod(path, fl['mode'])
+
+            if fl.get('owner') is not None or fl.get('group') is not None:
+                # -1 means do not change
+                os.chown(path, fl.get('owner', -1), fl.get('group', -1))
+        except Exception as exc:
+            LOG.exception('Failed to process file %s', fl)
+            raise errors.CommandExecutionError(
+                'Failed to process file %s. %s: %s'
+                % (fl, type(exc).__class__, exc))
+
+
+@contextlib.contextmanager
+def _find_and_mount_path(path, partition, root_dev):
+    """Find the specified path on a device.
+
+    Tries to find the suitable device for the file based on the ``path`` and
+    ``partition``, mount the device and provides the actual full path.
+
+    :param path: Path to the file to find.
+    :param partition: Device to find the file on or None.
+    :param root_dev: Root device from the hardware manager.
+    :return: Context manager that yields the full path to the file.
+    """
+    path = os.path.normpath(path.strip('/'))  # to make path joining work
+    if partition:
+        try:
+            part_num = int(partition)
+        except ValueError:
+            with ironic_utils.mounted(partition) as part_path:
+                yield os.path.join(part_path, path)
+        else:
+            # TODO(dtantsur): switch to ironic-lib instead:
+            # https://review.opendev.org/c/openstack/ironic-lib/+/774502
+            part_template = '%s%s'
+            if 'nvme' in root_dev:
+                part_template = '%sp%s'
+            part_dev = part_template % (root_dev, part_num)
+
+            with ironic_utils.mounted(part_dev) as part_path:
+                yield os.path.join(part_path, path)
+    else:
+        try:
+            # This turns e.g. etc/sysctl.d/my.conf into etc + sysctl.d/my.conf
+            detect_dir, rest_dir = path.split('/', 1)
+        except ValueError:
+            # Validation ensures that files in / have "partition" present,
+            # checking here just in case.
+            raise errors.InvalidCommandParamsError(
+                "Invalid path %s, must be an absolute path to a file" % path)
+
+        with find_partition_with_path(detect_dir, root_dev) as part_path:
+            yield os.path.join(part_path, rest_dir)
+
+
+@contextlib.contextmanager
+def find_partition_with_path(path, device=None):
+    """Find a partition with the given path.
+
+    :param path: Expected path.
+    :param device: Target device. If None, the root device is used.
+    :returns: A context manager that will unmount and delete the temporary
+        mount point on exit.
+    """
+    if device is None:
+        device = hardware.dispatch_to_managers('get_os_install_device')
+    partitions = disk_utils.list_partitions(device)
+    # Make os.path.join work as expected
+    lookup_path = path.lstrip('/')
+
+    for part in partitions:
+        if 'lvm' in part['flags']:
+            LOG.debug('Skipping LVM partition %s', part)
+            continue
+
+        # TODO(dtantsur): switch to ironic-lib instead:
+        # https://review.opendev.org/c/openstack/ironic-lib/+/774502
+        part_template = '%s%s'
+        if 'nvme' in device:
+            part_template = '%sp%s'
+        part_path = part_template % (device, part['number'])
+
+        LOG.debug('Inspecting partition %s for path %s', part, path)
+        try:
+            with ironic_utils.mounted(part_path) as local_path:
+                found_path = os.path.join(local_path, lookup_path)
+                if not os.path.isdir(found_path):
+                    continue
+
+                LOG.info('Path %s has been found on partition %s', path, part)
+                yield found_path
+                return
+        except processutils.ProcessExecutionError as exc:
+            LOG.warning('Failure when inspecting partition %s: %s', part, exc)
+
+    raise errors.DeviceNotFound("No partition found with path %s, scanned: %s"
+                                % (path, partitions))
+
+
+def _validate_files(from_properties, from_args):
+    """Sanity check for files."""
+    if not isinstance(from_properties, list):
+        raise errors.InvalidCommandParamsError(
+            "The `inject_files` node property must be a list, got %s"
+            % type(from_properties).__name__)
+    if not isinstance(from_args, list):
+        raise errors.InvalidCommandParamsError(
+            "The `files` argument must be a list, got %s"
+            % type(from_args).__name__)
+
+    files = from_properties + from_args
+    failures = []
+
+    for fl in files:
+        unknown = set(fl) - {'path', 'partition', 'content', 'deleted', 'mode',
+                             'dirmode', 'owner', 'group'}
+        if unknown:
+            failures.append('unexpected fields in %s: %s'
+                            % (fl, ', '.join(unknown)))
+
+        if not fl.get('path'):
+            failures.append('expected a path in %s' % fl)
+        elif os.path.dirname(fl['path']) == '/' and not fl.get('partition'):
+            failures.append('%s in root directory requires "partition"' % fl)
+        elif fl['path'].endswith('/'):
+            failures.append('directories not supported for %s' % fl)
+
+        if fl.get('content') and fl.get('deleted'):
+            failures.append('content cannot be used with deleted in %s' % fl)
+
+        for field in ('owner', 'group', 'mode', 'dirmode'):
+            if field in fl and type(fl[field]) is not int:
+                failures.append('%s must be a number in %s' % (field, fl))
+
+    if failures:
+        raise errors.InvalidCommandParamsError(
+            "Validation of files failed: %s" % '; '.join(failures))
+
+    return files

ironic_python_agent/tests/unit/test_inject_files.py

@@ -0,0 +1,421 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import shutil
+import stat
+import tempfile
+from unittest import mock
+
+from ironic_python_agent import errors
+from ironic_python_agent import inject_files
+from ironic_python_agent.tests.unit import base
+
+
+@mock.patch('ironic_lib.utils.mounted', autospec=True)
+@mock.patch('ironic_lib.disk_utils.list_partitions', autospec=True)
+@mock.patch('ironic_python_agent.hardware.dispatch_to_managers',
+            lambda _call: '/dev/fake')
+class TestFindPartitionWithPath(base.IronicAgentTest):
+
+    def setUp(self):
+        super().setUp()
+        self.tempdir = tempfile.mkdtemp()
+        self.addCleanup(lambda: shutil.rmtree(self.tempdir))
+
+    def test_found(self, mock_list_parts, mock_mount):
+        mock_list_parts.return_value = [
+            {'number': 1, 'flags': 'lvm'},
+            {'number': 2, 'flags': 'boot'},
+        ]
+        mock_mount.return_value.__enter__.return_value = self.tempdir
+        expected = os.path.join(self.tempdir, "some/path")
+        os.makedirs(expected)
+
+        with inject_files.find_partition_with_path("/some/path") as path:
+            self.assertEqual(expected, path)
+
+        mock_mount.assert_called_once_with('/dev/fake2')
+
+    def test_found_with_dev(self, mock_list_parts, mock_mount):
+        mock_list_parts.return_value = [
+            {'number': 1, 'flags': 'lvm'},
+            {'number': 2, 'flags': 'boot'},
+        ]
+        mock_mount.return_value.__enter__.return_value = self.tempdir
+        expected = os.path.join(self.tempdir, "some/path")
+        os.makedirs(expected)
+
+        with inject_files.find_partition_with_path("/some/path",
+                                                   "/dev/nvme0n1") as path:
+            self.assertEqual(expected, path)
+
+        mock_mount.assert_called_once_with('/dev/nvme0n1p2')
+
+    def test_not_found(self, mock_list_parts, mock_mount):
+        mock_list_parts.return_value = [
+            {'number': 1, 'flags': 'lvm'},
+            {'number': 2, 'flags': 'boot'},
+            {'number': 3, 'flags': ''},
+        ]
+        mock_mount.return_value.__enter__.return_value = self.tempdir
+
+        self.assertRaises(
+            errors.DeviceNotFound,
+            inject_files.find_partition_with_path("/some/path").__enter__)
+
+        mock_mount.assert_has_calls([
+            mock.call('/dev/fake2'),
+            mock.call('/dev/fake3'),
+        ], any_order=True)
+
+
+class TestFindAndMountPath(base.IronicAgentTest):
+
+    @mock.patch.object(inject_files, 'find_partition_with_path', autospec=True)
+    def test_without_on(self, mock_find_part):
+        mock_find_part.return_value.__enter__.return_value = '/mount/path'
+        with inject_files._find_and_mount_path('/etc/sysctl.d/my.conf',
+                                               None, '/dev/fake') as result:
+            # "etc" is included in a real result of find_partition_with_path
+            self.assertEqual('/mount/path/sysctl.d/my.conf', result)
+        mock_find_part.assert_called_once_with('etc', '/dev/fake')
+
+    def test_without_on_wrong_path(self):
+        self.assertRaises(
+            errors.InvalidCommandParamsError,
+            inject_files._find_and_mount_path('/etc', None,
+                                              '/dev/fake').__enter__)
+
+    @mock.patch('ironic_lib.utils.mounted', autospec=True)
+    def test_with_on_as_path(self, mock_mount):
+        mock_mount.return_value.__enter__.return_value = '/mount/path'
+        with inject_files._find_and_mount_path('/etc/sysctl.d/my.conf',
+                                               '/dev/on',
+                                               '/dev/fake') as result:
+            self.assertEqual('/mount/path/etc/sysctl.d/my.conf', result)
+        mock_mount.assert_called_once_with('/dev/on')
+
+    @mock.patch('ironic_lib.utils.mounted', autospec=True)
+    def test_with_on_as_number(self, mock_mount):
+        mock_mount.return_value.__enter__.return_value = '/mount/path'
+        with inject_files._find_and_mount_path('/etc/sysctl.d/my.conf',
+                                               2, '/dev/fake') as result:
+            self.assertEqual('/mount/path/etc/sysctl.d/my.conf', result)
+        mock_mount.assert_called_once_with('/dev/fake2')
+
+    @mock.patch('ironic_lib.utils.mounted', autospec=True)
+    def test_with_on_as_number_nvme(self, mock_mount):
+        mock_mount.return_value.__enter__.return_value = '/mount/path'
+        with inject_files._find_and_mount_path('/etc/sysctl.d/my.conf',
+                                               2, '/dev/nvme0n1') as result:
+            self.assertEqual('/mount/path/etc/sysctl.d/my.conf', result)
+        mock_mount.assert_called_once_with('/dev/nvme0n1p2')
+
+
+@mock.patch.object(inject_files, '_find_and_mount_path', autospec=True)
+class TestInjectOne(base.IronicAgentTest):
+
+    def setUp(self):
+        super().setUp()
+        self.tempdir = tempfile.mkdtemp()
+        self.addCleanup(lambda: shutil.rmtree(self.tempdir))
+        self.dirpath = os.path.join(self.tempdir, 'dir1', 'dir2')
+        self.path = os.path.join(self.dirpath, 'file.name')
+
+        self.http_get = mock.MagicMock()
+        self.http_get.return_value.__enter__.return_value = iter(
+            [b'con', b'tent', b''])
+
+        self.node = {'uuid': '1234'}
+        self.ports = [{'address': 'aabb'}]
+
+    def test_delete(self, mock_find_and_mount):
+        mock_find_and_mount.return_value.__enter__.return_value = self.path
+
+        fl = {'path': '/etc/dir1/dir2/file.name', 'deleted': True}
+        os.makedirs(self.dirpath)
+        with open(self.path, 'wb') as fp:
+            fp.write(b'content')
+
+        inject_files._inject_one(self.node, self.ports, fl,
+                                 '/dev/root', self.http_get)
+
+        self.assertFalse(os.path.exists(self.path))
+        self.assertTrue(os.path.isdir(self.dirpath))
+        mock_find_and_mount.assert_called_once_with(fl['path'], None,
+                                                    '/dev/root')
+        self.http_get.assert_not_called()
+
+    def test_delete_not_exists(self, mock_find_and_mount):
+        mock_find_and_mount.return_value.__enter__.return_value = self.path
+
+        fl = {'path': '/etc/dir1/dir2/file.name', 'deleted': True}
+        inject_files._inject_one(self.node, self.ports, fl,
+                                 '/dev/root', self.http_get)
+
+        self.assertFalse(os.path.exists(self.path))
+        self.assertFalse(os.path.isdir(self.dirpath))
+        mock_find_and_mount.assert_called_once_with(fl['path'], None,
+                                                    '/dev/root')
+        self.http_get.assert_not_called()
+
+    def test_plain_content(self, mock_find_and_mount):
+        mock_find_and_mount.return_value.__enter__.return_value = self.path
+
+        fl = {'path': '/etc/dir1/dir2/file.name', 'content': 'Y29udGVudA=='}
+        inject_files._inject_one(self.node, self.ports, fl,
+                                 '/dev/root', self.http_get)
+
+        with open(self.path, 'rb') as fp:
+            self.assertEqual(b'content', fp.read())
+        mock_find_and_mount.assert_called_once_with(fl['path'], None,
+                                                    '/dev/root')
+        self.http_get.assert_not_called()
+
+    def test_plain_content_with_on(self, mock_find_and_mount):
+        mock_find_and_mount.return_value.__enter__.return_value = self.path
+
+        fl = {'path': '/etc/dir1/dir2/file.name', 'content': 'Y29udGVudA==',
+              'partition': '/dev/sda1'}
+        inject_files._inject_one(self.node, self.ports, fl,
+                                 '/dev/root', self.http_get)
+
+        with open(self.path, 'rb') as fp:
+            self.assertEqual(b'content', fp.read())
+        mock_find_and_mount.assert_called_once_with(fl['path'], '/dev/sda1',
+                                                    '/dev/root')
+        self.http_get.assert_not_called()
+
+    def test_plain_content_with_modes(self, mock_find_and_mount):
+        mock_find_and_mount.return_value.__enter__.return_value = self.path
+
+        fl = {'path': '/etc/dir1/dir2/file.name', 'content': 'Y29udGVudA==',
+              'mode': 0o602, 'dirmode': 0o703}
+        inject_files._inject_one(self.node, self.ports, fl,
+                                 '/dev/root', self.http_get)
+
+        with open(self.path, 'rb') as fp:
+            self.assertEqual(b'content', fp.read())
+        self.assertEqual(0o602, stat.S_IMODE(os.stat(self.path).st_mode))
+        self.assertEqual(0o703, stat.S_IMODE(os.stat(self.dirpath).st_mode))
+
+        mock_find_and_mount.assert_called_once_with(fl['path'], None,
+                                                    '/dev/root')
+        self.http_get.assert_not_called()
+
+    def test_plain_content_with_modes_exists(self, mock_find_and_mount):
+        mock_find_and_mount.return_value.__enter__.return_value = self.path
+
+        fl = {'path': '/etc/dir1/dir2/file.name', 'content': 'Y29udGVudA==',
+              'mode': 0o602, 'dirmode': 0o703}
+        os.makedirs(self.dirpath)
+        with open(self.path, 'wb') as fp:
+            fp.write(b"I'm not a cat, I'm a lawyer")
+
+        inject_files._inject_one(self.node, self.ports, fl,
+                                 '/dev/root', self.http_get)
+
+        with open(self.path, 'rb') as fp:
+            self.assertEqual(b'content', fp.read())
+        self.assertEqual(0o602, stat.S_IMODE(os.stat(self.path).st_mode))
+        # Exising directories do not change their permissions
+        self.assertNotEqual(0o703, stat.S_IMODE(os.stat(self.dirpath).st_mode))
+
+        mock_find_and_mount.assert_called_once_with(fl['path'], None,
+                                                    '/dev/root')
+        self.http_get.assert_not_called()
+
+    @mock.patch.object(os, 'chown', autospec=True)
+    def test_plain_content_with_owner(self, mock_chown, mock_find_and_mount):
+        mock_find_and_mount.return_value.__enter__.return_value = self.path
+
+        fl = {'path': '/etc/dir1/dir2/file.name', 'content': 'Y29udGVudA==',
+              'owner': 42}
+        inject_files._inject_one(self.node, self.ports, fl,
+                                 '/dev/root', self.http_get)
+
+        with open(self.path, 'rb') as fp:
+            self.assertEqual(b'content', fp.read())
+
+        mock_find_and_mount.assert_called_once_with(fl['path'], None,
+                                                    '/dev/root')
+        mock_chown.assert_called_once_with(self.path, 42, -1)
+        self.http_get.assert_not_called()
+
+    @mock.patch.object(os, 'chown', autospec=True)
+    def test_plain_content_with_owner_and_group(self, mock_chown,
+                                                mock_find_and_mount):
+        mock_find_and_mount.return_value.__enter__.return_value = self.path
+
+        fl = {'path': '/etc/dir1/dir2/file.name', 'content': 'Y29udGVudA==',
+              'owner': 0, 'group': 0}
+        inject_files._inject_one(self.node, self.ports, fl,
+                                 '/dev/root', self.http_get)
+
+        with open(self.path, 'rb') as fp:
+            self.assertEqual(b'content', fp.read())
+
+        mock_find_and_mount.assert_called_once_with(fl['path'], None,
+                                                    '/dev/root')
+        mock_chown.assert_called_once_with(self.path, 0, 0)
+        self.http_get.assert_not_called()
+
+    def test_url(self, mock_find_and_mount):
+        mock_find_and_mount.return_value.__enter__.return_value = self.path
+
+        fl = {'path': '/etc/dir1/dir2/file.name',
+              'content': 'http://example.com/path'}
+        inject_files._inject_one(self.node, self.ports, fl,
+                                 '/dev/root', self.http_get)
+
+        with open(self.path, 'rb') as fp:
+            self.assertEqual(b'content', fp.read())
+
+        mock_find_and_mount.assert_called_once_with(fl['path'], None,
+                                                    '/dev/root')
+        self.http_get.assert_called_once_with('http://example.com/path')
+
+    def test_url_formatting(self, mock_find_and_mount):
+        mock_find_and_mount.return_value.__enter__.return_value = self.path
+
+        fl = {'path': '/etc/dir1/dir2/file.name',
+              'content': 'http://example.com/{node[uuid]}/{ports[0][address]}'}
+        inject_files._inject_one(self.node, self.ports, fl,
+                                 '/dev/root', self.http_get)
+
+        with open(self.path, 'rb') as fp:
+            self.assertEqual(b'content', fp.read())
+
+        mock_find_and_mount.assert_called_once_with(fl['path'], None,
+                                                    '/dev/root')
+        self.http_get.assert_called_once_with('http://example.com/1234/aabb')
+
+
+@mock.patch('ironic_python_agent.hardware.dispatch_to_managers',
+            lambda _call: '/dev/root')
+@mock.patch.object(inject_files, '_inject_one', autospec=True)
+class TestInjectFiles(base.IronicAgentTest):
+
+    def test_empty(self, mock_inject):
+        node = {
+            'properties': {}
+        }
+
+        inject_files.inject_files(node, [mock.sentinel.port], [])
+        mock_inject.assert_not_called()
+
+    def test_ok(self, mock_inject):
+        node = {
+            'properties': {
+                'inject_files': [
+                    {'path': '/etc/default/grub', 'content': 'abcdef'},
+                    {'path': '/etc/default/bluetooth', 'deleted': True},
+                ]
+            }
+        }
+        files = [
+            {'path': '/boot/special.conf',
+             'content': 'http://example.com/data',
+             'mode': 0o600, 'dirmode': 0o750, 'owner': 0, 'group': 0},
+            {'path': 'service.conf', 'partition': '/dev/disk/by-label/OPT'},
+        ]
+
+        inject_files.inject_files(node, [mock.sentinel.port], files)
+
+        mock_inject.assert_has_calls([
+            mock.call(node, [mock.sentinel.port], fl, '/dev/root', mock.ANY)
+            for fl in node['properties']['inject_files'] + files
+        ])
+        http_get = mock_inject.call_args_list[0][0][4]
+        self.assertTrue(http_get.verify)
+        self.assertIsNone(http_get.cert)
+
+    def test_verify_false(self, mock_inject):
+        node = {
+            'properties': {
+                'inject_files': [
+                    {'path': '/etc/default/grub', 'content': 'abcdef'},
+                    {'path': '/etc/default/bluetooth', 'deleted': True},
+                ]
+            }
+        }
+        files = [
+            {'path': '/boot/special.conf',
+             'content': 'http://example.com/data',
+             'mode': 0o600, 'dirmode': 0o750, 'owner': 0, 'group': 0},
+            {'path': 'service.conf', 'partition': '/dev/disk/by-label/OPT'},
+        ]
+
+        inject_files.inject_files(node, [mock.sentinel.port], files, False)
+
+        mock_inject.assert_has_calls([
+            mock.call(node, [mock.sentinel.port], fl, '/dev/root', mock.ANY)
+            for fl in node['properties']['inject_files'] + files
+        ])
+        http_get = mock_inject.call_args_list[0][0][4]
+        self.assertFalse(http_get.verify)
+        self.assertIsNone(http_get.cert)
+
+    def test_invalid_type_on_node(self, mock_inject):
+        node = {
+            'properties': {
+                'inject_files': 42
+            }
+        }
+        self.assertRaises(errors.InvalidCommandParamsError,
+                          inject_files.inject_files, node, [], [])
+        mock_inject.assert_not_called()
+
+    def test_invalid_type_in_param(self, mock_inject):
+        node = {
+            'properties': {}
+        }
+        self.assertRaises(errors.InvalidCommandParamsError,
+                          inject_files.inject_files, node, [], 42)
+        mock_inject.assert_not_called()
+
+
+class TestValidateFiles(base.IronicAgentTest):
+
+    def test_missing_path(self):
+        fl = {'deleted': True}
+        self.assertRaisesRegex(errors.InvalidCommandParamsError, 'path',
+                               inject_files._validate_files, [fl], [])
+
+    def test_unknown_fields(self):
+        fl = {'path': '/etc/passwd', 'cat': 'meow'}
+        self.assertRaisesRegex(errors.InvalidCommandParamsError, 'cat',
+                               inject_files._validate_files, [fl], [])
+
+    def test_root_without_on(self):
+        fl = {'path': '/something', 'content': 'abcd'}
+        self.assertRaisesRegex(errors.InvalidCommandParamsError, 'partition',
+                               inject_files._validate_files, [fl], [])
+
+    def test_no_directories(self):
+        fl = {'path': '/something/else/', 'content': 'abcd'}
+        self.assertRaisesRegex(errors.InvalidCommandParamsError, 'directories',
+                               inject_files._validate_files, [fl], [])
+
+    def test_content_and_deleted(self):
+        fl = {'path': '/etc/password', 'content': 'abcd', 'deleted': True}
+        self.assertRaisesRegex(errors.InvalidCommandParamsError,
+                               'content .* with deleted',
+                               inject_files._validate_files, [fl], [])
+
+    def test_numeric_fields(self):
+        for field in ('owner', 'group', 'mode', 'dirmode'):
+            fl = {'path': '/etc/password', 'content': 'abcd', field: 'name'}
+            self.assertRaisesRegex(errors.InvalidCommandParamsError,
+                                   'must be a number',
+                                   inject_files._validate_files, [fl], [])

ironic_python_agent/tests/unit/test_utils.py

@@ -27,6 +27,7 @@ from ironic_lib import disk_utils
 from ironic_lib import utils as ironic_utils
 from oslo_concurrency import processutils
 from oslo_serialization import base64
+import requests
 import testtools
 
 from ironic_python_agent import errors
@@ -1132,3 +1133,33 @@ class TestCopyConfigFromVmedia(testtools.TestCase):
             mock.call(mock.ANY, '/etc/ironic-python-agent/ironic.crt'),
             mock.call(mock.ANY, '/etc/ironic-python-agent.d/ironic.conf'),
         ], any_order=True)
+
+
+@mock.patch.object(requests, 'get', autospec=True)
+class TestStreamingClient(ironic_agent_base.IronicAgentTest):
+
+    def test_ok(self, mock_get):
+        client = utils.StreamingClient()
+        self.assertTrue(client.verify)
+        self.assertIsNone(client.cert)
+
+        with client("http://url") as result:
+            response = mock_get.return_value.__enter__.return_value
+            self.assertIs(result, response.iter_content.return_value)
+
+        mock_get.assert_called_once_with("http://url", verify=True, cert=None,
+                                         stream=True, timeout=60)
+        response.iter_content.assert_called_once_with(1024 * 1024)
+
+    def test_retries(self, mock_get):
+        self.config(image_download_connection_retries=1,
+                    image_download_connection_retry_interval=1)
+        mock_get.side_effect = requests.ConnectionError
+
+        client = utils.StreamingClient()
+        self.assertRaises(errors.CommandExecutionError,
+                          client("http://url").__enter__)
+
+        mock_get.assert_called_with("http://url", verify=True, cert=None,
+                                    stream=True, timeout=60)
+        self.assertEqual(2, mock_get.call_count)

ironic_python_agent/utils.py

@@ -35,6 +35,8 @@ from oslo_log import log as logging
 from oslo_serialization import base64
 from oslo_serialization import jsonutils
 from oslo_utils import units
+import requests
+import tenacity
 
 from ironic_python_agent import errors
 
@@ -811,3 +813,42 @@ def create_partition_table(dev_name, partition_table_type):
         msg = "Failed to create partition table on {}: {}".format(
             dev_name, e)
         raise errors.CommandExecutionError(msg)
+
+
+class StreamingClient:
+    """A wrapper around HTTP client with TLS, streaming and error handling."""
+
+    _CHUNK_SIZE = 1 * units.Mi
+
+    def __init__(self, verify_ca=True):
+        if verify_ca:
+            self.verify, self.cert = get_ssl_client_options(CONF)
+        else:
+            self.verify, self.cert = False, None
+
+    @contextlib.contextmanager
+    def __call__(self, url):
+        """Execute a GET request and start streaming.
+
+        :param url: Target URL.
+        :return: A generator yielding chunks of data.
+        """
+        @tenacity.retry(
+            retry=tenacity.retry_if_exception_type(requests.ConnectionError),
+            stop=tenacity.stop_after_attempt(
+                CONF.image_download_connection_retries + 1),
+            wait=tenacity.wait_fixed(
+                CONF.image_download_connection_retry_interval),
+            reraise=True)
+        def _get_with_retries():
+            return requests.get(url, verify=self.verify, cert=self.cert,
+                                stream=True,
+                                timeout=CONF.image_download_connection_timeout)
+
+        try:
+            with _get_with_retries() as resp:
+                resp.raise_for_status()
+                yield resp.iter_content(self._CHUNK_SIZE)
+        except requests.RequestException as exc:
+            raise errors.CommandExecutionError(
+                "Unable to read data from %s: %s" % (url, exc))

releasenotes/notes/inject-files-b411369ce6856dac.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    Adds a new deploy step ``deploy.inject_files`` to inject arbitrary files
+    into the instance. See `the hardware managers documentation
+    <https://docs.openstack.org/ironic-python-agent/latest/admin/hardware_managers.html>`_
+    for details.

requirements.txt

@@ -17,6 +17,6 @@ requests>=2.14.2 # Apache-2.0
 rtslib-fb>=2.1.65 # Apache-2.0
 stevedore>=1.20.0 # Apache-2.0
 tenacity>=6.2.0 # Apache-2.0
-ironic-lib>=4.1.0 # Apache-2.0
+ironic-lib>=4.5.0 # Apache-2.0
 Werkzeug>=1.0.1 # BSD License
 cryptography>=2.3 # BSD/Apache-2.0