Add configurable delays to the fake drivers

Simulating workloads with the fake driver currently misses the reality
that some operations take time to complete, rather than occuring
instantly. This makes it difficult to mock real workloads for
performance and functional testing of ironic itself.

This change adds configurable random wait times for fake drivers in a
new ironic.conf [fake] section. Each supported driver having one
configuration option controlling the delay. These delays are applied
to operations which typically block in other drivers.

The default value of zero continues the existing behaviour of no
delay. A single integer value will result in a constant delay in
seconds. Two values separated by a comma will result in a triangular
distribution weighted by the first value, specifically in python[1]:

    random.triangular(a, b, a)

Change-Id: I7cb1b50d035939e6c4538b3373002a309bfedea4
[1] https://docs.python.org/3/library/random.html#random.triangular

ironic/conf/__init__.py

@@ -29,6 +29,7 @@ from ironic.conf import deploy
 from ironic.conf import dhcp
 from ironic.conf import dnsmasq
 from ironic.conf import drac
+from ironic.conf import fake
 from ironic.conf import glance
 from ironic.conf import healthcheck
 from ironic.conf import ibmc
@@ -64,6 +65,7 @@ deploy.register_opts(CONF)
 drac.register_opts(CONF)
 dhcp.register_opts(CONF)
 dnsmasq.register_opts(CONF)
+fake.register_opts(CONF)
 glance.register_opts(CONF)
 healthcheck.register_opts(CONF)
 ibmc.register_opts(CONF)

ironic/conf/fake.py

@@ -0,0 +1,85 @@
+#
+# Copyright 2022 Red Hat, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_config import cfg
+
+from ironic.common.i18n import _
+
+opts = [
+    cfg.StrOpt('power_delay',
+               default='0',
+               help=_('Delay in seconds for operations with the fake '
+                      'power driver. Two comma-delimited values will '
+                      'result in a delay with a triangular random '
+                      'distribution, weighted on the first value.')),
+    cfg.StrOpt('boot_delay',
+               default='0',
+               help=_('Delay in seconds for operations with the fake '
+                      'boot driver. Two comma-delimited values will '
+                      'result in a delay with a triangular random '
+                      'distribution, weighted on the first value.')),
+    cfg.StrOpt('deploy_delay',
+               default='0',
+               help=_('Delay in seconds for operations with the fake '
+                      'deploy driver. Two comma-delimited values will '
+                      'result in a delay with a triangular random '
+                      'distribution, weighted on the first value.')),
+    cfg.StrOpt('vendor_delay',
+               default='0',
+               help=_('Delay in seconds for operations with the fake '
+                      'vendor driver. Two comma-delimited values will '
+                      'result in a delay with a triangular random '
+                      'distribution, weighted on the first value.')),
+    cfg.StrOpt('management_delay',
+               default='0',
+               help=_('Delay in seconds for operations with the fake '
+                      'management driver. Two comma-delimited values will '
+                      'result in a delay with a triangular random '
+                      'distribution, weighted on the first value.')),
+    cfg.StrOpt('inspect_delay',
+               default='0',
+               help=_('Delay in seconds for operations with the fake '
+                      'inspect driver. Two comma-delimited values will '
+                      'result in a delay with a triangular random '
+                      'distribution, weighted on the first value.')),
+    cfg.StrOpt('raid_delay',
+               default='0',
+               help=_('Delay in seconds for operations with the fake '
+                      'raid driver. Two comma-delimited values will '
+                      'result in a delay with a triangular random '
+                      'distribution, weighted on the first value.')),
+    cfg.StrOpt('bios_delay',
+               default='0',
+               help=_('Delay in seconds for operations with the fake '
+                      'bios driver. Two comma-delimited values will '
+                      'result in a delay with a triangular random '
+                      'distribution, weighted on the first value.')),
+    cfg.StrOpt('storage_delay',
+               default='0',
+               help=_('Delay in seconds for operations with the fake '
+                      'storage driver. Two comma-delimited values will '
+                      'result in a delay with a triangular random '
+                      'distribution, weighted on the first value.')),
+    cfg.StrOpt('rescue_delay',
+               default='0',
+               help=_('Delay in seconds for operations with the fake '
+                      'rescue driver. Two comma-delimited values will '
+                      'result in a delay with a triangular random '
+                      'distribution, weighted on the first value.')),
+]
+
+
+def register_opts(conf):
+    conf.register_opts(opts, group='fake')

ironic/drivers/modules/fake.py

@@ -24,6 +24,9 @@ functionality between a power interface and a deploy interface, when both rely
 on separate vendor_passthru methods.
 """
 
+import random
+import time
+
 from oslo_log import log
 
 from ironic.common import boot_devices
@@ -32,6 +35,7 @@ from ironic.common import exception
 from ironic.common.i18n import _
 from ironic.common import indicator_states
 from ironic.common import states
+from ironic.conf import CONF
 from ironic.drivers import base
 from ironic import objects
 
@@ -39,6 +43,34 @@ from ironic import objects
 LOG = log.getLogger(__name__)
 
 
+def parse_sleep_range(sleep_range):
+    if not sleep_range:
+        return 0, 0
+
+    sleep_split = sleep_range.split(',')
+    if len(sleep_split) == 1:
+        a = sleep_split[0]
+        b = sleep_split[0]
+    else:
+        a = sleep_split[0]
+        b = sleep_split[1]
+    return int(a), int(b)
+
+
+def sleep(sleep_range):
+    earliest, latest = parse_sleep_range(sleep_range)
+    if earliest == 0 and latest == 0:
+        # no sleep
+        return
+    if earliest == latest:
+        # constant sleep
+        sleep = earliest
+    else:
+        # triangular random sleep, weighted towards the earliest
+        sleep = random.triangular(earliest, latest, earliest)
+    time.sleep(sleep)
+
+
 class FakePower(base.PowerInterface):
     """Example implementation of a simple power interface."""
 
@@ -49,12 +81,15 @@ class FakePower(base.PowerInterface):
         pass
 
     def get_power_state(self, task):
+        sleep(CONF.fake.power_delay)
         return task.node.power_state
 
     def reboot(self, task, timeout=None):
+        sleep(CONF.fake.power_delay)
         pass
 
     def set_power_state(self, task, power_state, timeout=None):
+        sleep(CONF.fake.power_delay)
         if power_state not in [states.POWER_ON, states.POWER_OFF,
                                states.SOFT_REBOOT, states.SOFT_POWER_OFF]:
             raise exception.InvalidParameterValue(
@@ -81,15 +116,19 @@ class FakeBoot(base.BootInterface):
         pass
 
     def prepare_ramdisk(self, task, ramdisk_params, mode='deploy'):
+        sleep(CONF.fake.boot_delay)
         pass
 
     def clean_up_ramdisk(self, task, mode='deploy'):
+        sleep(CONF.fake.boot_delay)
         pass
 
     def prepare_instance(self, task):
+        sleep(CONF.fake.boot_delay)
         pass
 
     def clean_up_instance(self, task):
+        sleep(CONF.fake.boot_delay)
         pass
 
 
@@ -108,18 +147,23 @@ class FakeDeploy(base.DeployInterface):
 
     @base.deploy_step(priority=100)
     def deploy(self, task):
+        sleep(CONF.fake.deploy_delay)
         return None
 
     def tear_down(self, task):
+        sleep(CONF.fake.deploy_delay)
         return states.DELETED
 
     def prepare(self, task):
+        sleep(CONF.fake.deploy_delay)
         pass
 
     def clean_up(self, task):
+        sleep(CONF.fake.deploy_delay)
         pass
 
     def take_over(self, task):
+        sleep(CONF.fake.deploy_delay)
         pass
 
 
@@ -140,6 +184,7 @@ class FakeVendorA(base.VendorInterface):
     @base.passthru(['POST'],
                    description=_("Test if the value of bar is baz"))
     def first_method(self, task, http_method, bar):
+        sleep(CONF.fake.vendor_delay)
         return True if bar == 'baz' else False
 
 
@@ -161,16 +206,19 @@ class FakeVendorB(base.VendorInterface):
     @base.passthru(['POST'],
                    description=_("Test if the value of bar is kazoo"))
     def second_method(self, task, http_method, bar):
+        sleep(CONF.fake.vendor_delay)
         return True if bar == 'kazoo' else False
 
     @base.passthru(['POST'], async_call=False,
                    description=_("Test if the value of bar is meow"))
     def third_method_sync(self, task, http_method, bar):
+        sleep(CONF.fake.vendor_delay)
         return True if bar == 'meow' else False
 
     @base.passthru(['POST'], require_exclusive_lock=False,
                    description=_("Test if the value of bar is woof"))
     def fourth_method_shared_lock(self, task, http_method, bar):
+        sleep(CONF.fake.vendor_delay)
         return True if bar == 'woof' else False
 
 
@@ -211,17 +259,21 @@ class FakeManagement(base.ManagementInterface):
         return [boot_devices.PXE]
 
     def set_boot_device(self, task, device, persistent=False):
+        sleep(CONF.fake.management_delay)
         if device not in self.get_supported_boot_devices(task):
             raise exception.InvalidParameterValue(_(
                 "Invalid boot device %s specified.") % device)
 
     def get_boot_device(self, task):
+        sleep(CONF.fake.management_delay)
         return {'boot_device': boot_devices.PXE, 'persistent': False}
 
     def get_sensors_data(self, task):
+        sleep(CONF.fake.management_delay)
         return {}
 
     def get_supported_indicators(self, task, component=None):
+        sleep(CONF.fake.management_delay)
         indicators = {
             components.CHASSIS: {
                 'led-0': {
@@ -248,6 +300,7 @@ class FakeManagement(base.ManagementInterface):
                 if not component or component == c}
 
     def get_indicator_state(self, task, component, indicator):
+        sleep(CONF.fake.management_delay)
         indicators = self.get_supported_indicators(task)
         if component not in indicators:
             raise exception.InvalidParameterValue(_(
@@ -271,6 +324,7 @@ class FakeInspect(base.InspectInterface):
         pass
 
     def inspect_hardware(self, task):
+        sleep(CONF.fake.inspect_delay)
         return states.MANAGEABLE
 
 
@@ -282,9 +336,11 @@ class FakeRAID(base.RAIDInterface):
 
     def create_configuration(self, task, create_root_volume=True,
                              create_nonroot_volumes=True):
+        sleep(CONF.fake.raid_delay)
         pass
 
     def delete_configuration(self, task):
+        sleep(CONF.fake.raid_delay)
         pass
 
 
@@ -302,6 +358,7 @@ class FakeBIOS(base.BIOSInterface):
                      'to contain a dictionary with name/value pairs'),
                      'required': True}})
     def apply_configuration(self, task, settings):
+        sleep(CONF.fake.bios_delay)
         # Note: the implementation of apply_configuration in fake interface
         # is just for testing purpose, for real driver implementation, please
         # refer to develop doc at https://docs.openstack.org/ironic/latest/
@@ -328,6 +385,7 @@ class FakeBIOS(base.BIOSInterface):
 
     @base.clean_step(priority=0)
     def factory_reset(self, task):
+        sleep(CONF.fake.bios_delay)
         # Note: the implementation of factory_reset in fake interface is
         # just for testing purpose, for real driver implementation, please
         # refer to develop doc at https://docs.openstack.org/ironic/latest/
@@ -340,6 +398,7 @@ class FakeBIOS(base.BIOSInterface):
 
     @base.clean_step(priority=0)
     def cache_bios_settings(self, task):
+        sleep(CONF.fake.bios_delay)
         # Note: the implementation of cache_bios_settings in fake interface
         # is just for testing purpose, for real driver implementation, please
         # refer to develop doc at https://docs.openstack.org/ironic/latest/
@@ -357,9 +416,11 @@ class FakeStorage(base.StorageInterface):
         return {}
 
     def attach_volumes(self, task):
+        sleep(CONF.fake.storage_delay)
         pass
 
     def detach_volumes(self, task):
+        sleep(CONF.fake.storage_delay)
         pass
 
     def should_write_image(self, task):
@@ -376,7 +437,9 @@ class FakeRescue(base.RescueInterface):
         pass
 
     def rescue(self, task):
+        sleep(CONF.fake.rescue_delay)
         return states.RESCUE
 
     def unrescue(self, task):
+        sleep(CONF.fake.rescue_delay)
         return states.ACTIVE

ironic/tests/unit/drivers/test_fake_hardware.py

@@ -17,6 +17,8 @@
 
 """Test class for Fake driver."""
 
+import time
+from unittest import mock
 
 from ironic.common import boot_devices
 from ironic.common import boot_modes
@@ -26,6 +28,7 @@ from ironic.common import indicator_states
 from ironic.common import states
 from ironic.conductor import task_manager
 from ironic.drivers import base as driver_base
+from ironic.drivers.modules import fake
 from ironic.tests.unit.db import base as db_base
 from ironic.tests.unit.db import utils as db_utils
 
@@ -164,3 +167,29 @@ class FakeHardwareTestCase(db_base.DbTestCase):
         self.assertEqual({}, self.driver.inspect.get_properties())
         self.driver.inspect.validate(self.task)
         self.driver.inspect.inspect_hardware(self.task)
+
+    def test_parse_sleep_range(self):
+        self.assertEqual((0, 0), fake.parse_sleep_range('0'))
+        self.assertEqual((0, 0), fake.parse_sleep_range(''))
+        self.assertEqual((1, 1), fake.parse_sleep_range('1'))
+        self.assertEqual((1, 10), fake.parse_sleep_range('1,10'))
+        self.assertEqual((10, 20), fake.parse_sleep_range('10, 20'))
+
+    @mock.patch.object(time, 'sleep', autospec=True)
+    def test_sleep_zero(self, mock_sleep):
+        fake.sleep("0")
+        mock_sleep.assert_not_called()
+
+    @mock.patch.object(time, 'sleep', autospec=True)
+    def test_sleep_one(self, mock_sleep):
+        fake.sleep("1")
+        mock_sleep.assert_called_once_with(1)
+
+    @mock.patch.object(time, 'sleep', autospec=True)
+    def test_sleep_range(self, mock_sleep):
+        for i in range(100):
+            fake.sleep("1,10")
+        for call in mock_sleep.call_args_list:
+            v = call[0][0]
+            self.assertGreaterEqual(v, 1)
+            self.assertLessEqual(v, 10)

releasenotes/notes/fakedelay-7eac23ad8881a736.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    There are now configurable random wait times for fake drivers in a new
+    ironic.conf [fake] section. Each supported driver having one configuration
+    option controlling the delay. These delays are applied to operations which
+    typically block in other drivers. This allows more realistic scenarios to
+    be arranged for performance and functional testing of ironic itself.