Merge "Add manual clean and automated verify steps to set BMC clock via Redfish Manager"

ironic/conf/redfish.py

@@ -129,6 +129,17 @@ opts = [
     cfg.StrOpt('verify_ca',
                help=_('The default verify_ca path when redfish_verify_ca '
                       'in driver_info is missing or set to True.')),
+    cfg.BoolOpt('enable_verify_bmc_clock',
+                default=False,
+                help=_('Whether to enable the automated verify step '
+                       'that checks and sets the BMC clock. '
+                       'When enabled, Ironic will automatically attempt '
+                       'to set the BMC\'s clock during node registration '
+                       '(in the verify phase) using Redfish\'s '
+                       'DateTime fields. '
+                       'This helps avoid TLS certificate issues '
+                       'caused by incorrect BMC time.')),
+
 ]
 
 

ironic/drivers/modules/redfish/management.py

@@ -14,9 +14,11 @@
 #    under the License.
 
 import collections
+from datetime import timezone
 import time
 from urllib.parse import urlparse
 
+from dateutil import parser
 from oslo_log import log
 from oslo_utils import timeutils
 import sushy
@@ -448,6 +450,101 @@ class RedfishManagement(base.ManagementInterface):
             for attr in [getattr(resource, field)]
         }
 
+    @base.clean_step(priority=0, abortable=False, argsinfo={
+        'target_datetime': {
+            'description': 'The datetime to set in ISO8601 format',
+            'required': True
+        },
+        'datetime_local_offset': {
+            'description': 'The local time offset from UTC',
+            'required': False
+        }
+    })
+    @task_manager.require_exclusive_lock
+    def set_bmc_clock(self, task, target_datetime, datetime_local_offset=None):
+        """Set the BMC clock using Redfish Manager resource.
+
+        :param task: a TaskManager instance containing the node to act on.
+        :param target_datetime: The datetime to set in ISO8601 format
+        :param datetime_local_offset: The local time offset from UTC (optional)
+        :raises: RedfishError if the operation fails
+        """
+        try:
+            system = redfish_utils.get_system(task.node)
+            manager = redfish_utils.get_manager(task.node, system)
+            LOG.debug("Setting BMC clock to %s (offset: %s)",
+                      target_datetime, datetime_local_offset)
+            manager._conn.timeout = 30
+            manager.set_datetime(
+                target_datetime,
+                datetime_local_offset
+            )
+
+            manager.refresh()
+            if manager.datetime != target_datetime:
+                raise exception.RedfishError(
+                    "BMC clock update failed: mismatch after setting datetime")
+
+            LOG.info(
+                "Successfully updated BMC clock for node %s",
+                task.node.uuid
+            )
+        except Exception as e:
+            LOG.exception("BMC clock update failed: %s", e)
+            raise exception.RedfishError(error=str(e))
+
+    @base.verify_step(priority=1)
+    @task_manager.require_exclusive_lock
+    def verify_bmc_clock(self, task):
+        """Verify and auto-set the BMC clock to the current UTC time.
+
+        This step compares the system UTC time to the BMC's Redfish datetime.
+        If the difference exceeds 1 second, it attempts to sync the time.
+        Verification fails only if the BMC time remains incorrect
+        after the update.
+        """
+        if not CONF.redfish.enable_verify_bmc_clock:
+            LOG.info("Skipping BMC clock verify step: disabled via config")
+            return
+
+        try:
+            system_time = timeutils.utcnow().replace(
+                tzinfo=timezone.utc).isoformat()
+            system = redfish_utils.get_system(task.node)
+            manager = redfish_utils.get_manager(task.node, system)
+            manager.refresh()
+
+            manager_time = parser.isoparse(manager.datetime)
+            local_time = parser.isoparse(system_time)
+
+            LOG.debug("BMC time: %s, Local time: %s",
+                      manager_time, local_time)
+            LOG.debug("manager.datetime_local_offset: %s",
+                      manager.datetimelocaloffset)
+
+            # Fail if the BMC clock differs from system time
+            # by more than 1 second
+            if abs((manager_time - local_time).total_seconds()) > 1:
+                LOG.info("BMC clock is out of sync. Updating...")
+                manager.set_datetime(system_time,
+                                     datetime_local_offset="+00:00")
+                manager.refresh()
+
+                updated_time = parser.isoparse(manager.datetime)
+                if abs((updated_time - local_time).total_seconds()) > 1:
+                    raise exception.RedfishError(
+                        "BMC clock still incorrect after update")
+
+            LOG.info("BMC clock update successful for node %s",
+                     task.node.uuid)
+
+        except Exception as e:
+            LOG.exception("BMC clock auto-update failed during verify: %s", e)
+            raise exception.NodeVerifyFailure(
+                node=getattr(task.node, 'uuid', 'unknown'),
+                reason="BMC clock verify step failed: %s" % str(e)
+            )
+
     @classmethod
     def _get_sensors_fan(cls, chassis):
         """Get fan sensors reading.

ironic/tests/unit/drivers/modules/redfish/test_management.py

@@ -555,6 +555,140 @@ class RedfishManagementTestCase(db_base.DbTestCase):
             expected = boot_modes.LEGACY_BIOS
             self.assertEqual(expected, response)
 
+    @mock.patch.object(redfish_utils, 'get_manager', autospec=True)
+    @mock.patch.object(redfish_utils, 'get_system', autospec=True)
+    def test_set_bmc_clock_success(self, mock_get_system, mock_get_manager):
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=False) as task:
+            mock_system = mock.Mock()
+            mock_manager = mock.Mock()
+            mock_manager.datetime = '2025-06-27T12:00:00'
+
+            mock_get_system.return_value = mock_system
+            mock_get_manager.return_value = mock_manager
+
+            target_datetime = '2025-06-27T12:00:00'
+            task.driver.management.set_bmc_clock(
+                task, target_datetime=target_datetime)
+
+            mock_manager.set_datetime.assert_called_once_with(
+                target_datetime, None)
+            mock_manager.refresh.assert_called_once()
+
+    @mock.patch.object(redfish_utils, 'get_manager', autospec=True)
+    @mock.patch.object(redfish_utils, 'get_system', autospec=True)
+    def test_set_bmc_clock_datetime_mismatch_raises(self, mock_get_system,
+                                                    mock_get_manager):
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=False) as task:
+            mock_system = mock.Mock()
+            mock_manager = mock.Mock()
+            mock_manager.datetime = 'wrong-time'
+
+            mock_get_system.return_value = mock_system
+            mock_get_manager.return_value = mock_manager
+
+            with self.assertRaisesRegex(
+                    exception.RedfishError,
+                    'BMC clock update failed: mismatch after setting '
+                    'datetime'):
+                task.driver.management.set_bmc_clock(
+                    task, target_datetime='2025-06-27T12:00:00')
+
+            mock_manager.set_datetime.assert_called_once()
+            mock_manager.refresh.assert_called_once()
+
+    @mock.patch.object(redfish_utils, 'get_manager', autospec=True)
+    @mock.patch.object(redfish_utils, 'get_system', autospec=True)
+    @mock.patch.object(CONF.redfish, 'enable_verify_bmc_clock', new=False)
+    def test_verify_bmc_clock_disabled_in_config(self, mock_get_system,
+                                                 mock_get_manager):
+
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=False) as task:
+            task.driver.management.verify_bmc_clock(task)
+
+        # Ensure Redfish was never called
+        mock_get_system.assert_not_called()
+        mock_get_manager.assert_not_called()
+
+    @mock.patch.object(redfish_utils, 'get_manager', autospec=True)
+    @mock.patch.object(redfish_utils, 'get_system', autospec=True)
+    @mock.patch.object(timeutils, 'utcnow', autospec=True)
+    def test_verify_bmc_clock_success_when_time_matches(self, mock_utcnow,
+                                                        mock_get_system,
+                                                        mock_get_manager):
+
+        self.config(enable_verify_bmc_clock=True, group='redfish')
+        current_datetime = datetime.datetime(2025, 7, 13, 3, 0, 0)
+        mock_utcnow.return_value = current_datetime
+
+        self.node.updated_at = datetime.datetime.utcnow()
+        self.node.save()
+
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=False) as task:
+            mock_system = mock.Mock()
+            mock_manager = mock.Mock()
+
+            mock_manager.datetime = current_datetime.replace(
+                tzinfo=datetime.timezone.utc).isoformat()
+            mock_manager.datetimelocaloffset = '+00:00'
+
+            mock_get_system.return_value = mock_system
+            mock_get_manager.return_value = mock_manager
+
+            task.driver.management.verify_bmc_clock(task)
+
+            mock_manager.set_datetime.assert_not_called()
+            mock_manager.refresh.assert_called_once()
+
+    @mock.patch.object(redfish_utils, 'get_manager', autospec=True)
+    @mock.patch.object(redfish_utils, 'get_system', autospec=True)
+    @mock.patch.object(timeutils, 'utcnow', autospec=True)
+    def test_verify_bmc_clock_fails_on_mismatch(self, mock_utcnow,
+                                                mock_get_system,
+                                                mock_get_manager):
+        self.config(enable_verify_bmc_clock=True, group='redfish')
+
+        current_datetime = datetime.datetime(2025, 7, 13, 3, 0, 0)
+        mock_utcnow.return_value = current_datetime
+
+        self.node.updated_at = datetime.datetime.utcnow()
+        self.node.save()
+
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=False) as task:
+
+            if task.node is None:
+                task.node = self.node
+            if not getattr(task.node, 'uuid', None):
+                task.node.uuid = self.node.uuid
+
+            mock_system = mock.Mock()
+            mock_manager = mock.Mock()
+
+            # Mismatched time (e.g., 5 seconds later)
+            mock_manager.datetime = (
+                current_datetime + datetime.timedelta(seconds=5)).replace(
+                    tzinfo=datetime.timezone.utc).isoformat()
+            mock_manager.datetimelocaloffset = "+00:00"
+
+            mock_get_system.return_value = mock_system
+            mock_get_manager.return_value = mock_manager
+
+            management = redfish_mgmt.RedfishManagement()
+
+        with self.assertRaisesRegex(exception.NodeVerifyFailure,
+                                    "BMC clock verify step failed"):
+            management.verify_bmc_clock(task)
+
+        self.assertEqual(mock_manager.refresh.call_count, 2)
+        mock_manager.set_datetime.assert_called_once_with(
+            current_datetime.replace(
+                tzinfo=datetime.timezone.utc).isoformat(),
+            datetime_local_offset="+00:00")
+
     @mock.patch.object(redfish_utils, 'get_system', autospec=True)
     def test_inject_nmi(self, mock_get_system):
         fake_system = mock.Mock()

releasenotes/notes/add-bmc-clock-clean-verify-step-6b70b04a618bf6e1.yaml

@@ -0,0 +1,17 @@
+---
+features:
+  - |
+    Adds two new capabilities to the Redfish managemnet interface
+    for managing the BMC clock.
+
+    1. A manual cleaning step ``set_bmc_clock`` that allows operators
+       to set the BMC's hardware clock to a specific datetime (in ISO8601 format),
+       optionally including a datetimelocaloffset.
+
+    2. An automated verify step ``verify_bmc_clock``  that compares the
+       BMC's Redfish datetime to the system UTC time, and automatically
+       updates the BMC clock if needed. Verification fails if the difference
+       exceeds 1 second after the update.
+
+    These steps helps ensure BMC clock synchronization in baremetal environments
+    where incorrect or drifting BMC clocks may lead to TLS  certificate validation failures.

requirements.txt

@@ -38,7 +38,7 @@ psutil>=3.2.2 # BSD
 futurist>=1.2.0 # Apache-2.0
 tooz>=2.7.0 # Apache-2.0
 openstacksdk>=0.99.0 # Apache-2.0
-sushy>=4.8.0
+sushy>=5.7.0
 construct>=2.9.39 # MIT
 netaddr>=0.9.0 # BSD
 microversion-parse>=1.0.1 # Apache-2.0