Implement take_over for iscsi_ilo driver

When iscsi_ilo driver is using a web server for storing boot images,
an active node is tied down to a specific conductor running the
webserver.  When it is taken over by a different conductor, the boot
image needs to be recreated on the new conductor when it takes over
the node.

Implements: blueprint remove-swift-dependency-for-ilo-drivers

Change-Id: I70cb8e7d624c8a91b4aa2bb550a5f4b6f64cb2b6

ironic/drivers/modules/ilo/deploy.py

@@ -71,6 +71,50 @@ CONF.import_opt('swift_ilo_container', 'ironic.drivers.modules.ilo.common',
 CONF.register_opts(clean_opts, group='ilo')
 
 
+def _recreate_and_populate_ilo_boot_iso(task):
+    """Recreate the boot iso for the node.
+
+    Recreates the boot iso for the image and host it
+    on a valid image service and populates the new boot iso
+    in the instance_info of the node.
+
+    :param task: a TaskManager instance containing the node to act on.
+    """
+    instance_info = task.node.instance_info
+    root_uuid = task.node.driver_internal_info.get('root_uuid_or_disk_id')
+    boot_iso = None
+    if root_uuid:
+        try:
+            # Recreate the boot iso
+            boot_iso = _get_boot_iso(task, root_uuid)
+        except Exception as e:
+            LOG.warning(_LW("Boot iso recreation failed during take over. "
+                            "Reason: %(reason)s. The node %(node)s may not "
+                            "come up with current boot_iso %(boot_iso)s. "),
+                        {'boot_iso': instance_info['ilo_boot_iso'],
+                         'reason': e, 'node': task.node.uuid})
+        # populate the new ilo_boot_iso in node.instance_info.
+        if boot_iso:
+            instance_info['ilo_boot_iso'] = boot_iso
+            task.node.instance_info = instance_info
+            task.node.save()
+        else:
+            LOG.warning(_LW("Boot iso recreation failed during take over. "
+                            "The node %(node)s may not come up "
+                            "with current boot_iso %(boot_iso)s. "),
+                        {'boot_iso': instance_info['ilo_boot_iso'],
+                         'node': task.node.uuid})
+    else:
+        LOG.warning(_LW("There is not enough information to recreate "
+                        "boot iso. The UUID for the root partition "
+                        "could not be found. The boot-iso cannot be "
+                        "created without root_uuid. The node %(node)s may "
+                        "not come up with current boot_iso "
+                        "%(boot_iso)s "),
+                    {'boot_iso': instance_info['ilo_boot_iso'],
+                     'node': task.node.uuid})
+
+
 def _get_boot_iso_object_name(node):
     """Returns the boot iso object name for a given node.
 
@@ -171,11 +215,14 @@ def _get_boot_iso(task, root_uuid):
                                kernel_href, ramdisk_href,
                                deploy_iso_uuid, root_uuid,
                                kernel_params, boot_mode)
-
         if CONF.ilo.use_web_server_for_images:
             boot_iso_url = (
                 ilo_common.copy_image_to_web_server(boot_iso_tmp_file,
                                                     boot_iso_object_name))
+            driver_internal_info = task.node.driver_internal_info
+            driver_internal_info['boot_iso_created_in_web_server'] = True
+            task.node.driver_internal_info = driver_internal_info
+            task.node.save()
             LOG.debug("Created boot_iso %(boot_iso)s for node %(node)s",
                       {'boot_iso': boot_iso_url, 'node': task.node.uuid})
             return boot_iso_url
@@ -490,6 +537,11 @@ class IloVirtualMediaIscsiDeploy(base.DeployInterface):
         """
         manager_utils.node_power_action(task, states.POWER_OFF)
         _disable_secure_boot_if_supported(task)
+        driver_internal_info = task.node.driver_internal_info
+        driver_internal_info.pop('boot_iso_created_in_web_server', None)
+        driver_internal_info.pop('root_uuid_or_disk_id', None)
+        task.node.driver_internal_info = driver_internal_info
+        task.node.save()
         return states.DELETED
 
     def prepare(self, task):
@@ -515,7 +567,19 @@ class IloVirtualMediaIscsiDeploy(base.DeployInterface):
             ilo_common.destroy_floppy_image_from_web_server(task.node)
 
     def take_over(self, task):
-        pass
+        """Enables boot up of an ACTIVE node.
+
+        It ensures that the ACTIVE node can be booted up successfully
+        when node is taken over by another conductor.
+
+        :param: task: a TaskManager instance containing the node to act on.
+        """
+        driver_internal_info = task.node.driver_internal_info
+        boot_iso_created_in_web_server = (
+            driver_internal_info.get('boot_iso_created_in_web_server'))
+        if (CONF.ilo.use_web_server_for_images
+                and boot_iso_created_in_web_server):
+            _recreate_and_populate_ilo_boot_iso(task)
 
 
 class IloVirtualMediaAgentDeploy(base.DeployInterface):
@@ -936,6 +1000,10 @@ class VendorPassthru(agent_base_vendor.BaseAgentVendor):
         uuid_dict = iscsi_deploy.continue_deploy(task, **kwargs)
         root_uuid_or_disk_id = uuid_dict.get(
             'root uuid', uuid_dict.get('disk identifier'))
+        driver_internal_info = task.node.driver_internal_info
+        driver_internal_info['root_uuid_or_disk_id'] = root_uuid_or_disk_id
+        task.node.driver_internal_info = driver_internal_info
+        task.node.save()
 
         try:
             # Set boot mode

ironic/tests/drivers/ilo/test_deploy.py

@@ -546,6 +546,89 @@ class IloDeployPrivateMethodsTestCase(db_base.DbTestCase):
             deploy_boot_mode = task.node.instance_info.get('deploy_boot_mode')
             self.assertIsNone(deploy_boot_mode)
 
+    @mock.patch.object(ilo_deploy.LOG, 'warning', spec_set=True,
+                       autospec=True)
+    @mock.patch.object(ilo_deploy, '_get_boot_iso', spec_set=True,
+                       autospec=True)
+    def test__recreate_and_populate_boot_iso_root_uuid_set(self,
+                                                           get_boot_iso_mock,
+                                                           log_mock):
+        driver_internal_info = {}
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=False) as task:
+            driver_internal_info['root_uuid_or_disk_id'] = 'root-uuid'
+            task.node.driver_internal_info = driver_internal_info
+            r_uuid = task.node.driver_internal_info['root_uuid_or_disk_id']
+            get_boot_iso_mock.return_value = 'boot-uuid'
+            ilo_deploy._recreate_and_populate_ilo_boot_iso(task)
+            self.assertEqual(task.node.instance_info['ilo_boot_iso'],
+                             'boot-uuid')
+            get_boot_iso_mock.assert_called_once_with(task, r_uuid)
+            self.assertFalse(log_mock.called)
+
+    @mock.patch.object(ilo_deploy.LOG, 'warning', spec_set=True,
+                       autospec=True)
+    @mock.patch.object(ilo_deploy, '_get_boot_iso', spec_set=True,
+                       autospec=True)
+    def test__recreate_and_populate_boot_iso_root_not_set(self,
+                                                          get_boot_iso_mock,
+                                                          log_mock):
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=False) as task:
+            task.node.instance_info['ilo_boot_iso'] = 'boot-uuid-old-iso'
+            ilo_deploy._recreate_and_populate_ilo_boot_iso(task)
+            self.assertEqual(task.node.instance_info['ilo_boot_iso'],
+                             'boot-uuid-old-iso')
+            self.assertFalse(get_boot_iso_mock.called)
+            self.assertTrue(log_mock.called)
+
+    @mock.patch.object(ilo_deploy.LOG, 'warning',
+                       spec_set=True, autospec=True)
+    @mock.patch.object(ilo_deploy, '_get_boot_iso',
+                       spec_set=True, autospec=True)
+    def test__recreate_and_populate_get_boot_iso_fails(self,
+                                                       get_boot_iso_mock,
+                                                       log_mock):
+        driver_internal_info = {}
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=False) as task:
+
+            driver_internal_info['boot_iso_created_in_web_server'] = True
+            driver_internal_info['root_uuid_or_disk_id'] = 'uuid'
+            task.node.instance_info['ilo_boot_iso'] = 'boot-uuid-old-iso'
+            task.node.driver_internal_info = driver_internal_info
+            task.node.save()
+            r_uuid = task.node.driver_internal_info.get('root_uuid_or_disk_id')
+            get_boot_iso_mock.side_effect = Exception
+            ilo_deploy._recreate_and_populate_ilo_boot_iso(task)
+            self.assertEqual(task.node.instance_info['ilo_boot_iso'],
+                             'boot-uuid-old-iso')
+            get_boot_iso_mock.assert_called_once_with(task, r_uuid)
+            self.assertTrue(log_mock.called)
+
+    @mock.patch.object(ilo_deploy.LOG, 'warning',
+                       spec_set=True, autospec=True)
+    @mock.patch.object(ilo_deploy, '_get_boot_iso',
+                       spec_set=True, autospec=True)
+    def test__recreate_and_populate_get_boot_iso_none(self,
+                                                      boot_iso_mock,
+                                                      log_mock):
+        driver_internal_info = {}
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=False) as task:
+            driver_internal_info['boot_iso_created_in_web_server'] = True
+            driver_internal_info['root_uuid_or_disk_id'] = 'uuid'
+            task.node.driver_internal_info = driver_internal_info
+            r_uuid = task.node.driver_internal_info.get('root_uuid_or_disk_id')
+            task.node.instance_info['ilo_boot_iso'] = 'boot-uuid-old-iso'
+            task.node.save()
+            boot_iso_mock.return_value = None
+            ilo_deploy._recreate_and_populate_ilo_boot_iso(task)
+            boot_iso_mock.assert_called_once_with(task, r_uuid)
+            self.assertEqual(task.node.instance_info['ilo_boot_iso'],
+                             'boot-uuid-old-iso')
+            self.assertTrue(log_mock.called)
+
 
 class IloVirtualMediaIscsiDeployTestCase(db_base.DbTestCase):
 
@@ -724,11 +807,19 @@ class IloVirtualMediaIscsiDeployTestCase(db_base.DbTestCase):
                        update_secure_boot_mode_mock):
         with task_manager.acquire(self.context, self.node.uuid,
                                   shared=False) as task:
+            driver_internal_info = task.node.driver_internal_info
+            driver_internal_info['boot_iso_created_in_web_server'] = True
+            driver_internal_info['root_uuid_or_disk_id'] = 'uuid'
+            task.node.driver_internal_info = driver_internal_info
+            task.node.save()
             returned_state = task.driver.deploy.tear_down(task)
             node_power_action_mock.assert_called_once_with(task,
                                                            states.POWER_OFF)
             update_secure_boot_mode_mock.assert_called_once_with(task, False)
             self.assertEqual(states.DELETED, returned_state)
+            dinfo = task.node.driver_internal_info
+            self.assertNotIn('boot_iso_created_in_web_server', dinfo)
+            self.assertNotIn('root_uuid_or_disk_id', dinfo)
 
     @mock.patch.object(ilo_deploy.LOG, 'warn', spec_set=True, autospec=True)
     @mock.patch.object(ilo_deploy, 'exception', spec_set=True, autospec=True)
@@ -794,6 +885,40 @@ class IloVirtualMediaIscsiDeployTestCase(db_base.DbTestCase):
             task.driver.deploy.prepare(task)
             self.assertFalse(func_prepare_node_for_deploy.called)
 
+    @mock.patch.object(ilo_deploy, '_recreate_and_populate_ilo_boot_iso',
+                       spec_set=True, autospec=True)
+    def test_take_over_recreate_iso_config_and_dif_set(self, mock_recreate):
+        driver_internal_info = {}
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=False) as task:
+            CONF.ilo.use_web_server_for_images = True
+            driver_internal_info['boot_iso_created_in_web_server'] = True
+            task.node.driver_internal_info = driver_internal_info
+            task.node.save()
+            task.driver.deploy.take_over(task)
+            mock_recreate.assert_called_once_with(task)
+
+    @mock.patch.object(ilo_deploy, '_recreate_and_populate_ilo_boot_iso',
+                       spec_set=True, autospec=True)
+    def test_take_over_recreate_iso_config_set_and_dif_not_set(self,
+                                                               mock_recreate):
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=False) as task:
+            CONF.ilo.use_web_server_for_images = True
+            task.node.save()
+            task.driver.deploy.take_over(task)
+            self.assertFalse(mock_recreate.called)
+
+    @mock.patch.object(ilo_deploy, '_recreate_and_populate_ilo_boot_iso',
+                       spec_set=True, autospec=True)
+    def test_take_over_recreate_iso_config_not_set(self, mock_recreate):
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=False) as task:
+            CONF.ilo.use_web_server_for_images = False
+            task.node.save()
+            task.driver.deploy.take_over(task)
+            self.assertFalse(mock_recreate.called)
+
 
 class IloVirtualMediaAgentDeployTestCase(db_base.DbTestCase):
 
@@ -1129,6 +1254,8 @@ class VendorPassthruTestCase(db_base.DbTestCase):
 
             self.assertEqual('boot-iso',
                              task.node.instance_info['ilo_boot_iso'])
+            info = task.node.driver_internal_info['root_uuid_or_disk_id']
+            self.assertEqual('root-uuid', info)
             notify_ramdisk_to_proceed_mock.assert_called_once_with('123456')
 
     @mock.patch.object(ilo_common, 'cleanup_vmedia_boot', spec_set=True,