# Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. """Tests for service bits.""" from unittest import mock from oslo_config import cfg from oslo_utils import uuidutils from ironic.common import exception from ironic.common import faults from ironic.common import states from ironic.conductor import servicing from ironic.conductor import steps as conductor_steps from ironic.conductor import task_manager from ironic.conductor import utils as conductor_utils from ironic.drivers.modules import fake from ironic.drivers.modules.network import flat as n_flat from ironic.tests.unit.db import base as db_base from ironic.tests.unit.objects import utils as obj_utils CONF = cfg.CONF # NOTE(TheJulia): This file is based upon test_cleaning.py with logic # for automated cleaning out and switched over for the service steps # framework. It *largely* exists to ensure we have similar consistency # between the frameworks, similar was done for deploy steps in the past. class DoNodeServiceTestCase(db_base.DbTestCase): def setUp(self): super(DoNodeServiceTestCase, self).setUp() self.power_update = { 'step': 'update_firmware', 'priority': 10, 'interface': 'power'} self.deploy_update = { 'step': 'update_firmware', 'priority': 10, 'interface': 'deploy'} self.deploy_magic = { 'step': 'magic_firmware', 'priority': 10, 'interface': 'deploy'} self.next_service_step_index = 1 self.deploy_raid = { 'step': 'build_raid', 'priority': 0, 'interface': 'deploy'} self.service_steps = [self.deploy_update, self.power_update, self.deploy_magic] def __do_node_service_validate_fail(self, mock_validate, service_steps=None): tgt_prov_state = states.ACTIVE node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=states.SERVICING, target_provision_state=tgt_prov_state) with task_manager.acquire( self.context, node.uuid, shared=False) as task: servicing.do_node_service(task, service_steps=service_steps) node.refresh() self.assertEqual(states.SERVICEFAIL, node.provision_state) self.assertEqual(tgt_prov_state, node.target_provision_state) self.assertFalse(node.maintenance) self.assertIsNone(node.fault) mock_validate.assert_called_once_with(mock.ANY, mock.ANY) def __do_node_service_validate_fail_invalid(self, mock_validate, service_steps=None): # InvalidParameterValue should cause node to go to SERVICEFAIL mock_validate.side_effect = exception.InvalidParameterValue('error') self.__do_node_service_validate_fail(mock_validate, service_steps=service_steps) @mock.patch('ironic.drivers.modules.fake.FakePower.validate', autospec=True) def test__do_node_service_automated_power_validate_fail(self, mock_validate): self.__do_node_service_validate_fail_invalid(mock_validate) @mock.patch('ironic.drivers.modules.fake.FakePower.validate', autospec=True) def test__do_node_service_manual_power_validate_fail(self, mock_validate): self.__do_node_service_validate_fail_invalid(mock_validate, service_steps=[]) @mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate', autospec=True) def test__do_node_service_automated_network_validate_fail(self, mock_validate): self.__do_node_service_validate_fail_invalid(mock_validate) @mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate', autospec=True) def test__do_node_service_manual_network_validate_fail(self, mock_validate): self.__do_node_service_validate_fail_invalid(mock_validate, service_steps=[]) @mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate', autospec=True) def test__do_node_service_network_error_fail(self, mock_validate): # NetworkError should cause node to go to CLEANFAIL mock_validate.side_effect = exception.NetworkError() self.__do_node_service_validate_fail(mock_validate) @mock.patch.object(conductor_steps, 'set_node_service_steps', autospec=True) @mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate', autospec=True) @mock.patch('ironic.drivers.modules.fake.FakeDeploy.prepare_service', autospec=True) def test__do_node_service_prepare_service_fail(self, mock_prep, mock_validate, mock_steps, service_steps=[]): # NOTE(janders) after removing unconditional initial reboot into # ramdisk, set_node_service_steps needs to InvalidParameterValue # to force boot into ramdisk mock_steps.side_effect = exception.InvalidParameterValue('error') # Exception from task.driver.deploy.prepare_service should cause node # to go to SERVICEFAIL mock_prep.side_effect = exception.InvalidParameterValue('error') tgt_prov_state = states.ACTIVE node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=states.SERVICING, target_provision_state=tgt_prov_state) with task_manager.acquire( self.context, node.uuid, shared=False) as task: servicing.do_node_service(task, service_steps=service_steps) node.refresh() self.assertEqual(states.SERVICEFAIL, node.provision_state) self.assertEqual(tgt_prov_state, node.target_provision_state) mock_prep.assert_called_once_with(mock.ANY, task) mock_validate.assert_called_once_with(mock.ANY, task) self.assertFalse(node.maintenance) self.assertIsNone(node.fault) @mock.patch.object(conductor_steps, 'set_node_service_steps', autospec=True) @mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate', autospec=True) @mock.patch('ironic.drivers.modules.fake.FakeDeploy.prepare_service', autospec=True) def test__do_node_service_prepare_service_wait(self, mock_prep, mock_validate, mock_steps): service_steps = [ {'step': 'trigger_servicewait', 'priority': 10, 'interface': 'vendor'} ] mock_steps.side_effect = exception.InvalidParameterValue('error') mock_prep.return_value = states.SERVICEWAIT tgt_prov_state = states.ACTIVE node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=states.SERVICING, target_provision_state=tgt_prov_state, vendor_interface='fake') with task_manager.acquire( self.context, node.uuid, shared=False) as task: servicing.do_node_service(task, service_steps=service_steps) node.refresh() self.assertEqual(states.SERVICEWAIT, node.provision_state) self.assertEqual(tgt_prov_state, node.target_provision_state) mock_prep.assert_called_once_with(mock.ANY, mock.ANY) mock_validate.assert_called_once_with(mock.ANY, mock.ANY) @mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate', autospec=True) @mock.patch('ironic.drivers.modules.fake.FakeDeploy.prepare_service', autospec=True) def test__do_node_service_out_of_band(self, mock_prep, mock_validate): # NOTE(janders) this test ensures ramdisk isn't prepared if # steps do not require it service_steps = [ {'step': 'trigger_servicewait', 'priority': 10, 'interface': 'vendor'} ] mock_prep.return_value = states.SERVICEWAIT tgt_prov_state = states.ACTIVE node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=states.SERVICING, target_provision_state=tgt_prov_state, vendor_interface='fake') with task_manager.acquire( self.context, node.uuid, shared=False) as task: servicing.do_node_service(task, service_steps=service_steps) node.refresh() self.assertEqual(states.SERVICEWAIT, node.provision_state) self.assertEqual(tgt_prov_state, node.target_provision_state) mock_prep.assert_not_called() mock_validate.assert_called_once_with(mock.ANY, mock.ANY) @mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate', autospec=True) @mock.patch('ironic.drivers.modules.fake.FakeDeploy.prepare_service', autospec=True) def test__do_node_service_requires_ramdisk_fallback(self, mock_prep, mock_validate): # NOTE(janders): here we set 'requires_ramdisk': 'True' # to force ramdisk_needed to be set and trigger prepare ramdisk service_steps = [ {'step': 'trigger_servicewait', 'priority': 10, 'interface': 'vendor', 'requires_ramdisk': 'True'} ] mock_prep.return_value = states.SERVICEWAIT tgt_prov_state = states.ACTIVE node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=states.SERVICING, target_provision_state=tgt_prov_state, vendor_interface='fake') with task_manager.acquire( self.context, node.uuid, shared=False) as task: servicing.do_node_service(task, service_steps=service_steps) node.refresh() self.assertEqual(states.SERVICEWAIT, node.provision_state) self.assertEqual(tgt_prov_state, node.target_provision_state) mock_prep.assert_called_once_with(mock.ANY, mock.ANY) mock_validate.assert_called_once_with(mock.ANY, mock.ANY) @mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate', autospec=True) @mock.patch('ironic.drivers.modules.fake.FakeDeploy.prepare_service', autospec=True) def test__do_node_service_prepare_service_active(self, mock_prep, mock_validate): service_steps = [ {'step': 'log_passthrough', 'priority': 10, 'interface': 'vendor'} ] mock_prep.return_value = states.SERVICEWAIT tgt_prov_state = states.ACTIVE node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=states.SERVICING, target_provision_state=tgt_prov_state, vendor_interface='fake') with task_manager.acquire( self.context, node.uuid, shared=False) as task: servicing.do_node_service(task, service_steps=service_steps, disable_ramdisk=True) # Validate we went back to active, and did not trigger a ramdisk. node.refresh() self.assertEqual(states.ACTIVE, node.provision_state) self.assertEqual(states.NOSTATE, node.target_provision_state) mock_prep.assert_not_called() mock_validate.assert_not_called() @mock.patch.object(n_flat.FlatNetwork, 'validate', autospec=True) @mock.patch.object(conductor_steps, 'set_node_service_steps', autospec=True) def __do_node_service_steps_fail(self, mock_steps, mock_validate, service_steps=None): mock_steps.side_effect = exception.NodeCleaningFailure('failure') tgt_prov_state = states.ACTIVE node = obj_utils.create_test_node( self.context, driver='fake-hardware', uuid=uuidutils.generate_uuid(), provision_state=states.SERVICING, target_provision_state=tgt_prov_state) with task_manager.acquire( self.context, node.uuid, shared=False) as task: servicing.do_node_service(task, service_steps=service_steps) mock_validate.assert_called_once_with(mock.ANY, task) node.refresh() self.assertEqual(states.SERVICEFAIL, node.provision_state) self.assertEqual(tgt_prov_state, node.target_provision_state) mock_steps.assert_called_once_with(mock.ANY, disable_ramdisk=False) self.assertFalse(node.maintenance) self.assertIsNone(node.fault) @mock.patch('ironic.drivers.modules.fake.FakePower.set_power_state', autospec=True) @mock.patch.object(n_flat.FlatNetwork, 'validate', autospec=True) @mock.patch.object(conductor_steps, 'set_node_service_steps', autospec=True) def test_do_node_service_steps_fail_poweroff(self, mock_steps, mock_validate, mock_power, service_steps=None): mock_steps.side_effect = exception.NodeCleaningFailure('failure') tgt_prov_state = states.ACTIVE self.config(poweroff_in_cleanfail=True, group='conductor') node = obj_utils.create_test_node( self.context, driver='fake-hardware', uuid=uuidutils.generate_uuid(), provision_state=states.SERVICING, power_state=states.POWER_ON, target_provision_state=tgt_prov_state) with task_manager.acquire( self.context, node.uuid, shared=False) as task: servicing.do_node_service(task, service_steps=service_steps) mock_validate.assert_called_once_with(mock.ANY, task) node.refresh() self.assertEqual(states.SERVICEFAIL, node.provision_state) self.assertEqual(tgt_prov_state, node.target_provision_state) mock_steps.assert_called_once_with(mock.ANY, disable_ramdisk=False) self.assertFalse(mock_power.called) def test__do_node_service_steps_fail(self): self.__do_node_service_steps_fail(service_steps=[self.deploy_raid]) @mock.patch.object(conductor_steps, 'set_node_service_steps', autospec=True) @mock.patch.object(servicing, 'do_next_service_step', autospec=True) @mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate', autospec=True) @mock.patch('ironic.drivers.modules.fake.FakePower.validate', autospec=True) def __do_node_service(self, mock_power_valid, mock_network_valid, mock_next_step, mock_steps, service_steps=None, disable_ramdisk=False): tgt_prov_state = states.ACTIVE if not service_steps: service_steps = self.service_steps def set_steps(task, disable_ramdisk=None): dii = task.node.driver_internal_info dii['service_steps'] = service_steps task.node.driver_internal_info = dii task.node.save() mock_steps.side_effect = set_steps node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=states.SERVICING, target_provision_state=tgt_prov_state, last_error=None, power_state=states.POWER_OFF, driver_internal_info={'agent_secret_token': 'old'}) with task_manager.acquire( self.context, node.uuid, shared=False) as task: servicing.do_node_service(task, service_steps=service_steps, disable_ramdisk=disable_ramdisk) node.refresh() mock_power_valid.assert_called_once_with(mock.ANY, task) if disable_ramdisk: mock_network_valid.assert_not_called() else: mock_network_valid.assert_called_once_with(mock.ANY, task) mock_next_step.assert_called_once_with( task, 0, disable_ramdisk=disable_ramdisk) mock_steps.assert_called_once_with( task, disable_ramdisk=disable_ramdisk) if service_steps: self.assertEqual(service_steps, node.driver_internal_info['service_steps']) self.assertFalse(node.maintenance) self.assertNotIn('agent_secret_token', node.driver_internal_info) # Check that state didn't change self.assertEqual(states.SERVICING, node.provision_state) self.assertEqual(tgt_prov_state, node.target_provision_state) def test__do_node_service(self): self.__do_node_service() def test__do_node_service_disable_ramdisk(self): self.__do_node_service(service_steps=[self.deploy_raid], disable_ramdisk=True) @mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_service_step', autospec=True) def _do_next_service_step_first_step_async(self, return_state, mock_execute, service_steps=None): # Execute the first async clean step on a node driver_internal_info = {'service_step_index': None} tgt_prov_state = states.ACTIVE if service_steps: driver_internal_info['service_steps'] = service_steps else: driver_internal_info['service_steps'] = self.service_steps node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=states.SERVICING, target_provision_state=tgt_prov_state, last_error=None, driver_internal_info=driver_internal_info, clean_step={}) mock_execute.return_value = return_state expected_first_step = node.driver_internal_info['service_steps'][0] with task_manager.acquire( self.context, node.uuid, shared=False) as task: servicing.do_next_service_step(task, 0) node.refresh() self.assertEqual(states.SERVICEWAIT, node.provision_state) self.assertEqual(tgt_prov_state, node.target_provision_state) self.assertEqual(expected_first_step, node.service_step) self.assertEqual(0, node.driver_internal_info['service_step_index']) mock_execute.assert_called_once_with( mock.ANY, mock.ANY, expected_first_step) def test_do_next_service_step_automated_first_step_async(self): self._do_next_service_step_first_step_async(states.SERVICEWAIT) def test_do_next_service_step_manual_first_step_async(self): self._do_next_service_step_first_step_async( states.SERVICEWAIT, service_steps=[self.deploy_raid]) @mock.patch('ironic.drivers.modules.fake.FakePower.execute_service_step', autospec=True) def _do_next_clean_step_continue_from_last_cleaning(self, return_state, mock_execute, manual=False): # Resume an in-progress servicing after the first async step tgt_prov_state = states.ACTIVE node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=states.SERVICING, target_provision_state=tgt_prov_state, last_error=None, driver_internal_info={'service_steps': self.service_steps, 'service_step_index': 0, 'servicing_polling': True}, service_step=self.service_steps[0]) mock_execute.return_value = return_state with task_manager.acquire( self.context, node.uuid, shared=False) as task: servicing.do_next_service_step(task, self.next_service_step_index) node.refresh() self.assertEqual(states.SERVICEWAIT, node.provision_state) self.assertEqual(tgt_prov_state, node.target_provision_state) self.assertEqual(self.service_steps[1], node.service_step) self.assertEqual(1, node.driver_internal_info['service_step_index']) mock_execute.assert_called_once_with( mock.ANY, mock.ANY, self.service_steps[1]) self.assertNotIn('servicing_polling', node.driver_internal_info) def test_do_next_clean_step_continue_from_last_cleaning(self): self._do_next_clean_step_continue_from_last_cleaning( states.SERVICEWAIT) def test_do_next_clean_step_manual_continue_from_last_cleaning(self): self._do_next_clean_step_continue_from_last_cleaning( states.SERVICEWAIT, manual=True) @mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_service_step', autospec=True) def _do_next_service_step_last_step_noop(self, mock_execute, manual=False): # Resume where last_step is the last cleaning step, should be noop tgt_prov_state = states.ACTIVE info = {'service_steps': self.service_steps, 'service_step_index': len(self.service_steps) - 1, 'agent_url': 'test-url', 'agent_secret_token': 'token'} node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=states.SERVICING, target_provision_state=tgt_prov_state, last_error=None, driver_internal_info=info, service_step=self.service_steps[-1]) with task_manager.acquire( self.context, node.uuid, shared=False) as task: servicing.do_next_service_step(task, None) node.refresh() # Cleaning should be complete without calling additional steps self.assertEqual(tgt_prov_state, node.provision_state) self.assertEqual(states.NOSTATE, node.target_provision_state) self.assertEqual({}, node.clean_step) self.assertNotIn('service_step_index', node.driver_internal_info) self.assertIsNone(node.driver_internal_info['service_steps']) self.assertFalse(mock_execute.called) self.assertNotIn('agent_url', node.driver_internal_info) self.assertNotIn('agent_secret_token', node.driver_internal_info) def test__do_next_service_step_automated_last_step_noop(self): self._do_next_service_step_last_step_noop() def test__do_next_service_step_manual_last_step_noop(self): self._do_next_service_step_last_step_noop(manual=True) @mock.patch('ironic.drivers.utils.collect_ramdisk_logs', autospec=True) @mock.patch('ironic.drivers.modules.fake.FakeDeploy.tear_down_service', autospec=True) @mock.patch('ironic.drivers.modules.fake.FakePower.execute_service_step', autospec=True) @mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_service_step', autospec=True) def _do_next_service_step_all(self, mock_deploy_execute, mock_power_execute, mock_tear_down, mock_collect_logs, disable_ramdisk=False): # Run all steps from start to finish (all synchronous) tgt_prov_state = states.ACTIVE node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=states.SERVICING, target_provision_state=tgt_prov_state, last_error=None, driver_internal_info={'service_steps': self.service_steps, 'service_step_index': None}, clean_step={}) def fake_deploy(conductor_obj, task, step): driver_internal_info = task.node.driver_internal_info driver_internal_info['goober'] = 'test' task.node.driver_internal_info = driver_internal_info task.node.save() mock_deploy_execute.side_effect = fake_deploy mock_power_execute.return_value = None with task_manager.acquire( self.context, node.uuid, shared=False) as task: servicing.do_next_service_step( task, 0, disable_ramdisk=disable_ramdisk) mock_power_execute.assert_called_once_with(task.driver.power, task, self.service_steps[1]) mock_deploy_execute.assert_has_calls( [mock.call(task.driver.deploy, task, self.service_steps[0]), mock.call(task.driver.deploy, task, self.service_steps[2])]) if disable_ramdisk: mock_tear_down.assert_not_called() else: mock_tear_down.assert_called_once_with( task.driver.deploy, task) node.refresh() # Servicing should be complete self.assertEqual(tgt_prov_state, node.provision_state) self.assertEqual(states.NOSTATE, node.target_provision_state) self.assertEqual({}, node.service_step) self.assertNotIn('service_step_index', node.driver_internal_info) self.assertEqual('test', node.driver_internal_info['goober']) self.assertIsNone(node.driver_internal_info['service_steps']) self.assertFalse(mock_collect_logs.called) def test_do_next_clean_step_all(self): self._do_next_service_step_all() def test_do_next_clean_step_all_disable_ramdisk(self): self._do_next_service_step_all(disable_ramdisk=True) @mock.patch('ironic.drivers.utils.collect_ramdisk_logs', autospec=True) @mock.patch('ironic.drivers.modules.fake.FakePower.execute_service_step', autospec=True) @mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_service_step', autospec=True) def test_do_next_clean_step_collect_logs(self, mock_deploy_execute, mock_power_execute, mock_collect_logs): CONF.set_override('deploy_logs_collect', 'always', group='agent') # Run all steps from start to finish (all synchronous) tgt_prov_state = states.ACTIVE node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=states.SERVICING, target_provision_state=tgt_prov_state, last_error=None, driver_internal_info={'service_steps': self.service_steps, 'service_step_index': None}, clean_step={}) def fake_deploy(conductor_obj, task, step): driver_internal_info = task.node.driver_internal_info driver_internal_info['goober'] = 'test' task.node.driver_internal_info = driver_internal_info task.node.save() mock_deploy_execute.side_effect = fake_deploy mock_power_execute.return_value = None with task_manager.acquire( self.context, node.uuid, shared=False) as task: servicing.do_next_service_step(task, 0) node.refresh() # Cleaning should be complete self.assertEqual(tgt_prov_state, node.provision_state) self.assertEqual(states.NOSTATE, node.target_provision_state) self.assertEqual({}, node.clean_step) self.assertNotIn('service_step_index', node.driver_internal_info) self.assertEqual('test', node.driver_internal_info['goober']) self.assertIsNone(node.driver_internal_info['service_steps']) mock_power_execute.assert_called_once_with(mock.ANY, mock.ANY, self.service_steps[1]) mock_deploy_execute.assert_has_calls( [mock.call(mock.ANY, mock.ANY, self.service_steps[0]), mock.call(mock.ANY, mock.ANY, self.service_steps[2])]) mock_collect_logs.assert_called_once_with(mock.ANY, label='service') @mock.patch('ironic.drivers.utils.collect_ramdisk_logs', autospec=True) @mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_service_step', autospec=True) @mock.patch.object(fake.FakeDeploy, 'tear_down_service', autospec=True) def _do_next_service_step_execute_fail(self, tear_mock, mock_execute, mock_collect_logs): # When a clean step fails, go to CLEANFAIL tgt_prov_state = states.ACTIVE node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=states.SERVICING, target_provision_state=tgt_prov_state, last_error=None, driver_internal_info={'service_steps': self.service_steps, 'service_step_index': None}, clean_step={}) mock_execute.side_effect = Exception() with task_manager.acquire( self.context, node.uuid, shared=False) as task: servicing.do_next_service_step(task, 0) tear_mock.assert_called_once_with(task.driver.deploy, task) node.refresh() # Make sure we go to SERVICEFAIL, clear service_steps self.assertEqual(states.SERVICEFAIL, node.provision_state) self.assertEqual(tgt_prov_state, node.target_provision_state) self.assertEqual({}, node.service_step) self.assertNotIn('service_step_index', node.driver_internal_info) self.assertIsNotNone(node.last_error) self.assertTrue(node.maintenance) self.assertEqual(faults.SERVICE_FAILURE, node.fault) mock_execute.assert_called_once_with( mock.ANY, mock.ANY, self.service_steps[0]) mock_collect_logs.assert_called_once_with(mock.ANY, label='service') def test__do_next_clean_step_automated_execute_fail(self): self._do_next_service_step_execute_fail() @mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_service_step', autospec=True) def test_do_next_service_step_oob_reboot(self, mock_execute): tgt_prov_state = states.ACTIVE node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=states.SERVICING, target_provision_state=tgt_prov_state, last_error=None, driver_internal_info={'service_steps': self.service_steps, 'service_step_index': None, 'servicing_reboot': True}, service_step={}) mock_execute.side_effect = exception.AgentConnectionFailed( reason='failed') with task_manager.acquire( self.context, node.uuid, shared=False) as task: servicing.do_next_service_step(task, 0) node.refresh() # Make sure we go to SERVICEWAIT self.assertEqual(states.SERVICEWAIT, node.provision_state) self.assertEqual(tgt_prov_state, node.target_provision_state) self.assertEqual(self.service_steps[0], node.service_step) self.assertEqual(0, node.driver_internal_info['service_step_index']) self.assertFalse( node.driver_internal_info['skip_current_service_step']) mock_execute.assert_called_once_with( mock.ANY, mock.ANY, self.service_steps[0]) @mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_service_step', autospec=True) def test_do_next_service_step_agent_busy(self, mock_execute): tgt_prov_state = states.ACTIVE node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=states.SERVICING, target_provision_state=tgt_prov_state, last_error=None, driver_internal_info={'service_steps': self.service_steps, 'service_step_index': None, 'servicing_reboot': True}, service_step={}) mock_execute.side_effect = exception.AgentInProgress( reason='still meowing') with task_manager.acquire( self.context, node.uuid, shared=False) as task: servicing.do_next_service_step(task, 0) node.refresh() # Make sure we go to SERVICEWAIT self.assertEqual(states.SERVICEWAIT, node.provision_state) self.assertEqual(tgt_prov_state, node.target_provision_state) self.assertEqual(self.service_steps[0], node.service_step) self.assertEqual(0, node.driver_internal_info['service_step_index']) self.assertFalse( node.driver_internal_info['skip_current_service_step']) mock_execute.assert_called_once_with( mock.ANY, mock.ANY, self.service_steps[0]) @mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_service_step', autospec=True) def test_do_next_service_step_oob_reboot_last_step(self, mock_execute): # Resume where last_step is the last service step tgt_prov_state = states.ACTIVE info = {'service_steps': self.service_steps, 'servicing_reboot': True, 'service_step_index': len(self.service_steps) - 1} node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=states.SERVICING, target_provision_state=tgt_prov_state, last_error=None, driver_internal_info=info, service_step=self.service_steps[-1]) with task_manager.acquire( self.context, node.uuid, shared=False) as task: servicing.do_next_service_step(task, None) node.refresh() # Servicing should be complete without calling additional steps self.assertEqual(tgt_prov_state, node.provision_state) self.assertEqual(states.NOSTATE, node.target_provision_state) self.assertEqual({}, node.service_step) self.assertNotIn('service_step_index', node.driver_internal_info) self.assertNotIn('servicing_reboot', node.driver_internal_info) self.assertIsNone(node.driver_internal_info['service_steps']) self.assertFalse(mock_execute.called) @mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_service_step', autospec=True) @mock.patch.object(fake.FakeDeploy, 'tear_down_service', autospec=True) def test_do_next_service_step_oob_reboot_fail(self, tear_mock, mock_execute): # When a service step fails with no reboot requested go to SERVICEFAIL tgt_prov_state = states.ACTIVE node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=states.SERVICING, target_provision_state=tgt_prov_state, last_error=None, driver_internal_info={'service_steps': self.service_steps, 'service_step_index': None}, service_step={}) mock_execute.side_effect = exception.AgentConnectionFailed( reason='failed') with task_manager.acquire( self.context, node.uuid, shared=False) as task: servicing.do_next_service_step(task, 0) tear_mock.assert_called_once_with(task.driver.deploy, task) node.refresh() # Make sure we go to SERVICEFAIL, clear service_steps self.assertEqual(states.SERVICEFAIL, node.provision_state) self.assertEqual(tgt_prov_state, node.target_provision_state) self.assertEqual({}, node.service_step) self.assertNotIn('service_step_index', node.driver_internal_info) self.assertNotIn('skip_current_service_step', node.driver_internal_info) self.assertIsNotNone(node.last_error) self.assertTrue(node.maintenance) mock_execute.assert_called_once_with( mock.ANY, mock.ANY, self.service_steps[0]) @mock.patch.object(conductor_utils, 'LOG', autospec=True) @mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_service_step', autospec=True) @mock.patch('ironic.drivers.modules.fake.FakePower.execute_service_step', autospec=True) @mock.patch.object(fake.FakeDeploy, 'tear_down_service', autospec=True) def _do_next_service_step_fail_in_tear_down_service( self, tear_mock, power_exec_mock, deploy_exec_mock, log_mock): tgt_prov_state = states.ACTIVE node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=states.SERVICING, target_provision_state=tgt_prov_state, last_error=None, driver_internal_info={'service_steps': self.service_steps, 'service_step_index': None}, service_step={}) deploy_exec_mock.return_value = None power_exec_mock.return_value = None tear_mock.side_effect = Exception('boom') with task_manager.acquire( self.context, node.uuid, shared=False) as task: servicing.do_next_service_step(task, 0) node.refresh() # Make sure we go to SERVICEFAIL, clear service_steps self.assertEqual(states.SERVICEFAIL, node.provision_state) self.assertEqual(tgt_prov_state, node.target_provision_state) self.assertEqual({}, node.service_step) self.assertNotIn('clean_step_index', node.driver_internal_info) self.assertIsNotNone(node.last_error) self.assertEqual(1, tear_mock.call_count) self.assertFalse(node.maintenance) # no step is running deploy_exec_calls = [ mock.call(mock.ANY, mock.ANY, self.service_steps[0]), mock.call(mock.ANY, mock.ANY, self.service_steps[2]), ] self.assertEqual(deploy_exec_calls, deploy_exec_mock.call_args_list) power_exec_calls = [ mock.call(mock.ANY, mock.ANY, self.service_steps[1]), ] self.assertEqual(power_exec_calls, power_exec_mock.call_args_list) log_mock.error.assert_called_once_with( 'Failed to tear down from service for node {}, reason: boom' .format(node.uuid), exc_info=True) def test__do_next_service_step_automated_fail_in_tear_down_service(self): self._do_next_service_step_fail_in_tear_down_service() @mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_service_step', autospec=True) def _do_next_service_step_no_steps(self, mock_execute, manual=False): for info in ({'service_steps': None, 'service_step_index': None, 'agent_url': 'test-url', 'agent_secret_token': 'magic'}, {'service_steps': None, 'agent_url': 'test-url', 'agent_secret_token': 'it_is_a_kind_of_magic'}): # Resume where there are no steps, should be a noop tgt_prov_state = states.ACTIVE node = obj_utils.create_test_node( self.context, driver='fake-hardware', uuid=uuidutils.generate_uuid(), provision_state=states.SERVICING, target_provision_state=tgt_prov_state, last_error=None, driver_internal_info=info, service_step={}) with task_manager.acquire( self.context, node.uuid, shared=False) as task: servicing.do_next_service_step(task, None) node.refresh() # Cleaning should be complete without calling additional steps self.assertEqual(tgt_prov_state, node.provision_state) self.assertEqual(states.NOSTATE, node.target_provision_state) self.assertEqual({}, node.clean_step) self.assertNotIn('service_step_index', node.driver_internal_info) self.assertFalse(mock_execute.called) self.assertNotIn('agent_url', node.driver_internal_info) self.assertNotIn('agent_secret_token', node.driver_internal_info) mock_execute.reset_mock() def test__do_next_service_step_automated_no_steps(self): self._do_next_service_step_no_steps() @mock.patch('ironic.drivers.modules.fake.FakePower.execute_service_step', autospec=True) @mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_service_step', autospec=True) def test__do_next_service_step_bad_step_return_value( self, deploy_exec_mock, power_exec_mock, manual=False): # When a service step fails, go to CLEANFAIL tgt_prov_state = states.ACTIVE node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=states.SERVICING, target_provision_state=tgt_prov_state, last_error=None, driver_internal_info={'service_steps': self.service_steps, 'service_step_index': None}, service_step={}) deploy_exec_mock.return_value = "foo" with task_manager.acquire( self.context, node.uuid, shared=False) as task: servicing.do_next_service_step(task, 0) node.refresh() # Make sure we go to SERVICEFAIL, clear service_steps self.assertEqual(states.SERVICEFAIL, node.provision_state) self.assertEqual(tgt_prov_state, node.target_provision_state) self.assertEqual({}, node.service_step) self.assertNotIn('service_step_index', node.driver_internal_info) self.assertIsNotNone(node.last_error) self.assertTrue(node.maintenance) # the 1st clean step was running deploy_exec_mock.assert_called_once_with(mock.ANY, mock.ANY, self.service_steps[0]) # Make sure we don't execute any other step and return self.assertFalse(power_exec_mock.called) def _test_do_next_service_step_handles_hold(self, start_state): node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=start_state, driver_internal_info={ 'service_steps': [ { 'step': 'hold', 'priority': 10, 'interface': 'power' } ], 'service_step_index': None}, service_step=None) with task_manager.acquire( self.context, node.uuid, shared=False) as task: servicing.do_next_service_step(task, 0) node.refresh() self.assertEqual(states.SERVICEHOLD, node.provision_state) def test_do_next_service_step_handles_hold_from_active(self): # Start is from the conductor self._test_do_next_service_step_handles_hold(states.SERVICING) def test_do_next_service_step_handles_hold_from_wait(self): # Start is the continuation from a heartbeat. self._test_do_next_service_step_handles_hold(states.SERVICEWAIT) def test_do_next_service_step_handles_hold_from_failed(self): # Test that hold step from SERVICEFAIL transitions to SERVICEHOLD self._test_do_next_service_step_handles_hold(states.SERVICEFAIL) def test_do_next_service_step_handles_wait_from_failed(self): # Test that wait step from SERVICEFAIL transitions to SERVICEWAIT node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=states.SERVICEFAIL, driver_internal_info={ 'service_steps': [ { 'step': 'wait', 'priority': 10, 'interface': 'power' } ], 'service_step_index': None}, service_step=None) with task_manager.acquire( self.context, node.uuid, shared=False) as task: servicing.do_next_service_step(task, 0) node.refresh() self.assertEqual(states.SERVICEWAIT, node.provision_state) @mock.patch.object(servicing, 'do_next_service_step', autospec=True) def _continue_node_service(self, mock_next_step, skip=True): # test that skipping current step mechanism works driver_info = {'service_steps': self.service_steps, 'service_step_index': 0, 'servicing_polling': 'value'} if not skip: driver_info['skip_current_service_step'] = skip node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=states.SERVICING, target_provision_state=states.ACTIVE, driver_internal_info=driver_info, service_step=self.service_steps[0]) with task_manager.acquire(self.context, node.uuid) as task: servicing.continue_node_service(task) expected_step_index = 1 if skip else 0 self.assertNotIn( 'skip_current_service_step', task.node.driver_internal_info) self.assertNotIn( 'cleaning_polling', task.node.driver_internal_info) mock_next_step.assert_called_once_with(task, expected_step_index) def test_continue_node_service(self): self._continue_node_service(skip=True) def test_continue_node_service_no_skip_step(self): self._continue_node_service(skip=False) class DoNodeServiceAbortTestCase(db_base.DbTestCase): @mock.patch.object(fake.FakeDeploy, 'tear_down_service', autospec=True) def _test_do_node_service_abort(self, service_step, tear_mock=None): node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=states.SERVICEWAIT, target_provision_state=states.AVAILABLE, service_step=service_step, driver_internal_info={ 'agent_url': 'some url', 'agent_secret_token': 'token', 'service_step_index': 2, 'servicing_reboot': True, 'servicing_polling': True, 'skip_current_service_step': True}) with task_manager.acquire(self.context, node.uuid) as task: servicing.do_node_service_abort(task) self.assertIsNotNone(task.node.last_error) tear_mock.assert_called_once_with(task.driver.deploy, task) task.node.refresh() if service_step: self.assertIn(service_step['step'], task.node.last_error) # assert node's clean_step and metadata was cleaned up self.assertEqual({}, task.node.service_step) self.assertNotIn('service_step_index', task.node.driver_internal_info) self.assertNotIn('servicing_reboot', task.node.driver_internal_info) self.assertNotIn('servicing_polling', task.node.driver_internal_info) self.assertNotIn('skip_current_service_step', task.node.driver_internal_info) self.assertNotIn('agent_url', task.node.driver_internal_info) self.assertNotIn('agent_secret_token', task.node.driver_internal_info) def test_do_node_service_abort_early(self): self._test_do_node_service_abort(None) def test_do_node_service_abort_with_step(self): self._test_do_node_service_abort({'step': 'foo', 'interface': 'deploy', 'abortable': True}) @mock.patch.object(fake.FakeDeploy, 'tear_down_service', autospec=True) def test__do_node_service_abort_tear_down_fail(self, tear_mock): tear_mock.side_effect = Exception('Surprise') node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=states.SERVICEFAIL, target_provision_state=states.ACTIVE, service_step={'step': 'foo', 'abortable': True}) with task_manager.acquire(self.context, node.uuid) as task: servicing.do_node_service_abort(task) tear_mock.assert_called_once_with(task.driver.deploy, task) self.assertIsNotNone(task.node.last_error) self.assertIsNotNone(task.node.maintenance_reason) self.assertTrue(task.node.maintenance) self.assertEqual('service failure', task.node.fault) @mock.patch.object(fake.FakeDeploy, 'tear_down_service', autospec=True) def test__do_node_cleanhold_abort_tear_down_fail(self, tear_mock): tear_mock.side_effect = Exception('Surprise') node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=states.SERVICEHOLD, target_provision_state=states.ACTIVE, service_step={'step': 'hold', 'abortable': True}) with task_manager.acquire(self.context, node.uuid) as task: servicing.do_node_service_abort(task) tear_mock.assert_called_once_with(task.driver.deploy, task) self.assertIsNotNone(task.node.last_error) self.assertIsNotNone(task.node.maintenance_reason) self.assertTrue(task.node.maintenance) self.assertEqual('service failure', task.node.fault) class DoNodeCleanTestChildNodes(db_base.DbTestCase): def setUp(self): super(DoNodeCleanTestChildNodes, self).setUp() self.power_on_parent = { 'step': 'power_on', 'priority': 4, 'interface': 'power'} self.power_on_children = { 'step': 'power_on', 'priority': 5, 'interface': 'power', 'execute_on_child_nodes': True} self.update_firmware_on_children = { 'step': 'update_firmware', 'priority': 10, 'interface': 'management', 'execute_on_child_nodes': True} self.reboot_children = { 'step': 'reboot', 'priority': 5, 'interface': 'power', 'execute_on_child_nodes': True} self.power_off_children = { 'step': 'power_off', 'priority': 15, 'interface': 'power', 'execute_on_child_nodes': True} self.service_steps = [ self.power_on_parent, self.power_on_children, self.update_firmware_on_children, self.reboot_children, self.power_off_children] self.node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=states.SERVICING, target_provision_state=states.ACTIVE, last_error=None, power_state=states.POWER_ON, driver_internal_info={'agent_secret_token': 'old', 'service_steps': self.service_steps}) @mock.patch('ironic.drivers.modules.fake.FakePower.get_power_state', autospec=True) @mock.patch('ironic.drivers.modules.fake.FakePower.reboot', autospec=True) @mock.patch('ironic.drivers.modules.fake.FakePower.set_power_state', autospec=True) @mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate', autospec=True) @mock.patch('ironic.drivers.modules.fake.FakePower.validate', autospec=True) @mock.patch('ironic.drivers.modules.fake.FakePower.execute_service_step', autospec=True) @mock.patch('ironic.drivers.modules.fake.FakeManagement.' 'execute_service_step', autospec=True) @mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_service_step', autospec=True) def test_do_next_clean_step_with_children( self, mock_deploy, mock_mgmt, mock_power, mock_pv, mock_nv, mock_sps, mock_reboot, mock_gps): child_node1 = obj_utils.create_test_node( self.context, uuid=uuidutils.generate_uuid(), driver='fake-hardware', last_error=None, power_state=states.POWER_OFF, parent_node=self.node.uuid) child_node2 = obj_utils.create_test_node( self.context, uuid=uuidutils.generate_uuid(), driver='fake-hardware', last_error=None, power_state=states.POWER_OFF, parent_node=self.node.uuid) mock_gps.side_effect = [ states.POWER_OFF, states.POWER_ON, states.POWER_OFF, states.POWER_ON, states.POWER_OFF, states.POWER_ON] mock_deploy.return_value = None mock_mgmt.return_value = None mock_power.return_value = None child1_updated_at = str(child_node1.updated_at) child2_updated_at = str(child_node2.updated_at) with task_manager.acquire( self.context, self.node.uuid, shared=False) as task: servicing.do_next_service_step(task, 0, disable_ramdisk=True) self.node.refresh() child_node1.refresh() child_node2.refresh() # Confirm the objects *did* receive locks. self.assertNotEqual(child1_updated_at, child_node1.updated_at) self.assertNotEqual(child2_updated_at, child_node2.updated_at) # Confirm the child nodes have no errors self.assertFalse(child_node1.maintenance) self.assertFalse(child_node2.maintenance) self.assertIsNone(child_node1.last_error) self.assertIsNone(child_node2.last_error) self.assertIsNone(self.node.last_error) # Confirm the call counts expected self.assertEqual(0, mock_deploy.call_count) self.assertEqual(2, mock_mgmt.call_count) self.assertEqual(0, mock_power.call_count) self.assertEqual(0, mock_nv.call_count) self.assertEqual(0, mock_pv.call_count) self.assertEqual(3, mock_sps.call_count) self.assertEqual(2, mock_reboot.call_count) mock_sps.assert_has_calls([ mock.call(mock.ANY, mock.ANY, 'power on', timeout=None), mock.call(mock.ANY, mock.ANY, 'power on', timeout=None), mock.call(mock.ANY, mock.ANY, 'power off', timeout=None)]) @mock.patch('ironic.drivers.modules.fake.FakePower.set_power_state', autospec=True) @mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate', autospec=True) @mock.patch('ironic.drivers.modules.fake.FakePower.validate', autospec=True) @mock.patch('ironic.drivers.modules.fake.FakePower.execute_service_step', autospec=True) @mock.patch('ironic.drivers.modules.fake.FakeManagement.' 'execute_service_step', autospec=True) @mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_service_step', autospec=True) def test_do_next_clean_step_with_children_by_uuid( self, mock_deploy, mock_mgmt, mock_power, mock_pv, mock_nv, mock_sps): child_node1 = obj_utils.create_test_node( self.context, uuid=uuidutils.generate_uuid(), driver='fake-hardware', last_error=None, parent_node=self.node.uuid) child_node2 = obj_utils.create_test_node( self.context, uuid=uuidutils.generate_uuid(), driver='fake-hardware', last_error=None, parent_node=self.node.uuid) power_on_children = { 'step': 'power_on', 'priority': 5, 'interface': 'power', 'execute_on_child_nodes': True, 'limit_child_node_execution': [child_node1.uuid]} update_firmware_on_children = { 'step': 'update_firmware', 'priority': 10, 'interface': 'management', 'execute_on_child_nodes': True, 'limit_child_node_execution': [child_node1.uuid]} power_on_parent = { 'step': 'not_power', 'priority': 15, 'interface': 'power'} service_steps = [power_on_children, update_firmware_on_children, power_on_parent] dii = self.node.driver_internal_info dii['service_steps'] = service_steps self.node.driver_internal_info = dii self.node.save() mock_deploy.return_value = None mock_mgmt.return_value = None mock_power.return_value = None child1_updated_at = str(child_node1.updated_at) with task_manager.acquire( self.context, self.node.uuid, shared=False) as task: servicing.do_next_service_step(task, 0, disable_ramdisk=True) self.node.refresh() child_node1.refresh() child_node2.refresh() # Confirm the objects *did* receive locks. self.assertNotEqual(child1_updated_at, child_node1.updated_at) self.assertIsNone(child_node2.updated_at) # Confirm the child nodes have no errors self.assertFalse(child_node1.maintenance) self.assertFalse(child_node2.maintenance) self.assertIsNone(child_node1.last_error) self.assertIsNone(child_node2.last_error) self.assertIsNone(self.node.last_error) # Confirm the call counts expected self.assertEqual(0, mock_deploy.call_count) self.assertEqual(1, mock_mgmt.call_count) self.assertEqual(1, mock_power.call_count) self.assertEqual(0, mock_nv.call_count) self.assertEqual(0, mock_pv.call_count) mock_sps.assert_has_calls([ mock.call(mock.ANY, mock.ANY, 'power on', timeout=None)])