f1943cead5
Add missing state machine transitions from SERVICEFAIL to SERVICEWAIT and SERVICEHOLD for reserved wait/hold steps. This fixes the edge-case where nodes in service failed state would incorrectly transition directly to active state when wait/hold steps were executed, bypassing expected intermediate states. Closes-Bug: #2119990 Change-Id: I0a55ad45138c4d033570014bf45956dacaf11e72 Signed-off-by: Afonne-CID <afonnepaulc@gmail.com>
1280 lines
58 KiB
Python
1280 lines
58 KiB
Python
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
"""Tests for service bits."""
|
|
|
|
from unittest import mock
|
|
|
|
from oslo_config import cfg
|
|
from oslo_utils import uuidutils
|
|
|
|
from ironic.common import exception
|
|
from ironic.common import faults
|
|
from ironic.common import states
|
|
from ironic.conductor import servicing
|
|
from ironic.conductor import steps as conductor_steps
|
|
from ironic.conductor import task_manager
|
|
from ironic.conductor import utils as conductor_utils
|
|
from ironic.drivers.modules import fake
|
|
from ironic.drivers.modules.network import flat as n_flat
|
|
from ironic.tests.unit.db import base as db_base
|
|
from ironic.tests.unit.objects import utils as obj_utils
|
|
|
|
CONF = cfg.CONF
|
|
|
|
# NOTE(TheJulia): This file is based upon test_cleaning.py with logic
|
|
# for automated cleaning out and switched over for the service steps
|
|
# framework. It *largely* exists to ensure we have similar consistency
|
|
# between the frameworks, similar was done for deploy steps in the past.
|
|
|
|
|
|
class DoNodeServiceTestCase(db_base.DbTestCase):
|
|
def setUp(self):
|
|
super(DoNodeServiceTestCase, self).setUp()
|
|
self.power_update = {
|
|
'step': 'update_firmware', 'priority': 10, 'interface': 'power'}
|
|
self.deploy_update = {
|
|
'step': 'update_firmware', 'priority': 10, 'interface': 'deploy'}
|
|
self.deploy_magic = {
|
|
'step': 'magic_firmware', 'priority': 10, 'interface': 'deploy'}
|
|
self.next_service_step_index = 1
|
|
self.deploy_raid = {
|
|
'step': 'build_raid', 'priority': 0, 'interface': 'deploy'}
|
|
self.service_steps = [self.deploy_update,
|
|
self.power_update,
|
|
self.deploy_magic]
|
|
|
|
def __do_node_service_validate_fail(self, mock_validate,
|
|
service_steps=None):
|
|
tgt_prov_state = states.ACTIVE
|
|
node = obj_utils.create_test_node(
|
|
self.context, driver='fake-hardware',
|
|
provision_state=states.SERVICING,
|
|
target_provision_state=tgt_prov_state)
|
|
with task_manager.acquire(
|
|
self.context, node.uuid, shared=False) as task:
|
|
servicing.do_node_service(task, service_steps=service_steps)
|
|
node.refresh()
|
|
self.assertEqual(states.SERVICEFAIL, node.provision_state)
|
|
self.assertEqual(tgt_prov_state, node.target_provision_state)
|
|
self.assertFalse(node.maintenance)
|
|
self.assertIsNone(node.fault)
|
|
mock_validate.assert_called_once_with(mock.ANY, mock.ANY)
|
|
|
|
def __do_node_service_validate_fail_invalid(self, mock_validate,
|
|
service_steps=None):
|
|
# InvalidParameterValue should cause node to go to SERVICEFAIL
|
|
mock_validate.side_effect = exception.InvalidParameterValue('error')
|
|
self.__do_node_service_validate_fail(mock_validate,
|
|
service_steps=service_steps)
|
|
|
|
@mock.patch('ironic.drivers.modules.fake.FakePower.validate',
|
|
autospec=True)
|
|
def test__do_node_service_automated_power_validate_fail(self,
|
|
mock_validate):
|
|
self.__do_node_service_validate_fail_invalid(mock_validate)
|
|
|
|
@mock.patch('ironic.drivers.modules.fake.FakePower.validate',
|
|
autospec=True)
|
|
def test__do_node_service_manual_power_validate_fail(self, mock_validate):
|
|
self.__do_node_service_validate_fail_invalid(mock_validate,
|
|
service_steps=[])
|
|
|
|
@mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
|
|
autospec=True)
|
|
def test__do_node_service_automated_network_validate_fail(self,
|
|
mock_validate):
|
|
self.__do_node_service_validate_fail_invalid(mock_validate)
|
|
|
|
@mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
|
|
autospec=True)
|
|
def test__do_node_service_manual_network_validate_fail(self,
|
|
mock_validate):
|
|
self.__do_node_service_validate_fail_invalid(mock_validate,
|
|
service_steps=[])
|
|
|
|
@mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
|
|
autospec=True)
|
|
def test__do_node_service_network_error_fail(self, mock_validate):
|
|
# NetworkError should cause node to go to CLEANFAIL
|
|
mock_validate.side_effect = exception.NetworkError()
|
|
self.__do_node_service_validate_fail(mock_validate)
|
|
|
|
@mock.patch.object(conductor_steps, 'set_node_service_steps',
|
|
autospec=True)
|
|
@mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
|
|
autospec=True)
|
|
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.prepare_service',
|
|
autospec=True)
|
|
def test__do_node_service_prepare_service_fail(self, mock_prep,
|
|
mock_validate,
|
|
mock_steps,
|
|
service_steps=[]):
|
|
# NOTE(janders) after removing unconditional initial reboot into
|
|
# ramdisk, set_node_service_steps needs to InvalidParameterValue
|
|
# to force boot into ramdisk
|
|
mock_steps.side_effect = exception.InvalidParameterValue('error')
|
|
# Exception from task.driver.deploy.prepare_service should cause node
|
|
# to go to SERVICEFAIL
|
|
mock_prep.side_effect = exception.InvalidParameterValue('error')
|
|
tgt_prov_state = states.ACTIVE
|
|
node = obj_utils.create_test_node(
|
|
self.context, driver='fake-hardware',
|
|
provision_state=states.SERVICING,
|
|
target_provision_state=tgt_prov_state)
|
|
with task_manager.acquire(
|
|
self.context, node.uuid, shared=False) as task:
|
|
servicing.do_node_service(task, service_steps=service_steps)
|
|
node.refresh()
|
|
self.assertEqual(states.SERVICEFAIL, node.provision_state)
|
|
self.assertEqual(tgt_prov_state, node.target_provision_state)
|
|
mock_prep.assert_called_once_with(mock.ANY, task)
|
|
mock_validate.assert_called_once_with(mock.ANY, task)
|
|
self.assertFalse(node.maintenance)
|
|
self.assertIsNone(node.fault)
|
|
|
|
@mock.patch.object(conductor_steps, 'set_node_service_steps',
|
|
autospec=True)
|
|
@mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
|
|
autospec=True)
|
|
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.prepare_service',
|
|
autospec=True)
|
|
def test__do_node_service_prepare_service_wait(self, mock_prep,
|
|
mock_validate,
|
|
mock_steps):
|
|
service_steps = [
|
|
{'step': 'trigger_servicewait', 'priority': 10,
|
|
'interface': 'vendor'}
|
|
]
|
|
mock_steps.side_effect = exception.InvalidParameterValue('error')
|
|
mock_prep.return_value = states.SERVICEWAIT
|
|
tgt_prov_state = states.ACTIVE
|
|
node = obj_utils.create_test_node(
|
|
self.context, driver='fake-hardware',
|
|
provision_state=states.SERVICING,
|
|
target_provision_state=tgt_prov_state,
|
|
vendor_interface='fake')
|
|
with task_manager.acquire(
|
|
self.context, node.uuid, shared=False) as task:
|
|
servicing.do_node_service(task, service_steps=service_steps)
|
|
node.refresh()
|
|
self.assertEqual(states.SERVICEWAIT, node.provision_state)
|
|
self.assertEqual(tgt_prov_state, node.target_provision_state)
|
|
mock_prep.assert_called_once_with(mock.ANY, mock.ANY)
|
|
mock_validate.assert_called_once_with(mock.ANY, mock.ANY)
|
|
|
|
@mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
|
|
autospec=True)
|
|
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.prepare_service',
|
|
autospec=True)
|
|
def test__do_node_service_out_of_band(self, mock_prep,
|
|
mock_validate):
|
|
# NOTE(janders) this test ensures ramdisk isn't prepared if
|
|
# steps do not require it
|
|
service_steps = [
|
|
{'step': 'trigger_servicewait', 'priority': 10,
|
|
'interface': 'vendor'}
|
|
]
|
|
mock_prep.return_value = states.SERVICEWAIT
|
|
tgt_prov_state = states.ACTIVE
|
|
node = obj_utils.create_test_node(
|
|
self.context, driver='fake-hardware',
|
|
provision_state=states.SERVICING,
|
|
target_provision_state=tgt_prov_state,
|
|
vendor_interface='fake')
|
|
with task_manager.acquire(
|
|
self.context, node.uuid, shared=False) as task:
|
|
servicing.do_node_service(task, service_steps=service_steps)
|
|
node.refresh()
|
|
self.assertEqual(states.SERVICEWAIT, node.provision_state)
|
|
self.assertEqual(tgt_prov_state, node.target_provision_state)
|
|
mock_prep.assert_not_called()
|
|
mock_validate.assert_called_once_with(mock.ANY, mock.ANY)
|
|
|
|
@mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
|
|
autospec=True)
|
|
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.prepare_service',
|
|
autospec=True)
|
|
def test__do_node_service_requires_ramdisk_fallback(self, mock_prep,
|
|
mock_validate):
|
|
# NOTE(janders): here we set 'requires_ramdisk': 'True'
|
|
# to force ramdisk_needed to be set and trigger prepare ramdisk
|
|
service_steps = [
|
|
{'step': 'trigger_servicewait', 'priority': 10,
|
|
'interface': 'vendor', 'requires_ramdisk': 'True'}
|
|
]
|
|
mock_prep.return_value = states.SERVICEWAIT
|
|
tgt_prov_state = states.ACTIVE
|
|
node = obj_utils.create_test_node(
|
|
self.context, driver='fake-hardware',
|
|
provision_state=states.SERVICING,
|
|
target_provision_state=tgt_prov_state,
|
|
vendor_interface='fake')
|
|
with task_manager.acquire(
|
|
self.context, node.uuid, shared=False) as task:
|
|
servicing.do_node_service(task, service_steps=service_steps)
|
|
node.refresh()
|
|
self.assertEqual(states.SERVICEWAIT, node.provision_state)
|
|
self.assertEqual(tgt_prov_state, node.target_provision_state)
|
|
mock_prep.assert_called_once_with(mock.ANY, mock.ANY)
|
|
mock_validate.assert_called_once_with(mock.ANY, mock.ANY)
|
|
|
|
@mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
|
|
autospec=True)
|
|
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.prepare_service',
|
|
autospec=True)
|
|
def test__do_node_service_prepare_service_active(self, mock_prep,
|
|
mock_validate):
|
|
service_steps = [
|
|
{'step': 'log_passthrough', 'priority': 10, 'interface': 'vendor'}
|
|
]
|
|
|
|
mock_prep.return_value = states.SERVICEWAIT
|
|
tgt_prov_state = states.ACTIVE
|
|
node = obj_utils.create_test_node(
|
|
self.context, driver='fake-hardware',
|
|
provision_state=states.SERVICING,
|
|
target_provision_state=tgt_prov_state,
|
|
vendor_interface='fake')
|
|
with task_manager.acquire(
|
|
self.context, node.uuid, shared=False) as task:
|
|
servicing.do_node_service(task, service_steps=service_steps,
|
|
disable_ramdisk=True)
|
|
# Validate we went back to active, and did not trigger a ramdisk.
|
|
node.refresh()
|
|
self.assertEqual(states.ACTIVE, node.provision_state)
|
|
self.assertEqual(states.NOSTATE, node.target_provision_state)
|
|
mock_prep.assert_not_called()
|
|
mock_validate.assert_not_called()
|
|
|
|
@mock.patch.object(n_flat.FlatNetwork, 'validate', autospec=True)
|
|
@mock.patch.object(conductor_steps, 'set_node_service_steps',
|
|
autospec=True)
|
|
def __do_node_service_steps_fail(self, mock_steps, mock_validate,
|
|
service_steps=None):
|
|
mock_steps.side_effect = exception.NodeCleaningFailure('failure')
|
|
tgt_prov_state = states.ACTIVE
|
|
node = obj_utils.create_test_node(
|
|
self.context, driver='fake-hardware',
|
|
uuid=uuidutils.generate_uuid(),
|
|
provision_state=states.SERVICING,
|
|
target_provision_state=tgt_prov_state)
|
|
with task_manager.acquire(
|
|
self.context, node.uuid, shared=False) as task:
|
|
servicing.do_node_service(task, service_steps=service_steps)
|
|
mock_validate.assert_called_once_with(mock.ANY, task)
|
|
node.refresh()
|
|
self.assertEqual(states.SERVICEFAIL, node.provision_state)
|
|
self.assertEqual(tgt_prov_state, node.target_provision_state)
|
|
mock_steps.assert_called_once_with(mock.ANY, disable_ramdisk=False)
|
|
self.assertFalse(node.maintenance)
|
|
self.assertIsNone(node.fault)
|
|
|
|
@mock.patch('ironic.drivers.modules.fake.FakePower.set_power_state',
|
|
autospec=True)
|
|
@mock.patch.object(n_flat.FlatNetwork, 'validate', autospec=True)
|
|
@mock.patch.object(conductor_steps, 'set_node_service_steps',
|
|
autospec=True)
|
|
def test_do_node_service_steps_fail_poweroff(self, mock_steps,
|
|
mock_validate,
|
|
mock_power,
|
|
service_steps=None):
|
|
mock_steps.side_effect = exception.NodeCleaningFailure('failure')
|
|
tgt_prov_state = states.ACTIVE
|
|
self.config(poweroff_in_cleanfail=True, group='conductor')
|
|
node = obj_utils.create_test_node(
|
|
self.context, driver='fake-hardware',
|
|
uuid=uuidutils.generate_uuid(),
|
|
provision_state=states.SERVICING,
|
|
power_state=states.POWER_ON,
|
|
target_provision_state=tgt_prov_state)
|
|
with task_manager.acquire(
|
|
self.context, node.uuid, shared=False) as task:
|
|
servicing.do_node_service(task, service_steps=service_steps)
|
|
mock_validate.assert_called_once_with(mock.ANY, task)
|
|
node.refresh()
|
|
self.assertEqual(states.SERVICEFAIL, node.provision_state)
|
|
self.assertEqual(tgt_prov_state, node.target_provision_state)
|
|
mock_steps.assert_called_once_with(mock.ANY, disable_ramdisk=False)
|
|
self.assertFalse(mock_power.called)
|
|
|
|
def test__do_node_service_steps_fail(self):
|
|
self.__do_node_service_steps_fail(service_steps=[self.deploy_raid])
|
|
|
|
@mock.patch.object(conductor_steps, 'set_node_service_steps',
|
|
autospec=True)
|
|
@mock.patch.object(servicing, 'do_next_service_step', autospec=True)
|
|
@mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
|
|
autospec=True)
|
|
@mock.patch('ironic.drivers.modules.fake.FakePower.validate',
|
|
autospec=True)
|
|
def __do_node_service(self, mock_power_valid, mock_network_valid,
|
|
mock_next_step, mock_steps, service_steps=None,
|
|
disable_ramdisk=False):
|
|
tgt_prov_state = states.ACTIVE
|
|
|
|
if not service_steps:
|
|
service_steps = self.service_steps
|
|
|
|
def set_steps(task, disable_ramdisk=None):
|
|
dii = task.node.driver_internal_info
|
|
dii['service_steps'] = service_steps
|
|
task.node.driver_internal_info = dii
|
|
task.node.save()
|
|
|
|
mock_steps.side_effect = set_steps
|
|
|
|
node = obj_utils.create_test_node(
|
|
self.context, driver='fake-hardware',
|
|
provision_state=states.SERVICING,
|
|
target_provision_state=tgt_prov_state,
|
|
last_error=None,
|
|
power_state=states.POWER_OFF,
|
|
driver_internal_info={'agent_secret_token': 'old'})
|
|
|
|
with task_manager.acquire(
|
|
self.context, node.uuid, shared=False) as task:
|
|
servicing.do_node_service(task, service_steps=service_steps,
|
|
disable_ramdisk=disable_ramdisk)
|
|
|
|
node.refresh()
|
|
|
|
mock_power_valid.assert_called_once_with(mock.ANY, task)
|
|
if disable_ramdisk:
|
|
mock_network_valid.assert_not_called()
|
|
else:
|
|
mock_network_valid.assert_called_once_with(mock.ANY, task)
|
|
|
|
mock_next_step.assert_called_once_with(
|
|
task, 0, disable_ramdisk=disable_ramdisk)
|
|
mock_steps.assert_called_once_with(
|
|
task, disable_ramdisk=disable_ramdisk)
|
|
if service_steps:
|
|
self.assertEqual(service_steps,
|
|
node.driver_internal_info['service_steps'])
|
|
self.assertFalse(node.maintenance)
|
|
self.assertNotIn('agent_secret_token', node.driver_internal_info)
|
|
|
|
# Check that state didn't change
|
|
self.assertEqual(states.SERVICING, node.provision_state)
|
|
self.assertEqual(tgt_prov_state, node.target_provision_state)
|
|
|
|
def test__do_node_service(self):
|
|
self.__do_node_service()
|
|
|
|
def test__do_node_service_disable_ramdisk(self):
|
|
self.__do_node_service(service_steps=[self.deploy_raid],
|
|
disable_ramdisk=True)
|
|
|
|
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_service_step',
|
|
autospec=True)
|
|
def _do_next_service_step_first_step_async(self, return_state,
|
|
mock_execute,
|
|
service_steps=None):
|
|
# Execute the first async clean step on a node
|
|
driver_internal_info = {'service_step_index': None}
|
|
tgt_prov_state = states.ACTIVE
|
|
if service_steps:
|
|
driver_internal_info['service_steps'] = service_steps
|
|
else:
|
|
driver_internal_info['service_steps'] = self.service_steps
|
|
|
|
node = obj_utils.create_test_node(
|
|
self.context, driver='fake-hardware',
|
|
provision_state=states.SERVICING,
|
|
target_provision_state=tgt_prov_state,
|
|
last_error=None,
|
|
driver_internal_info=driver_internal_info,
|
|
clean_step={})
|
|
mock_execute.return_value = return_state
|
|
expected_first_step = node.driver_internal_info['service_steps'][0]
|
|
|
|
with task_manager.acquire(
|
|
self.context, node.uuid, shared=False) as task:
|
|
servicing.do_next_service_step(task, 0)
|
|
|
|
node.refresh()
|
|
self.assertEqual(states.SERVICEWAIT, node.provision_state)
|
|
self.assertEqual(tgt_prov_state, node.target_provision_state)
|
|
self.assertEqual(expected_first_step, node.service_step)
|
|
self.assertEqual(0, node.driver_internal_info['service_step_index'])
|
|
mock_execute.assert_called_once_with(
|
|
mock.ANY, mock.ANY, expected_first_step)
|
|
|
|
def test_do_next_service_step_automated_first_step_async(self):
|
|
self._do_next_service_step_first_step_async(states.SERVICEWAIT)
|
|
|
|
def test_do_next_service_step_manual_first_step_async(self):
|
|
self._do_next_service_step_first_step_async(
|
|
states.SERVICEWAIT, service_steps=[self.deploy_raid])
|
|
|
|
@mock.patch('ironic.drivers.modules.fake.FakePower.execute_service_step',
|
|
autospec=True)
|
|
def _do_next_clean_step_continue_from_last_cleaning(self, return_state,
|
|
mock_execute,
|
|
manual=False):
|
|
# Resume an in-progress servicing after the first async step
|
|
tgt_prov_state = states.ACTIVE
|
|
|
|
node = obj_utils.create_test_node(
|
|
self.context, driver='fake-hardware',
|
|
provision_state=states.SERVICING,
|
|
target_provision_state=tgt_prov_state,
|
|
last_error=None,
|
|
driver_internal_info={'service_steps': self.service_steps,
|
|
'service_step_index': 0,
|
|
'servicing_polling': True},
|
|
service_step=self.service_steps[0])
|
|
mock_execute.return_value = return_state
|
|
|
|
with task_manager.acquire(
|
|
self.context, node.uuid, shared=False) as task:
|
|
servicing.do_next_service_step(task, self.next_service_step_index)
|
|
|
|
node.refresh()
|
|
|
|
self.assertEqual(states.SERVICEWAIT, node.provision_state)
|
|
self.assertEqual(tgt_prov_state, node.target_provision_state)
|
|
self.assertEqual(self.service_steps[1], node.service_step)
|
|
self.assertEqual(1, node.driver_internal_info['service_step_index'])
|
|
mock_execute.assert_called_once_with(
|
|
mock.ANY, mock.ANY, self.service_steps[1])
|
|
self.assertNotIn('servicing_polling', node.driver_internal_info)
|
|
|
|
def test_do_next_clean_step_continue_from_last_cleaning(self):
|
|
self._do_next_clean_step_continue_from_last_cleaning(
|
|
states.SERVICEWAIT)
|
|
|
|
def test_do_next_clean_step_manual_continue_from_last_cleaning(self):
|
|
self._do_next_clean_step_continue_from_last_cleaning(
|
|
states.SERVICEWAIT, manual=True)
|
|
|
|
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_service_step',
|
|
autospec=True)
|
|
def _do_next_service_step_last_step_noop(self, mock_execute,
|
|
manual=False):
|
|
# Resume where last_step is the last cleaning step, should be noop
|
|
tgt_prov_state = states.ACTIVE
|
|
info = {'service_steps': self.service_steps,
|
|
'service_step_index': len(self.service_steps) - 1,
|
|
'agent_url': 'test-url',
|
|
'agent_secret_token': 'token'}
|
|
|
|
node = obj_utils.create_test_node(
|
|
self.context, driver='fake-hardware',
|
|
provision_state=states.SERVICING,
|
|
target_provision_state=tgt_prov_state,
|
|
last_error=None,
|
|
driver_internal_info=info,
|
|
service_step=self.service_steps[-1])
|
|
|
|
with task_manager.acquire(
|
|
self.context, node.uuid, shared=False) as task:
|
|
servicing.do_next_service_step(task, None)
|
|
|
|
node.refresh()
|
|
|
|
# Cleaning should be complete without calling additional steps
|
|
self.assertEqual(tgt_prov_state, node.provision_state)
|
|
self.assertEqual(states.NOSTATE, node.target_provision_state)
|
|
self.assertEqual({}, node.clean_step)
|
|
self.assertNotIn('service_step_index', node.driver_internal_info)
|
|
self.assertIsNone(node.driver_internal_info['service_steps'])
|
|
self.assertFalse(mock_execute.called)
|
|
self.assertNotIn('agent_url', node.driver_internal_info)
|
|
self.assertNotIn('agent_secret_token',
|
|
node.driver_internal_info)
|
|
|
|
def test__do_next_service_step_automated_last_step_noop(self):
|
|
self._do_next_service_step_last_step_noop()
|
|
|
|
def test__do_next_service_step_manual_last_step_noop(self):
|
|
self._do_next_service_step_last_step_noop(manual=True)
|
|
|
|
@mock.patch('ironic.drivers.utils.collect_ramdisk_logs', autospec=True)
|
|
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.tear_down_service',
|
|
autospec=True)
|
|
@mock.patch('ironic.drivers.modules.fake.FakePower.execute_service_step',
|
|
autospec=True)
|
|
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_service_step',
|
|
autospec=True)
|
|
def _do_next_service_step_all(self, mock_deploy_execute,
|
|
mock_power_execute, mock_tear_down,
|
|
mock_collect_logs,
|
|
disable_ramdisk=False):
|
|
# Run all steps from start to finish (all synchronous)
|
|
tgt_prov_state = states.ACTIVE
|
|
|
|
node = obj_utils.create_test_node(
|
|
self.context, driver='fake-hardware',
|
|
provision_state=states.SERVICING,
|
|
target_provision_state=tgt_prov_state,
|
|
last_error=None,
|
|
driver_internal_info={'service_steps': self.service_steps,
|
|
'service_step_index': None},
|
|
clean_step={})
|
|
|
|
def fake_deploy(conductor_obj, task, step):
|
|
driver_internal_info = task.node.driver_internal_info
|
|
driver_internal_info['goober'] = 'test'
|
|
task.node.driver_internal_info = driver_internal_info
|
|
task.node.save()
|
|
|
|
mock_deploy_execute.side_effect = fake_deploy
|
|
mock_power_execute.return_value = None
|
|
|
|
with task_manager.acquire(
|
|
self.context, node.uuid, shared=False) as task:
|
|
servicing.do_next_service_step(
|
|
task, 0, disable_ramdisk=disable_ramdisk)
|
|
|
|
mock_power_execute.assert_called_once_with(task.driver.power, task,
|
|
self.service_steps[1])
|
|
mock_deploy_execute.assert_has_calls(
|
|
[mock.call(task.driver.deploy, task, self.service_steps[0]),
|
|
mock.call(task.driver.deploy, task, self.service_steps[2])])
|
|
if disable_ramdisk:
|
|
mock_tear_down.assert_not_called()
|
|
else:
|
|
mock_tear_down.assert_called_once_with(
|
|
task.driver.deploy, task)
|
|
|
|
node.refresh()
|
|
|
|
# Servicing should be complete
|
|
self.assertEqual(tgt_prov_state, node.provision_state)
|
|
self.assertEqual(states.NOSTATE, node.target_provision_state)
|
|
self.assertEqual({}, node.service_step)
|
|
self.assertNotIn('service_step_index', node.driver_internal_info)
|
|
self.assertEqual('test', node.driver_internal_info['goober'])
|
|
self.assertIsNone(node.driver_internal_info['service_steps'])
|
|
self.assertFalse(mock_collect_logs.called)
|
|
|
|
def test_do_next_clean_step_all(self):
|
|
self._do_next_service_step_all()
|
|
|
|
def test_do_next_clean_step_all_disable_ramdisk(self):
|
|
self._do_next_service_step_all(disable_ramdisk=True)
|
|
|
|
@mock.patch('ironic.drivers.utils.collect_ramdisk_logs', autospec=True)
|
|
@mock.patch('ironic.drivers.modules.fake.FakePower.execute_service_step',
|
|
autospec=True)
|
|
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_service_step',
|
|
autospec=True)
|
|
def test_do_next_clean_step_collect_logs(self, mock_deploy_execute,
|
|
mock_power_execute,
|
|
mock_collect_logs):
|
|
CONF.set_override('deploy_logs_collect', 'always', group='agent')
|
|
# Run all steps from start to finish (all synchronous)
|
|
tgt_prov_state = states.ACTIVE
|
|
|
|
node = obj_utils.create_test_node(
|
|
self.context, driver='fake-hardware',
|
|
provision_state=states.SERVICING,
|
|
target_provision_state=tgt_prov_state,
|
|
last_error=None,
|
|
driver_internal_info={'service_steps': self.service_steps,
|
|
'service_step_index': None},
|
|
clean_step={})
|
|
|
|
def fake_deploy(conductor_obj, task, step):
|
|
driver_internal_info = task.node.driver_internal_info
|
|
driver_internal_info['goober'] = 'test'
|
|
task.node.driver_internal_info = driver_internal_info
|
|
task.node.save()
|
|
|
|
mock_deploy_execute.side_effect = fake_deploy
|
|
mock_power_execute.return_value = None
|
|
|
|
with task_manager.acquire(
|
|
self.context, node.uuid, shared=False) as task:
|
|
servicing.do_next_service_step(task, 0)
|
|
|
|
node.refresh()
|
|
|
|
# Cleaning should be complete
|
|
self.assertEqual(tgt_prov_state, node.provision_state)
|
|
self.assertEqual(states.NOSTATE, node.target_provision_state)
|
|
self.assertEqual({}, node.clean_step)
|
|
self.assertNotIn('service_step_index', node.driver_internal_info)
|
|
self.assertEqual('test', node.driver_internal_info['goober'])
|
|
self.assertIsNone(node.driver_internal_info['service_steps'])
|
|
mock_power_execute.assert_called_once_with(mock.ANY, mock.ANY,
|
|
self.service_steps[1])
|
|
mock_deploy_execute.assert_has_calls(
|
|
[mock.call(mock.ANY, mock.ANY, self.service_steps[0]),
|
|
mock.call(mock.ANY, mock.ANY, self.service_steps[2])])
|
|
mock_collect_logs.assert_called_once_with(mock.ANY, label='service')
|
|
|
|
@mock.patch('ironic.drivers.utils.collect_ramdisk_logs', autospec=True)
|
|
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_service_step',
|
|
autospec=True)
|
|
@mock.patch.object(fake.FakeDeploy, 'tear_down_service', autospec=True)
|
|
def _do_next_service_step_execute_fail(self, tear_mock, mock_execute,
|
|
mock_collect_logs):
|
|
# When a clean step fails, go to CLEANFAIL
|
|
tgt_prov_state = states.ACTIVE
|
|
|
|
node = obj_utils.create_test_node(
|
|
self.context, driver='fake-hardware',
|
|
provision_state=states.SERVICING,
|
|
target_provision_state=tgt_prov_state,
|
|
last_error=None,
|
|
driver_internal_info={'service_steps': self.service_steps,
|
|
'service_step_index': None},
|
|
clean_step={})
|
|
mock_execute.side_effect = Exception()
|
|
|
|
with task_manager.acquire(
|
|
self.context, node.uuid, shared=False) as task:
|
|
servicing.do_next_service_step(task, 0)
|
|
tear_mock.assert_called_once_with(task.driver.deploy, task)
|
|
|
|
node.refresh()
|
|
|
|
# Make sure we go to SERVICEFAIL, clear service_steps
|
|
self.assertEqual(states.SERVICEFAIL, node.provision_state)
|
|
self.assertEqual(tgt_prov_state, node.target_provision_state)
|
|
self.assertEqual({}, node.service_step)
|
|
self.assertNotIn('service_step_index', node.driver_internal_info)
|
|
self.assertIsNotNone(node.last_error)
|
|
self.assertTrue(node.maintenance)
|
|
self.assertEqual(faults.SERVICE_FAILURE, node.fault)
|
|
mock_execute.assert_called_once_with(
|
|
mock.ANY, mock.ANY, self.service_steps[0])
|
|
mock_collect_logs.assert_called_once_with(mock.ANY, label='service')
|
|
|
|
def test__do_next_clean_step_automated_execute_fail(self):
|
|
self._do_next_service_step_execute_fail()
|
|
|
|
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_service_step',
|
|
autospec=True)
|
|
def test_do_next_service_step_oob_reboot(self, mock_execute):
|
|
tgt_prov_state = states.ACTIVE
|
|
|
|
node = obj_utils.create_test_node(
|
|
self.context, driver='fake-hardware',
|
|
provision_state=states.SERVICING,
|
|
target_provision_state=tgt_prov_state,
|
|
last_error=None,
|
|
driver_internal_info={'service_steps': self.service_steps,
|
|
'service_step_index': None,
|
|
'servicing_reboot': True},
|
|
service_step={})
|
|
mock_execute.side_effect = exception.AgentConnectionFailed(
|
|
reason='failed')
|
|
|
|
with task_manager.acquire(
|
|
self.context, node.uuid, shared=False) as task:
|
|
servicing.do_next_service_step(task, 0)
|
|
|
|
node.refresh()
|
|
|
|
# Make sure we go to SERVICEWAIT
|
|
self.assertEqual(states.SERVICEWAIT, node.provision_state)
|
|
self.assertEqual(tgt_prov_state, node.target_provision_state)
|
|
self.assertEqual(self.service_steps[0], node.service_step)
|
|
self.assertEqual(0, node.driver_internal_info['service_step_index'])
|
|
self.assertFalse(
|
|
node.driver_internal_info['skip_current_service_step'])
|
|
mock_execute.assert_called_once_with(
|
|
mock.ANY, mock.ANY, self.service_steps[0])
|
|
|
|
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_service_step',
|
|
autospec=True)
|
|
def test_do_next_service_step_agent_busy(self, mock_execute):
|
|
tgt_prov_state = states.ACTIVE
|
|
|
|
node = obj_utils.create_test_node(
|
|
self.context, driver='fake-hardware',
|
|
provision_state=states.SERVICING,
|
|
target_provision_state=tgt_prov_state,
|
|
last_error=None,
|
|
driver_internal_info={'service_steps': self.service_steps,
|
|
'service_step_index': None,
|
|
'servicing_reboot': True},
|
|
service_step={})
|
|
mock_execute.side_effect = exception.AgentInProgress(
|
|
reason='still meowing')
|
|
with task_manager.acquire(
|
|
self.context, node.uuid, shared=False) as task:
|
|
servicing.do_next_service_step(task, 0)
|
|
|
|
node.refresh()
|
|
# Make sure we go to SERVICEWAIT
|
|
self.assertEqual(states.SERVICEWAIT, node.provision_state)
|
|
self.assertEqual(tgt_prov_state, node.target_provision_state)
|
|
self.assertEqual(self.service_steps[0], node.service_step)
|
|
self.assertEqual(0, node.driver_internal_info['service_step_index'])
|
|
self.assertFalse(
|
|
node.driver_internal_info['skip_current_service_step'])
|
|
mock_execute.assert_called_once_with(
|
|
mock.ANY, mock.ANY, self.service_steps[0])
|
|
|
|
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_service_step',
|
|
autospec=True)
|
|
def test_do_next_service_step_oob_reboot_last_step(self, mock_execute):
|
|
# Resume where last_step is the last service step
|
|
tgt_prov_state = states.ACTIVE
|
|
info = {'service_steps': self.service_steps,
|
|
'servicing_reboot': True,
|
|
'service_step_index': len(self.service_steps) - 1}
|
|
|
|
node = obj_utils.create_test_node(
|
|
self.context, driver='fake-hardware',
|
|
provision_state=states.SERVICING,
|
|
target_provision_state=tgt_prov_state,
|
|
last_error=None,
|
|
driver_internal_info=info,
|
|
service_step=self.service_steps[-1])
|
|
|
|
with task_manager.acquire(
|
|
self.context, node.uuid, shared=False) as task:
|
|
servicing.do_next_service_step(task, None)
|
|
|
|
node.refresh()
|
|
|
|
# Servicing should be complete without calling additional steps
|
|
self.assertEqual(tgt_prov_state, node.provision_state)
|
|
self.assertEqual(states.NOSTATE, node.target_provision_state)
|
|
self.assertEqual({}, node.service_step)
|
|
self.assertNotIn('service_step_index', node.driver_internal_info)
|
|
self.assertNotIn('servicing_reboot', node.driver_internal_info)
|
|
self.assertIsNone(node.driver_internal_info['service_steps'])
|
|
self.assertFalse(mock_execute.called)
|
|
|
|
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_service_step',
|
|
autospec=True)
|
|
@mock.patch.object(fake.FakeDeploy, 'tear_down_service', autospec=True)
|
|
def test_do_next_service_step_oob_reboot_fail(self, tear_mock,
|
|
mock_execute):
|
|
# When a service step fails with no reboot requested go to SERVICEFAIL
|
|
tgt_prov_state = states.ACTIVE
|
|
|
|
node = obj_utils.create_test_node(
|
|
self.context, driver='fake-hardware',
|
|
provision_state=states.SERVICING,
|
|
target_provision_state=tgt_prov_state,
|
|
last_error=None,
|
|
driver_internal_info={'service_steps': self.service_steps,
|
|
'service_step_index': None},
|
|
service_step={})
|
|
mock_execute.side_effect = exception.AgentConnectionFailed(
|
|
reason='failed')
|
|
|
|
with task_manager.acquire(
|
|
self.context, node.uuid, shared=False) as task:
|
|
servicing.do_next_service_step(task, 0)
|
|
tear_mock.assert_called_once_with(task.driver.deploy, task)
|
|
|
|
node.refresh()
|
|
|
|
# Make sure we go to SERVICEFAIL, clear service_steps
|
|
self.assertEqual(states.SERVICEFAIL, node.provision_state)
|
|
self.assertEqual(tgt_prov_state, node.target_provision_state)
|
|
self.assertEqual({}, node.service_step)
|
|
self.assertNotIn('service_step_index', node.driver_internal_info)
|
|
self.assertNotIn('skip_current_service_step',
|
|
node.driver_internal_info)
|
|
self.assertIsNotNone(node.last_error)
|
|
self.assertTrue(node.maintenance)
|
|
mock_execute.assert_called_once_with(
|
|
mock.ANY, mock.ANY, self.service_steps[0])
|
|
|
|
@mock.patch.object(conductor_utils, 'LOG', autospec=True)
|
|
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_service_step',
|
|
autospec=True)
|
|
@mock.patch('ironic.drivers.modules.fake.FakePower.execute_service_step',
|
|
autospec=True)
|
|
@mock.patch.object(fake.FakeDeploy, 'tear_down_service', autospec=True)
|
|
def _do_next_service_step_fail_in_tear_down_service(
|
|
self, tear_mock, power_exec_mock, deploy_exec_mock, log_mock):
|
|
tgt_prov_state = states.ACTIVE
|
|
|
|
node = obj_utils.create_test_node(
|
|
self.context, driver='fake-hardware',
|
|
provision_state=states.SERVICING,
|
|
target_provision_state=tgt_prov_state,
|
|
last_error=None,
|
|
driver_internal_info={'service_steps': self.service_steps,
|
|
'service_step_index': None},
|
|
service_step={})
|
|
|
|
deploy_exec_mock.return_value = None
|
|
power_exec_mock.return_value = None
|
|
tear_mock.side_effect = Exception('boom')
|
|
|
|
with task_manager.acquire(
|
|
self.context, node.uuid, shared=False) as task:
|
|
servicing.do_next_service_step(task, 0)
|
|
|
|
node.refresh()
|
|
|
|
# Make sure we go to SERVICEFAIL, clear service_steps
|
|
self.assertEqual(states.SERVICEFAIL, node.provision_state)
|
|
self.assertEqual(tgt_prov_state, node.target_provision_state)
|
|
self.assertEqual({}, node.service_step)
|
|
self.assertNotIn('clean_step_index', node.driver_internal_info)
|
|
self.assertIsNotNone(node.last_error)
|
|
self.assertEqual(1, tear_mock.call_count)
|
|
self.assertFalse(node.maintenance) # no step is running
|
|
deploy_exec_calls = [
|
|
mock.call(mock.ANY, mock.ANY, self.service_steps[0]),
|
|
mock.call(mock.ANY, mock.ANY, self.service_steps[2]),
|
|
]
|
|
self.assertEqual(deploy_exec_calls, deploy_exec_mock.call_args_list)
|
|
|
|
power_exec_calls = [
|
|
mock.call(mock.ANY, mock.ANY, self.service_steps[1]),
|
|
]
|
|
self.assertEqual(power_exec_calls, power_exec_mock.call_args_list)
|
|
log_mock.error.assert_called_once_with(
|
|
'Failed to tear down from service for node {}, reason: boom'
|
|
.format(node.uuid), exc_info=True)
|
|
|
|
def test__do_next_service_step_automated_fail_in_tear_down_service(self):
|
|
self._do_next_service_step_fail_in_tear_down_service()
|
|
|
|
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_service_step',
|
|
autospec=True)
|
|
def _do_next_service_step_no_steps(self, mock_execute, manual=False):
|
|
for info in ({'service_steps': None, 'service_step_index': None,
|
|
'agent_url': 'test-url', 'agent_secret_token': 'magic'},
|
|
{'service_steps': None, 'agent_url': 'test-url',
|
|
'agent_secret_token': 'it_is_a_kind_of_magic'}):
|
|
# Resume where there are no steps, should be a noop
|
|
tgt_prov_state = states.ACTIVE
|
|
|
|
node = obj_utils.create_test_node(
|
|
self.context, driver='fake-hardware',
|
|
uuid=uuidutils.generate_uuid(),
|
|
provision_state=states.SERVICING,
|
|
target_provision_state=tgt_prov_state,
|
|
last_error=None,
|
|
driver_internal_info=info,
|
|
service_step={})
|
|
|
|
with task_manager.acquire(
|
|
self.context, node.uuid, shared=False) as task:
|
|
servicing.do_next_service_step(task, None)
|
|
|
|
node.refresh()
|
|
|
|
# Cleaning should be complete without calling additional steps
|
|
self.assertEqual(tgt_prov_state, node.provision_state)
|
|
self.assertEqual(states.NOSTATE, node.target_provision_state)
|
|
self.assertEqual({}, node.clean_step)
|
|
self.assertNotIn('service_step_index', node.driver_internal_info)
|
|
self.assertFalse(mock_execute.called)
|
|
self.assertNotIn('agent_url', node.driver_internal_info)
|
|
self.assertNotIn('agent_secret_token',
|
|
node.driver_internal_info)
|
|
mock_execute.reset_mock()
|
|
|
|
def test__do_next_service_step_automated_no_steps(self):
|
|
self._do_next_service_step_no_steps()
|
|
|
|
@mock.patch('ironic.drivers.modules.fake.FakePower.execute_service_step',
|
|
autospec=True)
|
|
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_service_step',
|
|
autospec=True)
|
|
def test__do_next_service_step_bad_step_return_value(
|
|
self, deploy_exec_mock, power_exec_mock, manual=False):
|
|
# When a service step fails, go to CLEANFAIL
|
|
tgt_prov_state = states.ACTIVE
|
|
|
|
node = obj_utils.create_test_node(
|
|
self.context, driver='fake-hardware',
|
|
provision_state=states.SERVICING,
|
|
target_provision_state=tgt_prov_state,
|
|
last_error=None,
|
|
driver_internal_info={'service_steps': self.service_steps,
|
|
'service_step_index': None},
|
|
service_step={})
|
|
deploy_exec_mock.return_value = "foo"
|
|
|
|
with task_manager.acquire(
|
|
self.context, node.uuid, shared=False) as task:
|
|
servicing.do_next_service_step(task, 0)
|
|
|
|
node.refresh()
|
|
|
|
# Make sure we go to SERVICEFAIL, clear service_steps
|
|
self.assertEqual(states.SERVICEFAIL, node.provision_state)
|
|
self.assertEqual(tgt_prov_state, node.target_provision_state)
|
|
self.assertEqual({}, node.service_step)
|
|
self.assertNotIn('service_step_index', node.driver_internal_info)
|
|
self.assertIsNotNone(node.last_error)
|
|
self.assertTrue(node.maintenance) # the 1st clean step was running
|
|
deploy_exec_mock.assert_called_once_with(mock.ANY, mock.ANY,
|
|
self.service_steps[0])
|
|
# Make sure we don't execute any other step and return
|
|
self.assertFalse(power_exec_mock.called)
|
|
|
|
def _test_do_next_service_step_handles_hold(self, start_state):
|
|
node = obj_utils.create_test_node(
|
|
self.context, driver='fake-hardware',
|
|
provision_state=start_state,
|
|
driver_internal_info={
|
|
'service_steps': [
|
|
{
|
|
'step': 'hold',
|
|
'priority': 10,
|
|
'interface': 'power'
|
|
}
|
|
],
|
|
'service_step_index': None},
|
|
service_step=None)
|
|
|
|
with task_manager.acquire(
|
|
self.context, node.uuid, shared=False) as task:
|
|
servicing.do_next_service_step(task, 0)
|
|
node.refresh()
|
|
self.assertEqual(states.SERVICEHOLD, node.provision_state)
|
|
|
|
def test_do_next_service_step_handles_hold_from_active(self):
|
|
# Start is from the conductor
|
|
self._test_do_next_service_step_handles_hold(states.SERVICING)
|
|
|
|
def test_do_next_service_step_handles_hold_from_wait(self):
|
|
# Start is the continuation from a heartbeat.
|
|
self._test_do_next_service_step_handles_hold(states.SERVICEWAIT)
|
|
|
|
def test_do_next_service_step_handles_hold_from_failed(self):
|
|
# Test that hold step from SERVICEFAIL transitions to SERVICEHOLD
|
|
self._test_do_next_service_step_handles_hold(states.SERVICEFAIL)
|
|
|
|
def test_do_next_service_step_handles_wait_from_failed(self):
|
|
# Test that wait step from SERVICEFAIL transitions to SERVICEWAIT
|
|
node = obj_utils.create_test_node(
|
|
self.context, driver='fake-hardware',
|
|
provision_state=states.SERVICEFAIL,
|
|
driver_internal_info={
|
|
'service_steps': [
|
|
{
|
|
'step': 'wait',
|
|
'priority': 10,
|
|
'interface': 'power'
|
|
}
|
|
],
|
|
'service_step_index': None},
|
|
service_step=None)
|
|
|
|
with task_manager.acquire(
|
|
self.context, node.uuid, shared=False) as task:
|
|
servicing.do_next_service_step(task, 0)
|
|
node.refresh()
|
|
self.assertEqual(states.SERVICEWAIT, node.provision_state)
|
|
|
|
@mock.patch.object(servicing, 'do_next_service_step', autospec=True)
|
|
def _continue_node_service(self, mock_next_step, skip=True):
|
|
# test that skipping current step mechanism works
|
|
driver_info = {'service_steps': self.service_steps,
|
|
'service_step_index': 0,
|
|
'servicing_polling': 'value'}
|
|
if not skip:
|
|
driver_info['skip_current_service_step'] = skip
|
|
node = obj_utils.create_test_node(
|
|
self.context, driver='fake-hardware',
|
|
provision_state=states.SERVICING,
|
|
target_provision_state=states.ACTIVE,
|
|
driver_internal_info=driver_info,
|
|
service_step=self.service_steps[0])
|
|
with task_manager.acquire(self.context, node.uuid) as task:
|
|
servicing.continue_node_service(task)
|
|
expected_step_index = 1 if skip else 0
|
|
self.assertNotIn(
|
|
'skip_current_service_step', task.node.driver_internal_info)
|
|
self.assertNotIn(
|
|
'cleaning_polling', task.node.driver_internal_info)
|
|
mock_next_step.assert_called_once_with(task, expected_step_index)
|
|
|
|
def test_continue_node_service(self):
|
|
self._continue_node_service(skip=True)
|
|
|
|
def test_continue_node_service_no_skip_step(self):
|
|
self._continue_node_service(skip=False)
|
|
|
|
|
|
class DoNodeServiceAbortTestCase(db_base.DbTestCase):
|
|
@mock.patch.object(fake.FakeDeploy, 'tear_down_service', autospec=True)
|
|
def _test_do_node_service_abort(self, service_step,
|
|
tear_mock=None):
|
|
node = obj_utils.create_test_node(
|
|
self.context, driver='fake-hardware',
|
|
provision_state=states.SERVICEWAIT,
|
|
target_provision_state=states.AVAILABLE,
|
|
service_step=service_step,
|
|
driver_internal_info={
|
|
'agent_url': 'some url',
|
|
'agent_secret_token': 'token',
|
|
'service_step_index': 2,
|
|
'servicing_reboot': True,
|
|
'servicing_polling': True,
|
|
'skip_current_service_step': True})
|
|
|
|
with task_manager.acquire(self.context, node.uuid) as task:
|
|
servicing.do_node_service_abort(task)
|
|
self.assertIsNotNone(task.node.last_error)
|
|
tear_mock.assert_called_once_with(task.driver.deploy, task)
|
|
task.node.refresh()
|
|
if service_step:
|
|
self.assertIn(service_step['step'], task.node.last_error)
|
|
# assert node's clean_step and metadata was cleaned up
|
|
self.assertEqual({}, task.node.service_step)
|
|
self.assertNotIn('service_step_index',
|
|
task.node.driver_internal_info)
|
|
self.assertNotIn('servicing_reboot',
|
|
task.node.driver_internal_info)
|
|
self.assertNotIn('servicing_polling',
|
|
task.node.driver_internal_info)
|
|
self.assertNotIn('skip_current_service_step',
|
|
task.node.driver_internal_info)
|
|
self.assertNotIn('agent_url',
|
|
task.node.driver_internal_info)
|
|
self.assertNotIn('agent_secret_token',
|
|
task.node.driver_internal_info)
|
|
|
|
def test_do_node_service_abort_early(self):
|
|
self._test_do_node_service_abort(None)
|
|
|
|
def test_do_node_service_abort_with_step(self):
|
|
self._test_do_node_service_abort({'step': 'foo', 'interface': 'deploy',
|
|
'abortable': True})
|
|
|
|
@mock.patch.object(fake.FakeDeploy, 'tear_down_service', autospec=True)
|
|
def test__do_node_service_abort_tear_down_fail(self, tear_mock):
|
|
tear_mock.side_effect = Exception('Surprise')
|
|
|
|
node = obj_utils.create_test_node(
|
|
self.context, driver='fake-hardware',
|
|
provision_state=states.SERVICEFAIL,
|
|
target_provision_state=states.ACTIVE,
|
|
service_step={'step': 'foo', 'abortable': True})
|
|
|
|
with task_manager.acquire(self.context, node.uuid) as task:
|
|
servicing.do_node_service_abort(task)
|
|
tear_mock.assert_called_once_with(task.driver.deploy, task)
|
|
self.assertIsNotNone(task.node.last_error)
|
|
self.assertIsNotNone(task.node.maintenance_reason)
|
|
self.assertTrue(task.node.maintenance)
|
|
self.assertEqual('service failure', task.node.fault)
|
|
|
|
@mock.patch.object(fake.FakeDeploy, 'tear_down_service', autospec=True)
|
|
def test__do_node_cleanhold_abort_tear_down_fail(self, tear_mock):
|
|
tear_mock.side_effect = Exception('Surprise')
|
|
|
|
node = obj_utils.create_test_node(
|
|
self.context, driver='fake-hardware',
|
|
provision_state=states.SERVICEHOLD,
|
|
target_provision_state=states.ACTIVE,
|
|
service_step={'step': 'hold', 'abortable': True})
|
|
|
|
with task_manager.acquire(self.context, node.uuid) as task:
|
|
servicing.do_node_service_abort(task)
|
|
tear_mock.assert_called_once_with(task.driver.deploy, task)
|
|
self.assertIsNotNone(task.node.last_error)
|
|
self.assertIsNotNone(task.node.maintenance_reason)
|
|
self.assertTrue(task.node.maintenance)
|
|
self.assertEqual('service failure', task.node.fault)
|
|
|
|
|
|
class DoNodeCleanTestChildNodes(db_base.DbTestCase):
|
|
def setUp(self):
|
|
super(DoNodeCleanTestChildNodes, self).setUp()
|
|
self.power_on_parent = {
|
|
'step': 'power_on', 'priority': 4, 'interface': 'power'}
|
|
self.power_on_children = {
|
|
'step': 'power_on', 'priority': 5, 'interface': 'power',
|
|
'execute_on_child_nodes': True}
|
|
self.update_firmware_on_children = {
|
|
'step': 'update_firmware', 'priority': 10,
|
|
'interface': 'management', 'execute_on_child_nodes': True}
|
|
self.reboot_children = {
|
|
'step': 'reboot', 'priority': 5, 'interface': 'power',
|
|
'execute_on_child_nodes': True}
|
|
self.power_off_children = {
|
|
'step': 'power_off', 'priority': 15, 'interface': 'power',
|
|
'execute_on_child_nodes': True}
|
|
self.service_steps = [
|
|
self.power_on_parent,
|
|
self.power_on_children,
|
|
self.update_firmware_on_children,
|
|
self.reboot_children,
|
|
self.power_off_children]
|
|
self.node = obj_utils.create_test_node(
|
|
self.context, driver='fake-hardware',
|
|
provision_state=states.SERVICING,
|
|
target_provision_state=states.ACTIVE,
|
|
last_error=None,
|
|
power_state=states.POWER_ON,
|
|
driver_internal_info={'agent_secret_token': 'old',
|
|
'service_steps': self.service_steps})
|
|
|
|
@mock.patch('ironic.drivers.modules.fake.FakePower.get_power_state',
|
|
autospec=True)
|
|
@mock.patch('ironic.drivers.modules.fake.FakePower.reboot',
|
|
autospec=True)
|
|
@mock.patch('ironic.drivers.modules.fake.FakePower.set_power_state',
|
|
autospec=True)
|
|
@mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
|
|
autospec=True)
|
|
@mock.patch('ironic.drivers.modules.fake.FakePower.validate',
|
|
autospec=True)
|
|
@mock.patch('ironic.drivers.modules.fake.FakePower.execute_service_step',
|
|
autospec=True)
|
|
@mock.patch('ironic.drivers.modules.fake.FakeManagement.'
|
|
'execute_service_step', autospec=True)
|
|
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_service_step',
|
|
autospec=True)
|
|
def test_do_next_clean_step_with_children(
|
|
self, mock_deploy, mock_mgmt, mock_power, mock_pv, mock_nv,
|
|
mock_sps, mock_reboot, mock_gps):
|
|
child_node1 = obj_utils.create_test_node(
|
|
self.context,
|
|
uuid=uuidutils.generate_uuid(),
|
|
driver='fake-hardware',
|
|
last_error=None,
|
|
power_state=states.POWER_OFF,
|
|
parent_node=self.node.uuid)
|
|
child_node2 = obj_utils.create_test_node(
|
|
self.context,
|
|
uuid=uuidutils.generate_uuid(),
|
|
driver='fake-hardware',
|
|
last_error=None,
|
|
power_state=states.POWER_OFF,
|
|
parent_node=self.node.uuid)
|
|
|
|
mock_gps.side_effect = [
|
|
states.POWER_OFF,
|
|
states.POWER_ON,
|
|
states.POWER_OFF,
|
|
states.POWER_ON,
|
|
states.POWER_OFF,
|
|
states.POWER_ON]
|
|
mock_deploy.return_value = None
|
|
mock_mgmt.return_value = None
|
|
mock_power.return_value = None
|
|
child1_updated_at = str(child_node1.updated_at)
|
|
child2_updated_at = str(child_node2.updated_at)
|
|
with task_manager.acquire(
|
|
self.context, self.node.uuid, shared=False) as task:
|
|
|
|
servicing.do_next_service_step(task, 0,
|
|
disable_ramdisk=True)
|
|
self.node.refresh()
|
|
child_node1.refresh()
|
|
child_node2.refresh()
|
|
|
|
# Confirm the objects *did* receive locks.
|
|
self.assertNotEqual(child1_updated_at, child_node1.updated_at)
|
|
self.assertNotEqual(child2_updated_at, child_node2.updated_at)
|
|
|
|
# Confirm the child nodes have no errors
|
|
self.assertFalse(child_node1.maintenance)
|
|
self.assertFalse(child_node2.maintenance)
|
|
self.assertIsNone(child_node1.last_error)
|
|
self.assertIsNone(child_node2.last_error)
|
|
self.assertIsNone(self.node.last_error)
|
|
|
|
# Confirm the call counts expected
|
|
self.assertEqual(0, mock_deploy.call_count)
|
|
self.assertEqual(2, mock_mgmt.call_count)
|
|
self.assertEqual(0, mock_power.call_count)
|
|
self.assertEqual(0, mock_nv.call_count)
|
|
self.assertEqual(0, mock_pv.call_count)
|
|
self.assertEqual(3, mock_sps.call_count)
|
|
self.assertEqual(2, mock_reboot.call_count)
|
|
mock_sps.assert_has_calls([
|
|
mock.call(mock.ANY, mock.ANY, 'power on', timeout=None),
|
|
mock.call(mock.ANY, mock.ANY, 'power on', timeout=None),
|
|
mock.call(mock.ANY, mock.ANY, 'power off', timeout=None)])
|
|
|
|
@mock.patch('ironic.drivers.modules.fake.FakePower.set_power_state',
|
|
autospec=True)
|
|
@mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
|
|
autospec=True)
|
|
@mock.patch('ironic.drivers.modules.fake.FakePower.validate',
|
|
autospec=True)
|
|
@mock.patch('ironic.drivers.modules.fake.FakePower.execute_service_step',
|
|
autospec=True)
|
|
@mock.patch('ironic.drivers.modules.fake.FakeManagement.'
|
|
'execute_service_step', autospec=True)
|
|
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_service_step',
|
|
autospec=True)
|
|
def test_do_next_clean_step_with_children_by_uuid(
|
|
self, mock_deploy, mock_mgmt, mock_power, mock_pv, mock_nv,
|
|
mock_sps):
|
|
child_node1 = obj_utils.create_test_node(
|
|
self.context,
|
|
uuid=uuidutils.generate_uuid(),
|
|
driver='fake-hardware',
|
|
last_error=None,
|
|
parent_node=self.node.uuid)
|
|
child_node2 = obj_utils.create_test_node(
|
|
self.context,
|
|
uuid=uuidutils.generate_uuid(),
|
|
driver='fake-hardware',
|
|
last_error=None,
|
|
parent_node=self.node.uuid)
|
|
power_on_children = {
|
|
'step': 'power_on', 'priority': 5, 'interface': 'power',
|
|
'execute_on_child_nodes': True,
|
|
'limit_child_node_execution': [child_node1.uuid]}
|
|
update_firmware_on_children = {
|
|
'step': 'update_firmware', 'priority': 10,
|
|
'interface': 'management',
|
|
'execute_on_child_nodes': True,
|
|
'limit_child_node_execution': [child_node1.uuid]}
|
|
power_on_parent = {
|
|
'step': 'not_power', 'priority': 15, 'interface': 'power'}
|
|
service_steps = [power_on_children, update_firmware_on_children,
|
|
power_on_parent]
|
|
dii = self.node.driver_internal_info
|
|
dii['service_steps'] = service_steps
|
|
self.node.driver_internal_info = dii
|
|
self.node.save()
|
|
|
|
mock_deploy.return_value = None
|
|
mock_mgmt.return_value = None
|
|
mock_power.return_value = None
|
|
child1_updated_at = str(child_node1.updated_at)
|
|
|
|
with task_manager.acquire(
|
|
self.context, self.node.uuid, shared=False) as task:
|
|
|
|
servicing.do_next_service_step(task, 0,
|
|
disable_ramdisk=True)
|
|
self.node.refresh()
|
|
child_node1.refresh()
|
|
child_node2.refresh()
|
|
|
|
# Confirm the objects *did* receive locks.
|
|
self.assertNotEqual(child1_updated_at, child_node1.updated_at)
|
|
self.assertIsNone(child_node2.updated_at)
|
|
|
|
# Confirm the child nodes have no errors
|
|
self.assertFalse(child_node1.maintenance)
|
|
self.assertFalse(child_node2.maintenance)
|
|
self.assertIsNone(child_node1.last_error)
|
|
self.assertIsNone(child_node2.last_error)
|
|
self.assertIsNone(self.node.last_error)
|
|
|
|
# Confirm the call counts expected
|
|
self.assertEqual(0, mock_deploy.call_count)
|
|
self.assertEqual(1, mock_mgmt.call_count)
|
|
self.assertEqual(1, mock_power.call_count)
|
|
self.assertEqual(0, mock_nv.call_count)
|
|
self.assertEqual(0, mock_pv.call_count)
|
|
mock_sps.assert_has_calls([
|
|
mock.call(mock.ANY, mock.ANY, 'power on', timeout=None)])
|