From 084da02a324ac9271b678fd029e3352b9bdb04dc Mon Sep 17 00:00:00 2001 From: Shivanand Tendulker Date: Tue, 3 Oct 2017 14:09:50 -0400 Subject: [PATCH] Adds RPC calls for rescue interface This commit adds RPC calls for rescue interface. It also adds transitions to/from the rescue-related states. Change-Id: I12cc8c3b89588394ff10837f05dd6ad5e9b55ee7 Partial-bug: #1526449 Co-Authored-By: Jay Faulkner Co-Authored-By: Josh Gachnang Co-Authored-By: Jesse J. Cook Co-Authored-By: Mario Villaplana Co-Authored-By: Aparna Co-Authored-By: Shivanand Tendulker --- doc/source/images/states.svg | 538 +++++++++++++------- ironic/common/exception.py | 10 + ironic/common/release_mappings.py | 2 +- ironic/common/states.py | 69 ++- ironic/conductor/manager.py | 252 ++++++++- ironic/conductor/rpcapi.py | 44 +- ironic/conductor/utils.py | 38 +- ironic/tests/unit/conductor/mgr_utils.py | 2 +- ironic/tests/unit/conductor/test_manager.py | 353 ++++++++++++- ironic/tests/unit/conductor/test_rpcapi.py | 28 + ironic/tests/unit/conductor/test_utils.py | 43 ++ 11 files changed, 1163 insertions(+), 216 deletions(-) diff --git a/doc/source/images/states.svg b/doc/source/images/states.svg index 2c8ceccb5a..b112291116 100644 --- a/doc/source/images/states.svg +++ b/doc/source/images/states.svg @@ -1,338 +1,482 @@ - - - + + Ironic states - + enroll - -enroll + +enroll verifying - -verifying + +verifying enroll->verifying - - -manage (via API) + + +manage (via API) -verifying->enroll - - -fail +verifying->enroll + + +fail manageable - -manageable + +manageable -verifying->manageable - - -done +verifying->manageable + + +done cleaning - -cleaning + +cleaning manageable->cleaning - - -provide (via API) + + +provide (via API) manageable->cleaning - - -clean (via API) + + +clean (via API) inspecting - -inspecting + +inspecting manageable->inspecting - - -inspect (via API) + + +inspect (via API) adopting - -adopting + +adopting manageable->adopting - - -adopt (via API) + + +adopt (via API) -cleaning->manageable - - -manage +cleaning->manageable + + +manage available - -available + +available -cleaning->available - - -done +cleaning->available + + +done -clean failed - -clean failed +clean failed + +clean failed -cleaning->clean failed - - -fail +cleaning->clean failed + + +fail -clean wait - -clean wait +clean wait + +clean wait -cleaning->clean wait - - -wait +cleaning->clean wait + + +wait -inspecting->manageable - - -done +inspecting->manageable + + +done -inspect failed - -inspect failed +inspect failed + +inspect failed -inspecting->inspect failed - - -fail +inspecting->inspect failed + + +fail active - -active + +active -adopting->active - - -done +adopting->active + + +done -adopt failed - -adopt failed +adopt failed + +adopt failed -adopting->adopt failed - - -fail +adopting->adopt failed + + +fail available->manageable - - -manage (via API) + + +manage (via API) deploying - -deploying + +deploying available->deploying - - -active (via API) + + +active (via API) -deploying->active - - -done +deploying->active + + +done -deploy failed - -deploy failed +deploy failed + +deploy failed -deploying->deploy failed - - -fail +deploying->deploy failed + + +fail -wait call-back - -wait call-back +wait call-back + +wait call-back -deploying->wait call-back - - -wait +deploying->wait call-back + + +wait active->deploying - - -rebuild (via API) + + +rebuild (via API) deleting - -deleting + +deleting active->deleting - - -deleted (via API) + + +deleted (via API) + + +rescuing + +rescuing + + +active->rescuing + + +rescue (via API) -deleting->cleaning - - -clean +deleting->cleaning + + +clean -error - -error +error + +error -deleting->error - - -error +deleting->error + + +error + + +rescue + +rescue + + +rescuing->rescue + + +done + + +rescue wait + +rescue wait + + +rescuing->rescue wait + + +wait + + +rescue failed + +rescue failed + + +rescuing->rescue failed + + +fail -error->deploying - - -rebuild (via API) +error->deploying + + +rebuild (via API) -error->deleting - - -deleted (via API) +error->deleting + + +deleted (via API) + + +rescue->deleting + + +deleted (via API) + + +rescue->rescuing + + +rescue (via API) + + +unrescuing + +unrescuing + + +rescue->unrescuing + + +unrescue (via API) + + +unrescuing->active + + +done + + +unrescue failed + +unrescue failed + + +unrescuing->unrescue failed + + +fail -deploy failed->deploying - - -rebuild (via API) +deploy failed->deploying + + +rebuild (via API) -deploy failed->deploying - - -active (via API) +deploy failed->deploying + + +active (via API) -deploy failed->deleting - - -deleted (via API) +deploy failed->deleting + + +deleted (via API) -wait call-back->deploying - - -resume +wait call-back->deploying + + +resume -wait call-back->deleting - - -deleted (via API) +wait call-back->deleting + + +deleted (via API) -wait call-back->deploy failed - - -fail +wait call-back->deploy failed + + +fail -clean failed->manageable - - -manage (via API) +clean failed->manageable + + +manage (via API) -clean wait->cleaning - - -resume +clean wait->cleaning + + +resume -clean wait->clean failed - - -fail +clean wait->clean failed + + +fail -clean wait->clean failed - - -abort (via API) +clean wait->clean failed + + +abort (via API) -inspect failed->manageable - - -manage (via API) +inspect failed->manageable + + +manage (via API) -inspect failed->inspecting - - -inspect (via API) +inspect failed->inspecting + + +inspect (via API) -adopt failed->manageable - - -manage (via API) +adopt failed->manageable + + +manage (via API) -adopt failed->adopting - - -adopt (via API) +adopt failed->adopting + + +adopt (via API) + + +rescue wait->deleting + + +deleted (via API) + + +rescue wait->rescuing + + +resume + + +rescue wait->rescue failed + + +fail + + +rescue wait->rescue failed + + +abort (via API) + + +rescue failed->deleting + + +deleted (via API) + + +rescue failed->rescuing + + +rescue (via API) + + +rescue failed->unrescuing + + +unrescue (via API) + + +unrescue failed->deleting + + +deleted (via API) + + +unrescue failed->rescuing + + +rescue (via API) + + +unrescue failed->unrescuing + + +unrescue (via API) diff --git a/ironic/common/exception.py b/ironic/common/exception.py index dcdb1ca693..4fd79ba65f 100644 --- a/ironic/common/exception.py +++ b/ironic/common/exception.py @@ -762,3 +762,13 @@ class AgentAPIError(IronicException): class NodeTraitNotFound(IronicException): _msg_fmt = _("Node %(node_id)s doesn't have a trait '%(trait)s'") + + +class InstanceRescueFailure(IronicException): + _msg_fmt = _('Failed to rescue instance %(instance)s for node ' + '%(node)s: %(reason)s') + + +class InstanceUnrescueFailure(IronicException): + _msg_fmt = _('Failed to unrescue instance %(instance)s for node ' + '%(node)s: %(reason)s') diff --git a/ironic/common/release_mappings.py b/ironic/common/release_mappings.py index 83acae0d15..61e21d3bf5 100644 --- a/ironic/common/release_mappings.py +++ b/ironic/common/release_mappings.py @@ -122,7 +122,7 @@ RELEASE_MAPPING = { }, 'master': { 'api': '1.36', - 'rpc': '1.42', + 'rpc': '1.43', 'objects': { 'Node': ['1.22'], 'Conductor': ['1.2'], diff --git a/ironic/common/states.py b/ironic/common/states.py index ddd767db08..0525f29d9a 100644 --- a/ironic/common/states.py +++ b/ironic/common/states.py @@ -48,6 +48,8 @@ VERBS = { 'abort': 'abort', 'clean': 'clean', 'adopt': 'adopt', + 'rescue': 'rescue', + 'unrescue': 'unrescue', } """ Mapping of state-changing events that are PUT to the REST API @@ -208,17 +210,18 @@ UNRESCUING = 'unrescuing' """ Node is being restored from rescue mode (to active state). """ UPDATE_ALLOWED_STATES = (DEPLOYFAIL, INSPECTING, INSPECTFAIL, CLEANFAIL, ERROR, - VERIFYING, ADOPTFAIL) + VERIFYING, ADOPTFAIL, RESCUEFAIL, UNRESCUEFAIL) """Transitional states in which we allow updating a node.""" DELETE_ALLOWED_STATES = (AVAILABLE, MANAGEABLE, ENROLL, ADOPTFAIL) """States in which node deletion is allowed.""" -STABLE_STATES = (ENROLL, MANAGEABLE, AVAILABLE, ACTIVE, ERROR) +STABLE_STATES = (ENROLL, MANAGEABLE, AVAILABLE, ACTIVE, ERROR, RESCUE) """States that will not transition unless receiving a request.""" UNSTABLE_STATES = (DEPLOYING, DEPLOYWAIT, CLEANING, CLEANWAIT, VERIFYING, - DELETING, INSPECTING, ADOPTING) + DELETING, INSPECTING, ADOPTING, RESCUING, RESCUEWAIT, + UNRESCUING) """States that can be changed without external request.""" ############## @@ -294,6 +297,13 @@ machine.add_state(INSPECTFAIL, target=MANAGEABLE, **watchers) machine.add_state(ADOPTING, target=ACTIVE, **watchers) machine.add_state(ADOPTFAIL, target=ACTIVE, **watchers) +# rescue states +machine.add_state(RESCUING, target=RESCUE, **watchers) +machine.add_state(RESCUEWAIT, target=RESCUE, **watchers) +machine.add_state(RESCUEFAIL, target=RESCUE, **watchers) +machine.add_state(UNRESCUING, target=ACTIVE, **watchers) +machine.add_state(UNRESCUEFAIL, target=ACTIVE, **watchers) + # A deployment may fail machine.add_transition(DEPLOYING, DEPLOYFAIL, 'fail') @@ -389,6 +399,59 @@ machine.add_transition(INSPECTFAIL, MANAGEABLE, 'manage') # Reinitiate the inspect after inspectfail. machine.add_transition(INSPECTFAIL, INSPECTING, 'inspect') +# A provisioned node may have a rescue initiated. +machine.add_transition(ACTIVE, RESCUING, 'rescue') + +# A rescue may succeed. +machine.add_transition(RESCUING, RESCUE, 'done') + +# A rescue may also wait on external callbacks +machine.add_transition(RESCUING, RESCUEWAIT, 'wait') +machine.add_transition(RESCUEWAIT, RESCUING, 'resume') + +# A rescued node may be re-rescued. +machine.add_transition(RESCUE, RESCUING, 'rescue') + +# A rescued node may be deleted. +machine.add_transition(RESCUE, DELETING, 'delete') + +# A rescue may fail. +machine.add_transition(RESCUEWAIT, RESCUEFAIL, 'fail') +machine.add_transition(RESCUING, RESCUEFAIL, 'fail') + +# While waiting for a rescue step to be finished, rescuing may be aborted +machine.add_transition(RESCUEWAIT, RESCUEFAIL, 'abort') + +# A failed rescue may be re-rescued. +machine.add_transition(RESCUEFAIL, RESCUING, 'rescue') + +# A failed rescue may be unrescued. +machine.add_transition(RESCUEFAIL, UNRESCUING, 'unrescue') + +# A failed rescue may be deleted. +machine.add_transition(RESCUEFAIL, DELETING, 'delete') + +# A rescuewait node may be deleted. +machine.add_transition(RESCUEWAIT, DELETING, 'delete') + +# A rescued node may be unrescued. +machine.add_transition(RESCUE, UNRESCUING, 'unrescue') + +# An unrescuing node may succeed +machine.add_transition(UNRESCUING, ACTIVE, 'done') + +# An unrescuing node may fail +machine.add_transition(UNRESCUING, UNRESCUEFAIL, 'fail') + +# A failed unrescue may be re-rescued +machine.add_transition(UNRESCUEFAIL, RESCUING, 'rescue') + +# A failed unrescue may be re-unrescued +machine.add_transition(UNRESCUEFAIL, UNRESCUING, 'unrescue') + +# A failed unrescue may be deleted. +machine.add_transition(UNRESCUEFAIL, DELETING, 'delete') + # Start power credentials verification machine.add_transition(ENROLL, VERIFYING, 'manage') diff --git a/ironic/conductor/manager.py b/ironic/conductor/manager.py index 0f8c37f5ce..0fd6bffc6f 100644 --- a/ironic/conductor/manager.py +++ b/ironic/conductor/manager.py @@ -94,7 +94,7 @@ class ConductorManager(base_manager.BaseConductorManager): # NOTE(rloo): This must be in sync with rpcapi.ConductorAPI's. # NOTE(pas-ha): This also must be in sync with # ironic.common.release_mappings.RELEASE_MAPPING['master'] - RPC_API_VERSION = '1.42' + RPC_API_VERSION = '1.43' target = messaging.Target(version=RPC_API_VERSION) @@ -527,6 +527,228 @@ class ConductorManager(base_manager.BaseConductorManager): return get_vendor_passthru_metadata(vendor.driver_routes) + @METRICS.timer('ConductorManager.do_node_rescue') + @messaging.expected_exceptions(exception.NoFreeConductorWorker, + exception.NodeInMaintenance, + exception.NodeLocked, + exception.InstanceRescueFailure, + exception.InvalidStateRequested, + exception.UnsupportedDriverExtension + ) + def do_node_rescue(self, context, node_id, rescue_password): + """RPC method to rescue an existing node deployment. + + Validate driver specific information synchronously, and then + spawn a background worker to rescue the node asynchronously. + + :param context: an admin context. + :param node_id: the id or uuid of a node. + :param rescue_password: string to be set as the password inside the + rescue environment. + :raises: InstanceRescueFailure if the node cannot be placed into + rescue mode. + :raises: InvalidStateRequested if the state transition is not supported + or allowed. + :raises: NoFreeConductorWorker when there is no free worker to start + async task. + :raises: NodeLocked if the node is locked by another conductor. + :raises: NodeInMaintenance if the node is in maintenance mode. + :raises: UnsupportedDriverExtension if rescue interface is not + supported by the driver. + """ + LOG.debug("RPC do_node_rescue called for node %s.", node_id) + + with task_manager.acquire(context, + node_id, purpose='node rescue') as task: + + node = task.node + if node.maintenance: + raise exception.NodeInMaintenance(op=_('rescuing'), + node=node.uuid) + + if not getattr(task.driver, 'rescue', None): + raise exception.UnsupportedDriverExtension( + driver=node.driver, extension='rescue') + # driver validation may check rescue_password, so save it on the + # node early + instance_info = node.instance_info + instance_info['rescue_password'] = rescue_password + node.instance_info = instance_info + node.save() + + try: + task.driver.power.validate(task) + task.driver.rescue.validate(task) + task.driver.network.validate(task) + except (exception.InvalidParameterValue, + exception.UnsupportedDriverExtension, + exception.MissingParameterValue) as e: + utils.remove_node_rescue_password(node, save=True) + raise exception.InstanceRescueFailure( + instance=node.instance_uuid, + node=node.uuid, + reason=_("Validation failed. Error: %s") % e) + try: + task.process_event( + 'rescue', + callback=self._spawn_worker, + call_args=(self._do_node_rescue, task), + err_handler=utils.spawn_rescue_error_handler) + except exception.InvalidState: + utils.remove_node_rescue_password(node, save=True) + raise exception.InvalidStateRequested( + action='rescue', node=node.uuid, + state=node.provision_state) + + def _do_node_rescue(self, task): + """Internal RPC method to rescue an existing node deployment.""" + node = task.node + + def handle_failure(e, errmsg, log_func=LOG.error): + utils.remove_node_rescue_password(node, save=False) + node.last_error = errmsg % e + task.process_event('fail') + log_func('Error while performing rescue operation for node ' + '%(node)s with instance %(instance)s: %(err)s', + {'node': node.uuid, 'instance': node.instance_uuid, + 'err': e}) + + try: + next_state = task.driver.rescue.rescue(task) + except exception.IronicException as e: + with excutils.save_and_reraise_exception(): + handle_failure(e, + _('Failed to rescue: %s')) + except Exception as e: + with excutils.save_and_reraise_exception(): + handle_failure(e, + _('Failed to rescue. Exception: %s'), + log_func=LOG.exception) + if next_state == states.RESCUEWAIT: + task.process_event('wait') + elif next_state == states.RESCUE: + task.process_event('done') + else: + error = (_("Driver returned unexpected state %s") % next_state) + handle_failure(error, + _('Failed to rescue: %s')) + + @METRICS.timer('ConductorManager.do_node_unrescue') + @messaging.expected_exceptions(exception.NoFreeConductorWorker, + exception.NodeInMaintenance, + exception.NodeLocked, + exception.InstanceUnrescueFailure, + exception.InvalidStateRequested, + exception.UnsupportedDriverExtension + ) + def do_node_unrescue(self, context, node_id): + """RPC method to unrescue a node in rescue mode. + + Validate driver specific information synchronously, and then + spawn a background worker to unrescue the node asynchronously. + + :param context: an admin context. + :param node_id: the id or uuid of a node. + :raises: InstanceUnrescueFailure if the node fails to be unrescued + :raises: InvalidStateRequested if the state transition is not supported + or allowed. + :raises: NoFreeConductorWorker when there is no free worker to start + async task + :raises: NodeLocked if the node is locked by another conductor. + :raises: NodeInMaintenance if the node is in maintenance mode. + :raises: UnsupportedDriverExtension if rescue interface is not + supported by the driver. + """ + LOG.debug("RPC do_node_unrescue called for node %s.", node_id) + + with task_manager.acquire(context, node_id, + purpose='node unrescue') as task: + node = task.node + if node.maintenance: + raise exception.NodeInMaintenance(op=_('unrescuing'), + node=node.uuid) + if not getattr(task.driver, 'rescue', None): + raise exception.UnsupportedDriverExtension( + driver=node.driver, extension='rescue') + try: + task.driver.power.validate(task) + except (exception.InvalidParameterValue, + exception.MissingParameterValue) as e: + raise exception.InstanceUnrescueFailure( + instance=node.instance_uuid, + node=node.uuid, + reason=_("Validation failed. Error: %s") % e) + + try: + task.process_event( + 'unrescue', + callback=self._spawn_worker, + call_args=(self._do_node_unrescue, task), + err_handler=utils.provisioning_error_handler) + except exception.InvalidState: + raise exception.InvalidStateRequested( + action='unrescue', node=node.uuid, + state=node.provision_state) + + def _do_node_unrescue(self, task): + """Internal RPC method to unrescue a node in rescue mode.""" + node = task.node + + def handle_failure(e, errmsg, log_func=LOG.error): + node.last_error = errmsg % e + task.process_event('fail') + log_func('Error while performing unrescue operation for node ' + '%(node)s with instance %(instance)s: %(err)s', + {'node': node.uuid, 'instance': node.instance_uuid, + 'err': e}) + + try: + next_state = task.driver.rescue.unrescue(task) + except exception.IronicException as e: + with excutils.save_and_reraise_exception(): + handle_failure(e, + _('Failed to unrescue: %s')) + except Exception as e: + with excutils.save_and_reraise_exception(): + handle_failure(e, + _('Failed to unrescue. Exception: %s'), + log_func=LOG.exception) + if next_state == states.ACTIVE: + task.process_event('done') + else: + error = (_("Driver returned unexpected state %s") % next_state) + handle_failure(error, + _('Failed to unrescue: %s')) + + @task_manager.require_exclusive_lock + def _do_node_rescue_abort(self, task): + """Internal method to abort an ongoing rescue operation. + + :param task: a TaskManager instance with an exclusive lock + """ + node = task.node + try: + task.driver.rescue.clean_up(task) + except Exception as e: + LOG.exception('Failed to clean up rescue for node %(node)s ' + 'after aborting the operation. Error: %(err)s', + {'node': node.uuid, 'err': e}) + error_msg = _('Failed to clean up rescue after aborting ' + 'the operation') + node.refresh() + node.last_error = error_msg + node.maintenance = True + node.maintenance_reason = error_msg + node.save() + return + + info_message = _('Rescue operation aborted for node %s.') % node.uuid + last_error = _('By request, the rescue operation was aborted.') + node.refresh() + node.last_error = last_error + node.save() + LOG.info(info_message) + @METRICS.timer('ConductorManager.do_node_deploy') @messaging.expected_exceptions(exception.NoFreeConductorWorker, exception.NodeLocked, @@ -657,7 +879,8 @@ class ConductorManager(base_manager.BaseConductorManager): task.process_event( 'delete', callback=self._spawn_worker, - call_args=(self._do_node_tear_down, task), + call_args=(self._do_node_tear_down, task, + task.node.provision_state), err_handler=utils.provisioning_error_handler) except exception.InvalidState: raise exception.InvalidStateRequested( @@ -665,10 +888,21 @@ class ConductorManager(base_manager.BaseConductorManager): state=task.node.provision_state) @task_manager.require_exclusive_lock - def _do_node_tear_down(self, task): - """Internal RPC method to tear down an existing node deployment.""" + def _do_node_tear_down(self, task, initial_state): + """Internal RPC method to tear down an existing node deployment. + + :param task: a task from TaskManager. + :param initial_state: The initial provision state from which node + has moved into deleting state. + """ node = task.node try: + if (initial_state in (states.RESCUEWAIT, states.RESCUE, + states.UNRESCUEFAIL, states.RESCUEFAIL)): + # Perform rescue clean up. Rescue clean up will remove + # rescuing network as well. + task.driver.rescue.clean_up(task) + task.driver.deploy.clean_up(task) task.driver.deploy.tear_down(task) except Exception as e: @@ -1216,6 +1450,16 @@ class ConductorManager(base_manager.BaseConductorManager): target_state=target_state) return + if (action == states.VERBS['abort'] and + node.provision_state == states.RESCUEWAIT): + utils.remove_node_rescue_password(node, save=True) + task.process_event( + 'abort', + callback=self._spawn_worker, + call_args=(self._do_node_rescue_abort, task), + err_handler=utils.provisioning_error_handler) + return + try: task.process_event(action) except exception.InvalidState: diff --git a/ironic/conductor/rpcapi.py b/ironic/conductor/rpcapi.py index 6563ce10f7..e3ebcce809 100644 --- a/ironic/conductor/rpcapi.py +++ b/ironic/conductor/rpcapi.py @@ -91,13 +91,14 @@ class ConductorAPI(object): | 1.40 - Added inject_nmi | 1.41 - Added create_port | 1.42 - Added optional agent_version to heartbeat + | 1.43 - Added do_node_rescue, do_node_unrescue and can_send_rescue """ # NOTE(rloo): This must be in sync with manager.ConductorManager's. # NOTE(pas-ha): This also must be in sync with # ironic.common.release_mappings.RELEASE_MAPPING['master'] - RPC_API_VERSION = '1.42' + RPC_API_VERSION = '1.43' def __init__(self, topic=None): super(ConductorAPI, self).__init__() @@ -158,6 +159,10 @@ class ConductorAPI(object): """Return whether the RPCAPI supports the create_port method.""" return self.client.can_send_version("1.41") + def can_send_rescue(self): + """Return whether the RPCAPI supports node rescue methods.""" + return self.client.can_send_version("1.43") + def create_node(self, context, node_obj, topic=None): """Synchronously, have a conductor validate and create a node. @@ -975,3 +980,40 @@ class ConductorAPI(object): """ cctxt = self.client.prepare(topic=topic or self.topic, version='1.38') return cctxt.call(context, 'vif_list', node_id=node_id) + + def do_node_rescue(self, context, node_id, rescue_password, topic=None): + """Signal to conductor service to perform a rescue. + + :param context: request context. + :param node_id: node ID or UUID. + :param rescue_password: A string representing the password to be set + inside the rescue environment. + :param topic: RPC topic. Defaults to self.topic. + :raises: InstanceRescueFailure + :raises: NoFreeConductorWorker when there is no free worker to start + async task. + + The node must already be configured and in the appropriate + state before this method is called. + + """ + cctxt = self.client.prepare(topic=topic or self.topic, version='1.43') + return cctxt.call(context, 'do_node_rescue', node_id=node_id, + rescue_password=rescue_password) + + def do_node_unrescue(self, context, node_id, topic=None): + """Signal to conductor service to perform an unrescue. + + :param context: request context. + :param node_id: node ID or UUID. + :param topic: RPC topic. Defaults to self.topic. + :raises: InstanceUnrescueFailure + :raises: NoFreeConductorWorker when there is no free worker to start + async task. + + The node must already be configured and in the appropriate + state before this method is called. + + """ + cctxt = self.client.prepare(topic=topic or self.topic, version='1.43') + return cctxt.call(context, 'do_node_unrescue', node_id=node_id) diff --git a/ironic/conductor/utils.py b/ironic/conductor/utils.py index 2667862e10..0ed6a68084 100644 --- a/ironic/conductor/utils.py +++ b/ironic/conductor/utils.py @@ -398,13 +398,26 @@ def cleaning_error_handler(task, msg, tear_down_cleaning=True, task.process_event('fail', target_state=target_state) -def spawn_cleaning_error_handler(e, node): - """Handle spawning error for node cleaning.""" +def _spawn_error_handler(e, node, state): + """Handle spawning error for node.""" if isinstance(e, exception.NoFreeConductorWorker): node.last_error = (_("No free conductor workers available")) node.save() LOG.warning("No free conductor workers available to perform " - "cleaning on node %(node)s", {'node': node.uuid}) + "%(state)s on node %(node)s", + {'state': state, 'node': node.uuid}) + + +def spawn_cleaning_error_handler(e, node): + """Handle spawning error for node cleaning.""" + _spawn_error_handler(e, node, states.CLEANING) + + +def spawn_rescue_error_handler(e, node): + """Handle spawning error for node rescue.""" + if isinstance(e, exception.NoFreeConductorWorker): + remove_node_rescue_password(node, save=False) + _spawn_error_handler(e, node, states.RESCUE) def power_state_error_handler(e, node, power_state): @@ -652,3 +665,22 @@ def validate_port_physnet(task, port_obj): raise exception.Conflict( msg % {'portgroup': portgroup.uuid, 'physnet': port_physnet, 'pg_physnet': pg_physnet}) + + +def remove_node_rescue_password(node, save=True): + """Helper to remove rescue password from a node. + + Removes rescue password from node. It saves node by default. + If node should not be saved, then caller needs to explicitly + indicate it. + + :param node: an Ironic node object. + :param save: Boolean; True (default) to save the node; False + otherwise. + """ + instance_info = node.instance_info + if 'rescue_password' in instance_info: + del instance_info['rescue_password'] + node.instance_info = instance_info + if save: + node.save() diff --git a/ironic/tests/unit/conductor/mgr_utils.py b/ironic/tests/unit/conductor/mgr_utils.py index a166594696..c531bcfe60 100644 --- a/ironic/tests/unit/conductor/mgr_utils.py +++ b/ironic/tests/unit/conductor/mgr_utils.py @@ -86,7 +86,7 @@ class CommonMixIn(object): if node is None: node = self._create_node(**node_attrs) task = mock.Mock(spec_set=['node', 'release_resources', - 'spawn_after', 'process_event']) + 'spawn_after', 'process_event', 'driver']) task.node = node return task diff --git a/ironic/tests/unit/conductor/test_manager.py b/ironic/tests/unit/conductor/test_manager.py index a19b62bd34..286d5b67b1 100644 --- a/ironic/tests/unit/conductor/test_manager.py +++ b/ironic/tests/unit/conductor/test_manager.py @@ -580,6 +580,7 @@ class UpdateNodeTestCase(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase): 'network_interface': UpdateInterfaces('flat', 'noop'), 'power_interface': UpdateInterfaces(None, 'fake'), 'raid_interface': UpdateInterfaces(None, 'fake'), + 'rescue_interface': UpdateInterfaces(None, 'no-rescue'), 'storage_interface': UpdateInterfaces('noop', 'cinder'), } @@ -1647,7 +1648,8 @@ class DoNodeDeployTearDownTestCase(mgr_utils.ServiceSetUpMixin, self._start_service() mock_tear_down.side_effect = exception.InstanceDeployFailure('test') self.assertRaises(exception.InstanceDeployFailure, - self.service._do_node_tear_down, task) + self.service._do_node_tear_down, task, + node.provision_state) node.refresh() self.assertEqual(states.ERROR, node.provision_state) self.assertEqual(states.NOSTATE, node.target_provision_state) @@ -1670,7 +1672,7 @@ class DoNodeDeployTearDownTestCase(mgr_utils.ServiceSetUpMixin, task = task_manager.TaskManager(self.context, node.uuid) self._start_service() - self.service._do_node_tear_down(task) + self.service._do_node_tear_down(task, node.provision_state) node.refresh() # Node will be moved to AVAILABLE after cleaning, not tested here self.assertEqual(states.CLEANING, node.provision_state) @@ -1682,12 +1684,15 @@ class DoNodeDeployTearDownTestCase(mgr_utils.ServiceSetUpMixin, mock_tear_down.assert_called_once_with(mock.ANY) mock_clean.assert_called_once_with(mock.ANY) + @mock.patch('ironic.drivers.modules.fake.FakeRescue.clean_up') @mock.patch('ironic.conductor.manager.ConductorManager._do_node_clean') @mock.patch('ironic.drivers.modules.fake.FakeDeploy.tear_down') - def _test_do_node_tear_down_from_state(self, init_state, mock_tear_down, - mock_clean): + def _test_do_node_tear_down_from_state(self, init_state, is_rescue_state, + mock_tear_down, mock_clean, + mock_rescue_clean): node = obj_utils.create_test_node( - self.context, driver='fake', uuid=uuidutils.generate_uuid(), + self.context, driver='fake-hardware', + uuid=uuidutils.generate_uuid(), provision_state=init_state, target_provision_state=states.AVAILABLE, driver_internal_info={'is_whole_disk_image': False}) @@ -1703,12 +1708,21 @@ class DoNodeDeployTearDownTestCase(mgr_utils.ServiceSetUpMixin, self.assertEqual({}, node.instance_info) mock_tear_down.assert_called_once_with(mock.ANY) mock_clean.assert_called_once_with(mock.ANY) + if is_rescue_state: + mock_rescue_clean.assert_called_once_with(mock.ANY) + else: + self.assertFalse(mock_rescue_clean.called) def test__do_node_tear_down_from_valid_states(self): valid_states = [states.ACTIVE, states.DEPLOYWAIT, states.DEPLOYFAIL, states.ERROR] for state in valid_states: - self._test_do_node_tear_down_from_state(state) + self._test_do_node_tear_down_from_state(state, False) + + valid_rescue_states = [states.RESCUEWAIT, states.RESCUE, + states.UNRESCUEFAIL, states.RESCUEFAIL] + for state in valid_rescue_states: + self._test_do_node_tear_down_from_state(state, True) # NOTE(deva): partial tear-down was broken. A node left in a state of # DELETING could not have tear_down called on it a second time @@ -2746,6 +2760,333 @@ class DoNodeCleanTestCase(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase): self.assertEqual(0, step_index) +class DoNodeRescueTestCase(mgr_utils.CommonMixIn, mgr_utils.ServiceSetUpMixin, + db_base.DbTestCase): + @mock.patch('ironic.conductor.task_manager.acquire', autospec=True) + def test_do_node_rescue(self, mock_acquire): + self._start_service() + task = self._create_task( + node_attrs=dict(driver='fake-hardware', + provision_state=states.ACTIVE, + instance_info={})) + mock_acquire.side_effect = self._get_acquire_side_effect(task) + self.service.do_node_rescue(self.context, task.node.uuid, + "password") + task.process_event.assert_called_once_with( + 'rescue', + callback=self.service._spawn_worker, + call_args=(self.service._do_node_rescue, task), + err_handler=conductor_utils.spawn_rescue_error_handler) + self.assertIn('rescue_password', task.node.instance_info) + + def test_do_node_rescue_invalid_state(self): + self._start_service() + node = obj_utils.create_test_node(self.context, driver='fake-hardware', + network_interface='noop', + provision_state=states.AVAILABLE, + instance_info={}) + exc = self.assertRaises(messaging.rpc.ExpectedException, + self.service.do_node_rescue, + self.context, node.uuid, "password") + node.refresh() + self.assertNotIn('rescue_password', node.instance_info) + self.assertEqual(exception.InvalidStateRequested, exc.exc_info[0]) + + def _test_do_node_rescue_when_validate_fail(self, mock_validate): + # InvalidParameterValue should be re-raised as InstanceRescueFailure + mock_validate.side_effect = exception.InvalidParameterValue('error') + node = obj_utils.create_test_node( + self.context, driver='fake-hardware', + provision_state=states.ACTIVE, + target_provision_state=states.NOSTATE, + instance_info={}) + exc = self.assertRaises(messaging.rpc.ExpectedException, + self.service.do_node_rescue, + self.context, node.uuid, "password") + node.refresh() + self.assertNotIn('rescue_password', node.instance_info) + # Compare true exception hidden by @messaging.expected_exceptions + self.assertEqual(exception.InstanceRescueFailure, exc.exc_info[0]) + + @mock.patch('ironic.drivers.modules.fake.FakeRescue.validate') + def test_do_node_rescue_when_rescue_validate_fail(self, mock_validate): + self._test_do_node_rescue_when_validate_fail(mock_validate) + + @mock.patch('ironic.drivers.modules.fake.FakePower.validate') + def test_do_node_rescue_when_power_validate_fail(self, mock_validate): + self._test_do_node_rescue_when_validate_fail(mock_validate) + + @mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate') + def test_do_node_rescue_when_network_validate_fail(self, mock_validate): + self._test_do_node_rescue_when_validate_fail(mock_validate) + + def test_do_node_rescue_not_supported(self): + node = obj_utils.create_test_node( + self.context, driver='fake', + provision_state=states.ACTIVE, + target_provision_state=states.NOSTATE, + instance_info={}) + exc = self.assertRaises(messaging.rpc.ExpectedException, + self.service.do_node_rescue, + self.context, node.uuid, "password") + self.assertEqual(exception.UnsupportedDriverExtension, + exc.exc_info[0]) + + def test_do_node_rescue_maintenance(self): + node = obj_utils.create_test_node( + self.context, driver='fake-hardware', + provision_state=states.ACTIVE, + maintenance=True, + target_provision_state=states.NOSTATE, + instance_info={}) + exc = self.assertRaises(messaging.rpc.ExpectedException, + self.service.do_node_rescue, + self.context, node['uuid'], "password") + # Compare true exception hidden by @messaging.expected_exceptions + self.assertEqual(exception.NodeInMaintenance, exc.exc_info[0]) + # This is a sync operation last_error should be None. + self.assertIsNone(node.last_error) + + @mock.patch('ironic.drivers.modules.fake.FakeRescue.rescue') + def test__do_node_rescue_returns_rescuewait(self, mock_rescue): + node = obj_utils.create_test_node(self.context, driver='fake-hardware', + provision_state=states.RESCUING, + instance_info={'rescue_password': + 'password'}) + self._start_service() + with task_manager.TaskManager(self.context, node.uuid) as task: + mock_rescue.return_value = states.RESCUEWAIT + self.service._do_node_rescue(task) + node.refresh() + self.assertEqual(states.RESCUEWAIT, node.provision_state) + self.assertEqual(states.RESCUE, node.target_provision_state) + self.assertIn('rescue_password', node.instance_info) + + @mock.patch('ironic.drivers.modules.fake.FakeRescue.rescue') + def test__do_node_rescue_returns_rescue(self, mock_rescue): + node = obj_utils.create_test_node(self.context, driver='fake-hardware', + provision_state=states.RESCUING, + instance_info={'rescue_password': + 'password'}) + self._start_service() + with task_manager.TaskManager(self.context, node.uuid) as task: + mock_rescue.return_value = states.RESCUE + self.service._do_node_rescue(task) + node.refresh() + self.assertEqual(states.RESCUE, node.provision_state) + self.assertEqual(states.NOSTATE, node.target_provision_state) + self.assertIn('rescue_password', node.instance_info) + + @mock.patch.object(manager, 'LOG') + @mock.patch('ironic.drivers.modules.fake.FakeRescue.rescue') + def test__do_node_rescue_errors(self, mock_rescue, mock_log): + node = obj_utils.create_test_node(self.context, driver='fake-hardware', + provision_state=states.RESCUING, + instance_info={'rescue_password': + 'password'}) + self._start_service() + mock_rescue.side_effect = exception.InstanceRescueFailure( + 'failed to rescue') + with task_manager.TaskManager(self.context, node.uuid) as task: + self.assertRaises(exception.InstanceRescueFailure, + self.service._do_node_rescue, task) + node.refresh() + self.assertEqual(states.RESCUEFAIL, node.provision_state) + self.assertEqual(states.RESCUE, node.target_provision_state) + self.assertNotIn('rescue_password', node.instance_info) + self.assertTrue(node.last_error.startswith('Failed to rescue')) + self.assertTrue(mock_log.error.called) + + @mock.patch.object(manager, 'LOG') + @mock.patch('ironic.drivers.modules.fake.FakeRescue.rescue') + def test__do_node_rescue_bad_state(self, mock_rescue, mock_log): + node = obj_utils.create_test_node(self.context, driver='fake-hardware', + provision_state=states.RESCUING, + instance_info={'rescue_password': + 'password'}) + self._start_service() + mock_rescue.return_value = states.ACTIVE + with task_manager.TaskManager(self.context, node.uuid) as task: + self.service._do_node_rescue(task) + node.refresh() + self.assertEqual(states.RESCUEFAIL, node.provision_state) + self.assertEqual(states.RESCUE, node.target_provision_state) + self.assertNotIn('rescue_password', node.instance_info) + self.assertTrue(node.last_error.startswith('Failed to rescue')) + self.assertTrue(mock_log.error.called) + + @mock.patch('ironic.conductor.task_manager.acquire', autospec=True) + def test_do_node_unrescue(self, mock_acquire): + self._start_service() + task = self._create_task( + node_attrs=dict(driver='fake-hardware', + provision_state=states.RESCUE)) + mock_acquire.side_effect = self._get_acquire_side_effect(task) + self.service.do_node_unrescue(self.context, task.node.uuid) + task.process_event.assert_called_once_with( + 'unrescue', + callback=self.service._spawn_worker, + call_args=(self.service._do_node_unrescue, task), + err_handler=conductor_utils.provisioning_error_handler) + + def test_do_node_unrescue_invalid_state(self): + self._start_service() + node = obj_utils.create_test_node(self.context, driver='fake-hardware', + provision_state=states.AVAILABLE) + exc = self.assertRaises(messaging.rpc.ExpectedException, + self.service.do_node_unrescue, + self.context, node.uuid) + self.assertEqual(exception.InvalidStateRequested, exc.exc_info[0]) + + @mock.patch('ironic.drivers.modules.fake.FakePower.validate') + def test_do_node_unrescue_validate_fail(self, mock_validate): + # InvalidParameterValue should be re-raised as InstanceUnrescueFailure + mock_validate.side_effect = exception.InvalidParameterValue('error') + node = obj_utils.create_test_node( + self.context, driver='fake-hardware', + provision_state=states.RESCUE, + target_provision_state=states.NOSTATE) + exc = self.assertRaises(messaging.rpc.ExpectedException, + self.service.do_node_unrescue, + self.context, node.uuid) + # Compare true exception hidden by @messaging.expected_exceptions + self.assertEqual(exception.InstanceUnrescueFailure, exc.exc_info[0]) + + def test_do_node_unrescue_not_supported(self): + node = obj_utils.create_test_node( + self.context, driver='fake', + provision_state=states.RESCUE, + instance_info={}) + exc = self.assertRaises(messaging.rpc.ExpectedException, + self.service.do_node_unrescue, + self.context, node.uuid) + self.assertEqual(exception.UnsupportedDriverExtension, + exc.exc_info[0]) + + def test_do_node_unrescue_maintenance(self): + node = obj_utils.create_test_node( + self.context, driver='fake-hardware', + provision_state=states.RESCUE, + maintenance=True, + target_provision_state=states.NOSTATE, + instance_info={}) + exc = self.assertRaises(messaging.rpc.ExpectedException, + self.service.do_node_unrescue, + self.context, node.uuid) + # Compare true exception hidden by @messaging.expected_exceptions + self.assertEqual(exception.NodeInMaintenance, exc.exc_info[0]) + # This is a sync operation last_error should be None. + node.refresh() + self.assertIsNone(node.last_error) + + @mock.patch('ironic.drivers.modules.fake.FakeRescue.unrescue') + def test__do_node_unrescue(self, mock_unrescue): + node = obj_utils.create_test_node(self.context, driver='fake-hardware', + provision_state=states.UNRESCUING, + target_provision_state=states.ACTIVE, + instance_info={}) + self._start_service() + with task_manager.TaskManager(self.context, node.uuid) as task: + mock_unrescue.return_value = states.ACTIVE + self.service._do_node_unrescue(task) + node.refresh() + self.assertEqual(states.ACTIVE, node.provision_state) + self.assertEqual(states.NOSTATE, node.target_provision_state) + + @mock.patch.object(manager, 'LOG') + @mock.patch('ironic.drivers.modules.fake.FakeRescue.unrescue') + def test__do_node_unrescue_ironic_error(self, mock_unrescue, mock_log): + node = obj_utils.create_test_node(self.context, driver='fake-hardware', + provision_state=states.UNRESCUING, + target_provision_state=states.ACTIVE, + instance_info={}) + self._start_service() + mock_unrescue.side_effect = exception.InstanceUnrescueFailure( + 'Unable to unrescue') + with task_manager.TaskManager(self.context, node.uuid) as task: + self.assertRaises(exception.InstanceUnrescueFailure, + self.service._do_node_unrescue, task) + node.refresh() + self.assertEqual(states.UNRESCUEFAIL, node.provision_state) + self.assertEqual(states.ACTIVE, node.target_provision_state) + self.assertTrue('Unable to unrescue' in node.last_error) + self.assertTrue(mock_log.error.called) + + @mock.patch.object(manager, 'LOG') + @mock.patch('ironic.drivers.modules.fake.FakeRescue.unrescue') + def test__do_node_unrescue_other_error(self, mock_unrescue, mock_log): + node = obj_utils.create_test_node(self.context, driver='fake-hardware', + provision_state=states.UNRESCUING, + target_provision_state=states.ACTIVE, + instance_info={}) + self._start_service() + mock_unrescue.side_effect = RuntimeError('Some failure') + with task_manager.TaskManager(self.context, node.uuid) as task: + self.assertRaises(RuntimeError, + self.service._do_node_unrescue, task) + node.refresh() + self.assertEqual(states.UNRESCUEFAIL, node.provision_state) + self.assertEqual(states.ACTIVE, node.target_provision_state) + self.assertTrue('Some failure' in node.last_error) + self.assertTrue(mock_log.exception.called) + + @mock.patch('ironic.drivers.modules.fake.FakeRescue.unrescue') + def test__do_node_unrescue_bad_state(self, mock_unrescue): + node = obj_utils.create_test_node(self.context, driver='fake-hardware', + provision_state=states.UNRESCUING, + instance_info={}) + self._start_service() + mock_unrescue.return_value = states.RESCUEWAIT + with task_manager.TaskManager(self.context, node.uuid) as task: + self.service._do_node_unrescue(task) + node.refresh() + self.assertEqual(states.UNRESCUEFAIL, node.provision_state) + self.assertEqual(states.ACTIVE, node.target_provision_state) + self.assertTrue('Driver returned unexpected state' in + node.last_error) + + @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker') + def test_provision_rescue_abort(self, mock_spawn): + node = obj_utils.create_test_node( + self.context, driver='fake-hardware', + provision_state=states.RESCUEWAIT, + target_provision_state=states.RESCUE, + instance_info={'rescue_password': 'password'}) + self._start_service() + self.service.do_provisioning_action(self.context, node.uuid, 'abort') + node.refresh() + self.assertEqual(states.RESCUEFAIL, node.provision_state) + self.assertIsNone(node.last_error) + self.assertNotIn('rescue_password', node.instance_info) + mock_spawn.assert_called_with(self.service._do_node_rescue_abort, + mock.ANY) + + @mock.patch.object(fake.FakeRescue, 'clean_up', autospec=True) + def test__do_node_rescue_abort(self, clean_up_mock): + node = obj_utils.create_test_node( + self.context, driver='fake-hardware', + provision_state=states.RESCUEFAIL, + target_provision_state=states.RESCUE) + with task_manager.acquire(self.context, node.uuid) as task: + self.service._do_node_rescue_abort(task) + clean_up_mock.assert_called_once_with(task.driver.rescue, task) + self.assertIsNotNone(task.node.last_error) + self.assertFalse(task.node.maintenance) + + @mock.patch.object(fake.FakeRescue, 'clean_up', autospec=True) + def test__do_node_rescue_abort_clean_up_fail(self, clean_up_mock): + clean_up_mock.side_effect = Exception('Surprise') + node = obj_utils.create_test_node( + self.context, driver='fake-hardware', + provision_state=states.RESCUEFAIL) + with task_manager.acquire(self.context, node.uuid) as task: + self.service._do_node_rescue_abort(task) + clean_up_mock.assert_called_once_with(task.driver.rescue, task) + self.assertIsNotNone(task.node.last_error) + self.assertIsNotNone(task.node.maintenance_reason) + self.assertTrue(task.node.maintenance) + + @mgr_utils.mock_record_keepalive class DoNodeVerifyTestCase(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase): @mock.patch('ironic.objects.node.NodeCorrectedPowerStateNotification') diff --git a/ironic/tests/unit/conductor/test_rpcapi.py b/ironic/tests/unit/conductor/test_rpcapi.py index e9a0d9850e..f384ae52ce 100644 --- a/ironic/tests/unit/conductor/test_rpcapi.py +++ b/ironic/tests/unit/conductor/test_rpcapi.py @@ -499,3 +499,31 @@ class RPCAPITestCase(db_base.DbTestCase): 'call', node_id='fake-node', version='1.38') + + def test_do_node_rescue(self): + self._test_rpcapi('do_node_rescue', + 'call', + version='1.43', + node_id=self.fake_node['uuid'], + rescue_password="password") + + def test_do_node_unrescue(self): + self._test_rpcapi('do_node_unrescue', + 'call', + version='1.43', + node_id=self.fake_node['uuid']) + + def _test_can_send_rescue(self, can_send): + rpcapi = conductor_rpcapi.ConductorAPI(topic='fake-topic') + with mock.patch.object(rpcapi.client, + "can_send_version") as mock_can_send_version: + mock_can_send_version.return_value = can_send + result = rpcapi.can_send_rescue() + self.assertEqual(can_send, result) + mock_can_send_version.assert_called_once_with("1.43") + + def test_can_send_rescue_true(self): + self._test_can_send_rescue(True) + + def test_can_send_rescue_false(self): + self._test_can_send_rescue(False) diff --git a/ironic/tests/unit/conductor/test_utils.py b/ironic/tests/unit/conductor/test_utils.py index 7c3649ef13..f4d547233f 100644 --- a/ironic/tests/unit/conductor/test_utils.py +++ b/ironic/tests/unit/conductor/test_utils.py @@ -1201,6 +1201,25 @@ class ErrorHandlersTestCase(tests_base.TestCase): self.assertFalse(self.node.save.called) self.assertFalse(log_mock.warning.called) + @mock.patch.object(conductor_utils, 'LOG') + def test_spawn_rescue_error_handler_no_worker(self, log_mock): + exc = exception.NoFreeConductorWorker() + self.node.instance_info = {'rescue_password': 'pass'} + conductor_utils.spawn_rescue_error_handler(exc, self.node) + self.node.save.assert_called_once_with() + self.assertIn('No free conductor workers', self.node.last_error) + self.assertTrue(log_mock.warning.called) + self.assertNotIn('rescue_password', self.node.instance_info) + + @mock.patch.object(conductor_utils, 'LOG') + def test_spawn_rescue_error_handler_other_error(self, log_mock): + exc = Exception('foo') + self.node.instance_info = {'rescue_password': 'pass'} + conductor_utils.spawn_rescue_error_handler(exc, self.node) + self.assertFalse(self.node.save.called) + self.assertFalse(log_mock.warning.called) + self.assertIn('rescue_password', self.node.instance_info) + @mock.patch.object(conductor_utils, 'LOG') def test_power_state_error_handler_no_worker(self, log_mock): exc = exception.NoFreeConductorWorker() @@ -1535,3 +1554,27 @@ class ValidatePortPhysnetTestCase(db_base.DbTestCase): current_physnet='physnet1', new_physnet=None, valid=False) + + +class MiscTestCase(db_base.DbTestCase): + def setUp(self): + super(MiscTestCase, self).setUp() + self.node = obj_utils.create_test_node( + self.context, + driver='fake', + instance_info={'rescue_password': 'pass'}) + + def _test_remove_node_rescue_password(self, save=True): + conductor_utils.remove_node_rescue_password(self.node, save=save) + self.assertNotIn('rescue_password', self.node.instance_info) + self.node.refresh() + if save: + self.assertNotIn('rescue_password', self.node.instance_info) + else: + self.assertIn('rescue_password', self.node.instance_info) + + def test_remove_node_rescue_password_save_true(self): + self._test_remove_node_rescue_password(save=True) + + def test_remove_node_rescue_password_save_false(self): + self._test_remove_node_rescue_password(save=False)