diff --git a/distributedcloud/dccommon/drivers/openstack/sysinv_v1.py b/distributedcloud/dccommon/drivers/openstack/sysinv_v1.py index 3399fad65..774fd4c83 100644 --- a/distributedcloud/dccommon/drivers/openstack/sysinv_v1.py +++ b/distributedcloud/dccommon/drivers/openstack/sysinv_v1.py @@ -235,6 +235,11 @@ class SysinvClient(base.DriverBase): LOG.error("delete_load exception={}".format(e)) raise e + def import_load(self, path_to_iso, path_to_sig): + """Import the particular software load.""" + return self.sysinv_client.load.import_load(path_to_iso=path_to_iso, + path_to_sig=path_to_sig) + def get_hosts(self): """Get a list of hosts.""" return self.sysinv_client.ihost.list() @@ -243,6 +248,21 @@ class SysinvClient(base.DriverBase): """Get a list of upgrades.""" return self.sysinv_client.upgrade.list() + def upgrade_activate(self): + """Invoke the API for 'system upgrade-activate', which is an update """ + patch = [{'op': 'replace', + 'path': '/state', + 'value': 'activation-requested'}, ] + return self.sysinv_client.upgrade.update(patch) + + def upgrade_complete(self): + """Invoke the API for 'system upgrade-complete', which is a delete""" + return self.sysinv_client.upgrade.delete() + + def upgrade_start(self, force=False): + """Invoke the API for 'system upgrade-start', which is a create""" + return self.sysinv_client.upgrade.create(force) + def get_applications(self): """Get a list of containerized applications""" diff --git a/distributedcloud/dcmanager/common/consts.py b/distributedcloud/dcmanager/common/consts.py index 6d453ee95..2beb1a3d3 100644 --- a/distributedcloud/dcmanager/common/consts.py +++ b/distributedcloud/dcmanager/common/consts.py @@ -108,7 +108,8 @@ STRATEGY_STATE_LOCKING_CONTROLLER = "locking controller" STRATEGY_STATE_UPGRADING_SIMPLEX = "upgrading simplex" STRATEGY_STATE_MIGRATING_DATA = "migrating data" STRATEGY_STATE_UNLOCKING_CONTROLLER = "unlocking controller" -STRATEGY_STATE_ACTIVATING = "activating" +STRATEGY_STATE_ACTIVATING_UPGRADE = "activating upgrade" +STRATEGY_STATE_COMPLETING_UPGRADE = "completing upgrade" # Subcloud deploy status states DEPLOY_STATE_NONE = 'not-deployed' diff --git a/distributedcloud/dcmanager/common/exceptions.py b/distributedcloud/dcmanager/common/exceptions.py index 7ca9b93c8..c03a0d435 100644 --- a/distributedcloud/dcmanager/common/exceptions.py +++ b/distributedcloud/dcmanager/common/exceptions.py @@ -178,6 +178,10 @@ class LicenseMissingError(DCManagerException): message = _("License does not exist on subcloud: %(subcloud_id)s") +class VaultLoadMissingError(DCManagerException): + message = _("No matching: %(file_type) found in vault: %(vault_dir)") + + class StrategyStepNotFound(NotFound): message = _("StrategyStep with subcloud_id %(subcloud_id)s " "doesn't exist.") diff --git a/distributedcloud/dcmanager/manager/states/base.py b/distributedcloud/dcmanager/manager/states/base.py index 3da9752cc..d22d779d1 100644 --- a/distributedcloud/dcmanager/manager/states/base.py +++ b/distributedcloud/dcmanager/manager/states/base.py @@ -28,6 +28,13 @@ class BaseState(object): self.get_region_name(strategy_step), details)) + def info_log(self, strategy_step, details): + LOG.info("Stage: %s, State: %s, Subcloud: %s, Details: %s" + % (strategy_step.stage, + strategy_step.state, + self.get_region_name(strategy_step), + details)) + @staticmethod def get_region_name(strategy_step): """Get the region name for a strategy step""" diff --git a/distributedcloud/dcmanager/manager/states/lock_host.py b/distributedcloud/dcmanager/manager/states/lock_host.py index 51b11f4bc..edf54fab7 100644 --- a/distributedcloud/dcmanager/manager/states/lock_host.py +++ b/distributedcloud/dcmanager/manager/states/lock_host.py @@ -5,12 +5,9 @@ # import time -from oslo_log import log as logging - from dcmanager.common.consts import ADMIN_LOCKED from dcmanager.manager.states.base import BaseState -LOG = logging.getLogger(__name__) DEFAULT_MAX_QUERIES = 6 DEFAULT_SLEEP_DURATION = 10 @@ -46,7 +43,7 @@ class LockHostState(BaseState): if host.administrative == ADMIN_LOCKED: msg = "Host: %s already: %s." % (self.target_hostname, host.administrative) - self.debug_log(strategy_step, msg) + self.info_log(strategy_step, msg) return True # Invoke the action @@ -63,7 +60,7 @@ class LockHostState(BaseState): if host.administrative == ADMIN_LOCKED: msg = "Host: %s is now: %s" % (self.target_hostname, host.administrative) - self.debug_log(strategy_step, msg) + self.info_log(strategy_step, msg) break counter += 1 if counter >= self.max_queries: diff --git a/distributedcloud/dcmanager/manager/states/unlock_host.py b/distributedcloud/dcmanager/manager/states/unlock_host.py index fcf2f4644..e8b99c7a5 100644 --- a/distributedcloud/dcmanager/manager/states/unlock_host.py +++ b/distributedcloud/dcmanager/manager/states/unlock_host.py @@ -5,12 +5,9 @@ # import time -from oslo_log import log as logging - from dcmanager.common.consts import ADMIN_UNLOCKED from dcmanager.manager.states.base import BaseState -LOG = logging.getLogger(__name__) DEFAULT_MAX_QUERIES = 6 DEFAULT_SLEEP_DURATION = 10 @@ -51,7 +48,7 @@ class UnlockHostState(BaseState): if host.administrative == ADMIN_UNLOCKED: msg = "Host: %s already: %s." % (self.target_hostname, host.administrative) - self.debug_log(strategy_step, msg) + self.info_log(strategy_step, msg) return True # Invoke the action @@ -68,7 +65,7 @@ class UnlockHostState(BaseState): if host.administrative == ADMIN_UNLOCKED: msg = "Host: %s is now: %s" % (self.target_hostname, host.administrative) - self.debug_log(strategy_step, msg) + self.info_log(strategy_step, msg) break async_counter += 1 # check_async_counter throws exception if loops exceeded or aborted diff --git a/distributedcloud/dcmanager/manager/states/upgrade/activating.py b/distributedcloud/dcmanager/manager/states/upgrade/activating.py index 4afcdd3b0..86a4af274 100644 --- a/distributedcloud/dcmanager/manager/states/upgrade/activating.py +++ b/distributedcloud/dcmanager/manager/states/upgrade/activating.py @@ -3,18 +3,19 @@ # # SPDX-License-Identifier: Apache-2.0 # -from oslo_log import log as logging - from dcmanager.manager.states.base import BaseState -LOG = logging.getLogger(__name__) +ALREADY_ACTIVATING_STATES = ['activation-requested', + 'activation-failed', + 'activation-complete', + 'activating'] -class ActivatingState(BaseState): +class ActivatingUpgradeState(BaseState): """Upgrade state actions for activating an upgrade""" def __init__(self): - super(ActivatingState, self).__init__() + super(ActivatingUpgradeState, self).__init__() def perform_state_action(self, strategy_step): """Activate an upgrade on a subcloud @@ -22,9 +23,28 @@ class ActivatingState(BaseState): Any exceptions raised by this method set the strategy to FAILED Returning normally from this method set the strategy to the next step """ - LOG.warning("ActivatingState has not been implemented yet.") + # get the keystone and sysinv clients for the subcloud + ks_client = self.get_keystone_client(strategy_step.subcloud.name) + sysinv_client = self.get_sysinv_client(strategy_step.subcloud.name, + ks_client.session) + upgrades = sysinv_client.get_upgrades() + + # If there are no existing upgrades, there is nothing to activate + if len(upgrades) == 0: + raise Exception("No upgrades were found to activate") + + # The list of upgrades will never contain more than one entry. + for upgrade in upgrades: + # Check if an existing upgrade is already activated + if upgrade.state in ALREADY_ACTIVATING_STATES: + self.info_log(strategy_step, + "Already in activating state:%s" % upgrade.state) + break + else: + # invoke the API 'upgrade-activate'. + # Throws an exception on failure (no upgrade found, bad host state) + sysinv_client.upgrade_activate() # When we return from this method without throwing an exception, the # state machine can proceed to the next state - LOG.warning("Faking transition to next state") return True diff --git a/distributedcloud/dcmanager/manager/states/upgrade/completing.py b/distributedcloud/dcmanager/manager/states/upgrade/completing.py index 05956375a..022f46a2d 100644 --- a/distributedcloud/dcmanager/manager/states/upgrade/completing.py +++ b/distributedcloud/dcmanager/manager/states/upgrade/completing.py @@ -3,18 +3,14 @@ # # SPDX-License-Identifier: Apache-2.0 # -from oslo_log import log as logging - from dcmanager.manager.states.base import BaseState -LOG = logging.getLogger(__name__) - -class CompletingState(BaseState): +class CompletingUpgradeState(BaseState): """Upgrade state actions for completing an upgrade""" def __init__(self): - super(CompletingState, self).__init__() + super(CompletingUpgradeState, self).__init__() def perform_state_action(self, strategy_step): """Complete an upgrade on a subcloud @@ -22,9 +18,24 @@ class CompletingState(BaseState): Any exceptions raised by this method set the strategy to FAILED Returning normally from this method set the strategy to the next step """ - LOG.warning("CompletingState has not been implemented yet.") + # get the keystone and sysinv clients for the subcloud + ks_client = self.get_keystone_client(strategy_step.subcloud.name) + sysinv_client = self.get_sysinv_client(strategy_step.subcloud.name, + ks_client.session) + + # upgrade-complete causes the upgrade to be deleted. + # if no upgrade exists, there is no need to call it. + # The API should always return a list, but check for None anyways + upgrades = sysinv_client.get_upgrades() + if len(upgrades) == 0: + self.info_log(strategy_step, + "No upgrades exist. Nothing needs completing") + return True + + # invoke the API 'upgrade-complete' + # This is a blocking call that raises an exception on failure. + sysinv_client.upgrade_complete() # When we return from this method without throwing an exception, the # state machine can proceed to the next state - LOG.warning("Faking transition to next state") return True diff --git a/distributedcloud/dcmanager/manager/states/upgrade/import_load.py b/distributedcloud/dcmanager/manager/states/upgrade/import_load.py deleted file mode 100644 index d8bebd491..000000000 --- a/distributedcloud/dcmanager/manager/states/upgrade/import_load.py +++ /dev/null @@ -1,30 +0,0 @@ -# -# Copyright (c) 2020 Wind River Systems, Inc. -# -# SPDX-License-Identifier: Apache-2.0 -# -from oslo_log import log as logging - -from dcmanager.manager.states.base import BaseState - -LOG = logging.getLogger(__name__) - - -class ImportLoadState(BaseState): - """Upgrade state for importing a load""" - - def __init__(self): - super(ImportLoadState, self).__init__() - - def perform_state_action(self, strategy_step): - """Import a load on a subcloud - - Any exceptions raised by this method set the strategy to FAILED - Returning normally from this method set the strategy to the next step - """ - LOG.warning("ImportLoadState has not been implemented yet.") - - # When we return from this method without throwing an exception, the - # state machine can proceed to the next state - LOG.warning("Faking transition to next state") - return True diff --git a/distributedcloud/dcmanager/manager/states/upgrade/importing_load.py b/distributedcloud/dcmanager/manager/states/upgrade/importing_load.py new file mode 100644 index 000000000..d91881349 --- /dev/null +++ b/distributedcloud/dcmanager/manager/states/upgrade/importing_load.py @@ -0,0 +1,55 @@ +# +# Copyright (c) 2020 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +from dcmanager.common import consts +from dcmanager.manager.states.base import BaseState +from dcmanager.manager.states.upgrade import utils + + +class ImportingLoadState(BaseState): + """Upgrade state for importing a load""" + + def __init__(self): + super(ImportingLoadState, self).__init__() + + def perform_state_action(self, strategy_step): + """Import a load on a subcloud + + Any exceptions raised by this method set the strategy to FAILED + Returning normally from this method set the strategy to the next step + """ + # determine the version of the system controller in region one + local_ks_client = self.get_keystone_client() + local_sysinv_client = \ + self.get_sysinv_client(consts.DEFAULT_REGION_NAME, + local_ks_client.session) + target_version = local_sysinv_client.get_system().software_version + + # get the keystone and sysinv clients for the subcloud + ks_client = self.get_keystone_client(strategy_step.subcloud.name) + sysinv_client = self.get_sysinv_client(strategy_step.subcloud.name, + ks_client.session) + + # Check if the load is already imported by checking what the version is + current_loads = sysinv_client.get_loads() + for load in current_loads: + if load.software_version == target_version: + self.info_log(strategy_step, + "Load:%s already found" % target_version) + return True + + # If we are here, the load needs to be imported + # ISO and SIG files are found in the vault under a version directory + iso_path, sig_path = utils.get_vault_load_files(target_version) + + # Call the API. + imported_load = sysinv_client.import_load(iso_path, sig_path) + new_load = imported_load.get('new_load', {}) + if new_load.get('software_version') != target_version: + raise Exception("The imported load was not the expected version") + + # When we return from this method without throwing an exception, the + # state machine can proceed to the next state + return True diff --git a/distributedcloud/dcmanager/manager/states/upgrade/installing_license.py b/distributedcloud/dcmanager/manager/states/upgrade/installing_license.py new file mode 100644 index 000000000..874d1b458 --- /dev/null +++ b/distributedcloud/dcmanager/manager/states/upgrade/installing_license.py @@ -0,0 +1,86 @@ +# +# Copyright (c) 2020 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +from dcmanager.common import consts +from dcmanager.common import exceptions +from dcmanager.manager.states.base import BaseState + +# When a license is not installed, this will be part of the API error string +LICENSE_FILE_NOT_FOUND_SUBSTRING = "License file not found" + + +class InstallingLicenseState(BaseState): + """Upgrade state action for installing a license""" + + def __init__(self): + super(InstallingLicenseState, self).__init__() + + @staticmethod + def license_up_to_date(target_license, existing_license): + return target_license == existing_license + + def perform_state_action(self, strategy_step): + """Install the License for a software upgrade in this subcloud + + Any exceptions raised by this method set the strategy to FAILED + Returning normally from this method set the strategy to the next step + """ + + # check if the the system controller has a license + local_ks_client = self.get_keystone_client() + local_sysinv_client = \ + self.get_sysinv_client(consts.DEFAULT_REGION_NAME, + local_ks_client.session) + system_controller_license = local_sysinv_client.get_license() + # get_license returns a dictionary with keys: content and error + # 'content' can be an empty string in success or failure case. + # 'error' is an empty string only in success case. + target_license = system_controller_license.get('content') + target_error = system_controller_license.get('error') + + # If the system controller does not have a license, do not attempt + # to install licenses on subclouds, simply proceed to the next stage + if len(target_error) != 0: + if LICENSE_FILE_NOT_FOUND_SUBSTRING in target_error: + self.info_log(strategy_step, + "System Controller License missing: %s." + % target_error) + return True + else: + # An unexpected error occurred querying the license + raise exceptions.LicenseInstallError( + subcloud_id=consts.SYSTEM_CONTROLLER_NAME) + + # retrieve the keystone session for the subcloud and query its license + subcloud_ks_client = \ + self.get_keystone_client(strategy_step.subcloud.name) + subcloud_sysinv_client = \ + self.get_sysinv_client(strategy_step.subcloud.name, + subcloud_ks_client.session) + subcloud_license_response = subcloud_sysinv_client.get_license() + subcloud_license = subcloud_license_response.get('content') + subcloud_error = subcloud_license_response.get('error') + + # Skip license install if the license is already up to date + # If there was not an error, there might be a license + if len(subcloud_error) == 0: + if self.license_up_to_date(target_license, subcloud_license): + self.info_log(strategy_step, "License up to date.") + return True + else: + self.debug_log(strategy_step, "License mismatch. Updating.") + else: + self.debug_log(strategy_step, "License missing. Installing.") + + # Install the license + install_rc = subcloud_sysinv_client.install_license(target_license) + install_error = install_rc.get('error') + if len(install_error) != 0: + raise exceptions.LicenseInstallError( + subcloud_id=strategy_step.subcloud_id) + + # The license has been successfully installed. Move to the next stage + self.info_log(strategy_step, "License installed.") + return True diff --git a/distributedcloud/dcmanager/manager/states/upgrade/starting_upgrade.py b/distributedcloud/dcmanager/manager/states/upgrade/starting_upgrade.py index 511fba37b..2b79073ed 100644 --- a/distributedcloud/dcmanager/manager/states/upgrade/starting_upgrade.py +++ b/distributedcloud/dcmanager/manager/states/upgrade/starting_upgrade.py @@ -3,18 +3,15 @@ # # SPDX-License-Identifier: Apache-2.0 # -from oslo_log import log as logging - from dcmanager.manager.states.base import BaseState -LOG = logging.getLogger(__name__) - class StartingUpgradeState(BaseState): """Upgrade state for starting an upgrade on a subcloud""" - def __init__(self): + def __init__(self, force=False): super(StartingUpgradeState, self).__init__() + self.force = force def perform_state_action(self, strategy_step): """Start an upgrade on a subcloud @@ -22,9 +19,24 @@ class StartingUpgradeState(BaseState): Any exceptions raised by this method set the strategy to FAILED Returning normally from this method set the strategy to the next step """ - LOG.warning("StartingUpgradeState has not been implemented yet.") + # get the keystone and sysinv clients for the subcloud + ks_client = self.get_keystone_client(strategy_step.subcloud.name) + sysinv_client = self.get_sysinv_client(strategy_step.subcloud.name, + ks_client.session) + + # Check if an existing upgrade is already in progress. + # The list of upgrades will never contain more than one entry. + upgrades = sysinv_client.get_upgrades() + if upgrades is not None and len(upgrades) > 0: + for upgrade in upgrades: + # If a previous upgrade exists (even one that failed) skip + self.info_log(strategy_step, + "An upgrade already exists: %s" % upgrade) + else: + # invoke the API 'upgrade-start'. + # This call is synchronous and throws an exception on failure. + sysinv_client.upgrade_start(self.force) # When we return from this method without throwing an exception, the # state machine can proceed to the next state - LOG.warning("Faking transition to next state") return True diff --git a/distributedcloud/dcmanager/manager/states/upgrade/utils.py b/distributedcloud/dcmanager/manager/states/upgrade/utils.py new file mode 100644 index 000000000..e7e21fd64 --- /dev/null +++ b/distributedcloud/dcmanager/manager/states/upgrade/utils.py @@ -0,0 +1,38 @@ +# +# Copyright (c) 2020 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +import os + +from dcmanager.common.exceptions import VaultLoadMissingError + +VAULT_LOADS_PATH = '/opt/dc-vault/loads' + + +def get_vault_load_files(target_version): + """Return a tuple for the ISO and SIG for this load version from the vault. + + The files can be imported to the vault using any name, but must end + in 'iso' or 'sig'. + : param target_version: The software version to search under the vault + """ + vault_dir = "{}/{}/".format(VAULT_LOADS_PATH, target_version) + + matching_iso = None + matching_sig = None + for a_file in os.listdir(vault_dir): + if a_file.lower().endswith(".iso"): + matching_iso = os.path.join(vault_dir, a_file) + continue + elif a_file.lower().endswith(".sig"): + matching_sig = os.path.join(vault_dir, a_file) + continue + # If no .iso or .sig is found, raise an exception + if matching_iso is None: + raise VaultLoadMissingError(file_type='.iso', vault_dir=vault_dir) + if matching_sig is None: + raise VaultLoadMissingError(file_type='.sig', vault_dir=vault_dir) + + # return the iso and sig for this load + return (matching_iso, matching_sig) diff --git a/distributedcloud/dcmanager/manager/sw_upgrade_orch_thread.py b/distributedcloud/dcmanager/manager/sw_upgrade_orch_thread.py index ea8c898de..490e5eab8 100644 --- a/distributedcloud/dcmanager/manager/sw_upgrade_orch_thread.py +++ b/distributedcloud/dcmanager/manager/sw_upgrade_orch_thread.py @@ -25,19 +25,52 @@ import time from oslo_log import log as logging -from dccommon.drivers.openstack.sdk_platform import OpenStackDriver -from dccommon.drivers.openstack.sysinv_v1 import SysinvClient - from dcmanager.common import consts from dcmanager.common import context from dcmanager.common import exceptions from dcmanager.common import scheduler from dcmanager.db import api as db_api +from dcmanager.manager.states.lock_host import LockHostState +from dcmanager.manager.states.unlock_host import UnlockHostState +from dcmanager.manager.states.upgrade.activating import ActivatingUpgradeState +from dcmanager.manager.states.upgrade.completing import CompletingUpgradeState +from dcmanager.manager.states.upgrade.importing_load import ImportingLoadState +from dcmanager.manager.states.upgrade.installing_license \ + import InstallingLicenseState +from dcmanager.manager.states.upgrade.migrating_data \ + import MigratingDataState +from dcmanager.manager.states.upgrade.starting_upgrade \ + import StartingUpgradeState +from dcmanager.manager.states.upgrade.upgrading_simplex \ + import UpgradingSimplexState LOG = logging.getLogger(__name__) -# When a license is not installed, this will be part of the API error string -LICENSE_FILE_NOT_FOUND_SUBSTRING = "License file not found" +# The state machine transition order for an APPLY +ORDERED_STATES = [ + consts.STRATEGY_STATE_INSTALLING_LICENSE, + consts.STRATEGY_STATE_IMPORTING_LOAD, + consts.STRATEGY_STATE_STARTING_UPGRADE, + consts.STRATEGY_STATE_LOCKING_CONTROLLER, + consts.STRATEGY_STATE_UPGRADING_SIMPLEX, + consts.STRATEGY_STATE_MIGRATING_DATA, + consts.STRATEGY_STATE_UNLOCKING_CONTROLLER, + consts.STRATEGY_STATE_ACTIVATING_UPGRADE, + consts.STRATEGY_STATE_COMPLETING_UPGRADE, +] + +# every state in ORDERED_STATES should have an operator +STATE_OPERATORS = { + consts.STRATEGY_STATE_INSTALLING_LICENSE: InstallingLicenseState, + consts.STRATEGY_STATE_IMPORTING_LOAD: ImportingLoadState, + consts.STRATEGY_STATE_STARTING_UPGRADE: StartingUpgradeState, + consts.STRATEGY_STATE_LOCKING_CONTROLLER: LockHostState, + consts.STRATEGY_STATE_UPGRADING_SIMPLEX: UpgradingSimplexState, + consts.STRATEGY_STATE_MIGRATING_DATA: MigratingDataState, + consts.STRATEGY_STATE_UNLOCKING_CONTROLLER: UnlockHostState, + consts.STRATEGY_STATE_ACTIVATING_UPGRADE: ActivatingUpgradeState, + consts.STRATEGY_STATE_COMPLETING_UPGRADE: CompletingUpgradeState, +} class SwUpgradeOrchThread(threading.Thread): @@ -72,7 +105,7 @@ class SwUpgradeOrchThread(threading.Thread): self.subcloud_workers = dict() # When an upgrade is initiated, this is the first state - self.starting_state = consts.STRATEGY_STATE_INSTALLING_LICENSE + self.starting_state = ORDERED_STATES[0] def stopped(self): return self._stop.isSet() @@ -87,22 +120,6 @@ class SwUpgradeOrchThread(threading.Thread): self.thread_group_manager.stop() LOG.info("SwUpgradeOrchThread Stopped") - @staticmethod - def get_ks_client(region_name=consts.DEFAULT_REGION_NAME): - """This will get a cached keystone client (and token)""" - try: - os_client = OpenStackDriver( - region_name=region_name, - region_clients=None) - return os_client.keystone_client - except Exception: - LOG.warn('Failure initializing KeystoneClient') - raise - - @staticmethod - def get_sysinv_client(region_name, session): - return SysinvClient(region_name, session) - @staticmethod def get_region_name(strategy_step): """Get the region name for a strategy step""" @@ -121,8 +138,22 @@ class SwUpgradeOrchThread(threading.Thread): return details @staticmethod - def license_up_to_date(target_license, existing_license): - return target_license == existing_license + def determine_state_operator(strategy_step): + """Return the state operator for the current state""" + state_operator = STATE_OPERATORS.get(strategy_step.state) + # instantiate and return the state_operator class + return state_operator() + + @staticmethod + def determine_next_state(strategy_step): + """Return next state for the strategy step based on current state.""" + # todo(abailey): next_state may differ for AIO, STD, etc.. subclouds + next_index = ORDERED_STATES.index(strategy_step.state) + 1 + if next_index < len(ORDERED_STATES): + next_state = ORDERED_STATES[next_index] + else: + next_state = consts.STRATEGY_STATE_COMPLETE + return next_state def strategy_step_update(self, subcloud_id, state=None, details=None): """Update the strategy step in the DB @@ -137,13 +168,13 @@ class SwUpgradeOrchThread(threading.Thread): consts.STRATEGY_STATE_ABORTED, consts.STRATEGY_STATE_FAILED]: finished_at = datetime.datetime.now() - db_api.strategy_step_update( - self.context, - subcloud_id, - state=state, - details=details, - started_at=started_at, - finished_at=finished_at) + # Return the updated object, in case we need to use its updated values + return db_api.strategy_step_update(self.context, + subcloud_id, + state=state, + details=details, + started_at=started_at, + finished_at=finished_at) def upgrade_orch(self): while not self.stopped(): @@ -288,216 +319,23 @@ class SwUpgradeOrchThread(threading.Thread): continue # We are just getting started, enter the first state - self.strategy_step_update( + # Use the updated value for calling process_upgrade_step + strategy_step = self.strategy_step_update( strategy_step.subcloud_id, - state=consts.STRATEGY_STATE_INSTALLING_LICENSE) - - # Initial step should log an error if a greenthread exists - # All other steps should not. + state=self.starting_state) + # Starting state should log an error if greenthread exists self.process_upgrade_step(region, strategy_step, - self.install_subcloud_license, log_error=True) - # todo(abailey): state and their method invoked can be managed - # using a dictionary to make this more maintainable. - elif strategy_step.state == \ - consts.STRATEGY_STATE_INSTALLING_LICENSE: - self.process_upgrade_step(region, - strategy_step, - self.install_subcloud_license, - log_error=False) - elif strategy_step.state == \ - consts.STRATEGY_STATE_IMPORTING_LOAD: - self.process_upgrade_step(region, - strategy_step, - self.import_subcloud_load, - log_error=False) - elif strategy_step.state == \ - consts.STRATEGY_STATE_STARTING_UPGRADE: - self.process_upgrade_step(region, - strategy_step, - self.start_subcloud_upgrade, - log_error=False) - elif strategy_step.state == \ - consts.STRATEGY_STATE_LOCKING_CONTROLLER: - self.process_upgrade_step(region, - strategy_step, - self.lock_subcloud_controller, - log_error=False) - elif strategy_step.state == \ - consts.STRATEGY_STATE_UPGRADING_SIMPLEX: - self.process_upgrade_step(region, - strategy_step, - self.upgrade_subcloud_simplex, - log_error=False) - elif strategy_step.state == \ - consts.STRATEGY_STATE_MIGRATING_DATA: - self.process_upgrade_step(region, - strategy_step, - self.migrate_subcloud_data, - log_error=False) - elif strategy_step.state == \ - consts.STRATEGY_STATE_UNLOCKING_CONTROLLER: - self.process_upgrade_step(region, - strategy_step, - self.unlock_subcloud_controller, - log_error=False) - elif strategy_step.state == \ - consts.STRATEGY_STATE_ACTIVATING: - self.process_upgrade_step(region, - strategy_step, - self.activate_subcloud, - log_error=False) - # todo(abailey): Add calls to self.process_upgrade_step - # for each additional state, with the appropriate thread - # method called. else: - LOG.error("Unimplemented state %s" % strategy_step.state) - self.strategy_step_update( - strategy_step.subcloud_id, - state=consts.STRATEGY_STATE_FAILED, - details=("Upgrade state not implemented: %s" - % strategy_step.state)) + self.process_upgrade_step(region, + strategy_step, + log_error=False) if self.stopped(): LOG.info("Exiting because task is stopped") return - def process_upgrade_step(self, - region, - strategy_step, - upgrade_thread_method, - log_error=False): - if region in self.subcloud_workers: - # A worker already exists. Let it finish whatever it was doing. - if log_error: - LOG.error("Worker should not exist for %s." % region) - else: - LOG.debug("Update worker exists for %s." % region) - else: - # Create a greenthread to start processing the upgrade for the - # subcloud and invoke the specified upgrade_thread_method - self.subcloud_workers[region] = \ - self.thread_group_manager.start(upgrade_thread_method, - strategy_step) - - def install_subcloud_license(self, strategy_step): - """Install the license for the upgrade in this subcloud - - Removes the worker reference after the operation is complete. - """ - - try: - LOG.info("Stage: %s for subcloud %s" - % (strategy_step.stage, - self.get_region_name(strategy_step))) - self.do_install_subcloud_license(strategy_step) - except Exception: - # Catch ALL exceptions and set the strategy to failed - LOG.exception("Install license failed for %s" - % self.get_region_name(strategy_step)) - self.strategy_step_update(strategy_step.subcloud_id, - state=consts.STRATEGY_STATE_FAILED, - details=("Install license failed")) - finally: - # The worker is done. - region = self.get_region_name(strategy_step) - if region in self.subcloud_workers: - del self.subcloud_workers[region] - - def do_install_subcloud_license(self, strategy_step): - """Install the License for a software upgrade in this subcloud""" - - # Note: no need to catch exceptions in this method. - - # next_state is the next state that the strategy will use on - # successful completion of this state - next_state = consts.STRATEGY_STATE_IMPORTING_LOAD - - # We check the system controller license for system controller and - # subclouds - local_ks_client = self.get_ks_client() - local_sysinv_client = \ - self.get_sysinv_client(consts.DEFAULT_REGION_NAME, - local_ks_client.session) - system_controller_license = local_sysinv_client.get_license() - # get_license returns a dictionary with keys: content and error - # 'content' can be an empty string in success or failure case. - # 'error' is an empty string only in success case. - target_license = system_controller_license.get('content') - target_error = system_controller_license.get('error') - - # If the system controller does not have a license, do not attempt - # to install licenses on subclouds, and simply proceed to the next stage - if len(target_error) != 0: - if LICENSE_FILE_NOT_FOUND_SUBSTRING in target_error: - LOG.debug("Stage:<%s>, Subcloud:<%s>. " - "System Controller License missing: %s." - % (strategy_step.stage, - self.get_region_name(strategy_step), - target_error)) - self.strategy_step_update(strategy_step.subcloud_id, - state=next_state) - return - else: - # An unexpected API error was returned. Fail this stage. - LOG.warning("Stage:<%s>, Subcloud:<%s>. " - "System Controller License query failed: %s." - % (strategy_step.stage, - self.get_region_name(strategy_step), - target_error)) - raise exceptions.LicenseMissingError( - subcloud_id=consts.SYSTEM_CONTROLLER_NAME) - - # retrieve the keystone session for the subcloud and query its license - subcloud_ks_client = self.get_ks_client(strategy_step.subcloud.name) - subcloud_sysinv_client = \ - self.get_sysinv_client(strategy_step.subcloud.name, - subcloud_ks_client.session) - subcloud_license_response = subcloud_sysinv_client.get_license() - subcloud_license = subcloud_license_response.get('content') - subcloud_error = subcloud_license_response.get('error') - - # Skip license install if the license is already up to date - # If there was not an error, there might be a license - if len(subcloud_error) == 0: - if self.license_up_to_date(target_license, subcloud_license): - LOG.debug("Stage:<%s>, Subcloud:<%s>. License up to date." - % (strategy_step.stage, - self.get_region_name(strategy_step))) - self.strategy_step_update(strategy_step.subcloud_id, - state=next_state) - return - else: - LOG.debug("Stage:<%s>, Subcloud:<%s>. " - "License mismatch. Updating." - % (strategy_step.stage, - self.get_region_name(strategy_step))) - else: - LOG.debug("Stage:<%s>, Subcloud:<%s>. " - "License missing. Installing." - % (strategy_step.stage, - self.get_region_name(strategy_step))) - - # Install the license - install_rc = subcloud_sysinv_client.install_license(target_license) - install_error = install_rc.get('error') - if len(install_error) != 0: - LOG.warning("Stage:<%s>, Subcloud:<%s>. " - "License install failed:<%s>." - % (strategy_step.stage, - self.get_region_name(strategy_step), - install_error)) - raise exceptions.LicenseInstallError( - subcloud_id=strategy_step.subcloud_id) - - # The license has been successfully installed. Move to the next stage - LOG.debug("Stage:<%s>, Subcloud:<%s>. " - "License installed." - % (strategy_step.stage, - self.get_region_name(strategy_step))) - self.strategy_step_update(strategy_step.subcloud_id, state=next_state) - def abort(self, sw_update_strategy): """Abort an upgrade strategy""" @@ -536,15 +374,33 @@ class SwUpgradeOrchThread(threading.Thread): LOG.exception(e) raise e - def perform_state_action(self, strategy_step, state_operator, next_state): + def process_upgrade_step(self, region, strategy_step, log_error=False): + """manage the green thread for calling perform_state_action""" + if region in self.subcloud_workers: + # A worker already exists. Let it finish whatever it was doing. + if log_error: + LOG.error("Worker should not exist for %s." % region) + else: + LOG.debug("Update worker exists for %s." % region) + else: + # Create a greenthread to start processing the upgrade for the + # subcloud and invoke the specified upgrade_thread_method + self.subcloud_workers[region] = \ + self.thread_group_manager.start(self.perform_state_action, + strategy_step) + + def perform_state_action(self, strategy_step): """Extensible state handler for processing and transitioning states """ try: LOG.info("Stage: %s, State: %s, Subcloud: %s" % (strategy_step.stage, strategy_step.state, self.get_region_name(strategy_step))) + # Instantiate the state operator and perform the state actions + state_operator = self.determine_state_operator(strategy_step) state_operator.perform_state_action(strategy_step) # If we get here without an exception raised, proceed to next state + next_state = self.determine_next_state(strategy_step) self.strategy_step_update(strategy_step.subcloud_id, state=next_state) except Exception as e: @@ -562,47 +418,3 @@ class SwUpgradeOrchThread(threading.Thread): region = self.get_region_name(strategy_step) if region in self.subcloud_workers: del self.subcloud_workers[region] - - # todo(abailey): convert license install to the same pattern as the other states - - def import_subcloud_load(self, strategy_step): - from dcmanager.manager.states.upgrade.import_load import ImportLoadState - self.perform_state_action(strategy_step, - ImportLoadState(), - consts.STRATEGY_STATE_STARTING_UPGRADE) - - def start_subcloud_upgrade(self, strategy_step): - from dcmanager.manager.states.upgrade.starting_upgrade import StartingUpgradeState - self.perform_state_action(strategy_step, - StartingUpgradeState(), - consts.STRATEGY_STATE_LOCKING_CONTROLLER) - - def lock_subcloud_controller(self, strategy_step): - from dcmanager.manager.states.lock_host import LockHostState - self.perform_state_action(strategy_step, - LockHostState(), - consts.STRATEGY_STATE_UPGRADING_SIMPLEX) - - def upgrade_subcloud_simplex(self, strategy_step): - from dcmanager.manager.states.upgrade.upgrading_simplex import UpgradingSimplexState - self.perform_state_action(strategy_step, - UpgradingSimplexState(), - consts.STRATEGY_STATE_MIGRATING_DATA) - - def migrate_subcloud_data(self, strategy_step): - from dcmanager.manager.states.upgrade.migrating_data import MigratingDataState - self.perform_state_action(strategy_step, - MigratingDataState(), - consts.STRATEGY_STATE_UNLOCKING_CONTROLLER) - - def unlock_subcloud_controller(self, strategy_step): - from dcmanager.manager.states.unlock_host import UnlockHostState - self.perform_state_action(strategy_step, - UnlockHostState(), - consts.STRATEGY_STATE_ACTIVATING) - - def activate_subcloud(self, strategy_step): - from dcmanager.manager.states.upgrade.activating import ActivatingState - self.perform_state_action(strategy_step, - ActivatingState(), - consts.STRATEGY_STATE_COMPLETE) diff --git a/distributedcloud/dcmanager/tests/base.py b/distributedcloud/dcmanager/tests/base.py index dae3868d3..7455db811 100644 --- a/distributedcloud/dcmanager/tests/base.py +++ b/distributedcloud/dcmanager/tests/base.py @@ -44,7 +44,7 @@ SUBCLOUD_SAMPLE_DATA_0 = [ "subcloud-4", # name "demo subcloud", # description "Ottawa-Lab-Aisle_3-Rack_C", # location - "20.01", # software-version + "12.34", # software-version "managed", # management-state "online", # availability-status "fd01:3::0/64", # management_subnet diff --git a/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subclouds.py b/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subclouds.py index 6263968cc..b521f3586 100644 --- a/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subclouds.py +++ b/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subclouds.py @@ -56,7 +56,7 @@ FAKE_SUBCLOUD_DATA = {"name": "subcloud1", FAKE_SUBCLOUD_INSTALL_VALUES = { "image": "http://192.168.101.2:8080/iso/bootimage.iso", - "software_version": "20.01", + "software_version": "12.34", "bootstrap_interface": "eno1", "bootstrap_address": "128.224.151.183", "bootstrap_address_prefix": 23, diff --git a/distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_activating_upgrade.py b/distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_activating_upgrade.py new file mode 100644 index 000000000..83d9ffde0 --- /dev/null +++ b/distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_activating_upgrade.py @@ -0,0 +1,91 @@ +# +# Copyright (c) 2020 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +import mock + +from dcmanager.common import consts + +from dcmanager.tests.unit.manager.states.upgrade.test_base import FakeUpgrade +from dcmanager.tests.unit.manager.states.upgrade.test_base \ + import TestSwUpgradeState + +VALID_UPGRADE = FakeUpgrade(state='imported') +ACTIVATING_UPGRADE = FakeUpgrade(state='activation-requested') +ALREADY_ACTIVATED_UPGRADE = FakeUpgrade(state='activation-complete') + + +class TestSwUpgradeActivatingStage(TestSwUpgradeState): + + def setUp(self): + super(TestSwUpgradeActivatingStage, self).setUp() + + # next state after activating an upgrade is 'completing' + self.on_success_state = consts.STRATEGY_STATE_COMPLETING_UPGRADE + + # Add the strategy_step state being processed by this unit test + self.strategy_step = \ + self.setup_strategy_step(consts.STRATEGY_STATE_ACTIVATING_UPGRADE) + + # Add mock API endpoints for sysinv client calls invcked by this state + self.sysinv_client.upgrade_activate = mock.MagicMock() + self.sysinv_client.get_upgrades = mock.MagicMock() + + def test_upgrade_subcloud_activating_upgrade_failure(self): + """Test the activating upgrade API call fails.""" + + # upgrade_activate will only be called if an appropriate upgrade exists + self.sysinv_client.get_upgrades.return_value = [VALID_UPGRADE, ] + + # API call raises an exception when it is rejected + self.sysinv_client.upgrade_activate.side_effect = \ + Exception("upgrade activate failed for some reason") + + # invoke the strategy state operation on the orch thread + self.worker.perform_state_action(self.strategy_step) + + # verify the expected API call was invoked + self.sysinv_client.upgrade_activate.assert_called() + + # Verify the state moves to 'failed' + self.assert_step_updated(self.strategy_step.subcloud_id, + consts.STRATEGY_STATE_FAILED) + + def test_upgrade_subcloud_activating_upgrade_success(self): + """Test the activating upgrade step succeeds.""" + + # upgrade_activate will only be called if an appropriate upgrade exists + self.sysinv_client.get_upgrades.return_value = [VALID_UPGRADE, ] + + # API call will not raise an exception, and will return an upgrade + self.sysinv_client.upgrade_activate.return_value = ACTIVATING_UPGRADE + + # invoke the strategy state operation on the orch thread + self.worker.perform_state_action(self.strategy_step) + + # verify the API cvall was invoked + self.sysinv_client.upgrade_activate.assert_called() + + # On success, the state should be updated to the next state + self.assert_step_updated(self.strategy_step.subcloud_id, + self.on_success_state) + + def test_upgrade_subcloud_activating_upgrade_skip_already_activated(self): + """Test the activating upgrade step skipped if already activated.""" + + # upgrade_activate will only be called if an appropriate upgrade exists + self.sysinv_client.get_upgrades.return_value = \ + [ALREADY_ACTIVATED_UPGRADE, ] + + # API call will not be invoked, so no need to mock it + + # invoke the strategy state operation on the orch thread + self.worker.perform_state_action(self.strategy_step) + + # upgrade is already in one of the activating states so skip activating + self.sysinv_client.upgrade_activate.assert_not_called() + + # On success, the state is set to the next state + self.assert_step_updated(self.strategy_step.subcloud_id, + self.on_success_state) diff --git a/distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_base.py b/distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_base.py index 9cff06f70..31afac3ac 100644 --- a/distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_base.py +++ b/distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_base.py @@ -4,14 +4,15 @@ # SPDX-License-Identifier: Apache-2.0 # import mock +import uuid from dcmanager.manager.states.base import BaseState from sysinv.common import constants as sysinv_constants from dcmanager.tests.unit.manager.test_sw_upgrade import TestSwUpgrade -CURRENT_LOAD = '20.01' -UPDATED_LOAD = '20.06' +PREVIOUS_VERSION = '12.34' +UPGRADED_VERSION = '56.78' class FakeKeystoneClient(object): @@ -19,6 +20,47 @@ class FakeKeystoneClient(object): self.session = mock.MagicMock() +class FakeLoad(object): + def __init__(self, + obj_id, + compatible_version='N/A', + required_patches='N/A', + software_version=PREVIOUS_VERSION, + state='active', + created_at=None, + updated_at=None): + self.id = obj_id + self.uuid = uuid.uuid4() + self.required_patches = required_patches + self.software_version = software_version + self.state = state + self.created_at = created_at + self.updated_at = updated_at + + +class FakeSystem(object): + def __init__(self, + obj_id=1, + software_version=UPGRADED_VERSION): + self.id = obj_id + self.uuid = uuid.uuid4() + self.software_version = software_version + + +class FakeUpgrade(object): + def __init__(self, + obj_id=1, + state='completed', + from_release=PREVIOUS_VERSION, + to_release=UPGRADED_VERSION): + self.id = obj_id + self.uuid = uuid.uuid4() + self.state = state + self.from_release = from_release + self.to_release = to_release + self.links = [] + + class FakeSysinvClient(object): def __init__(self): pass @@ -31,7 +73,7 @@ class FakeController(object): administrative=sysinv_constants.ADMIN_UNLOCKED, availability=sysinv_constants.AVAILABILITY_AVAILABLE, ihost_action=None, - target_load=CURRENT_LOAD, + target_load=UPGRADED_VERSION, task=None): self.id = host_id self.hostname = hostname diff --git a/distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_completing_upgrade.py b/distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_completing_upgrade.py new file mode 100644 index 000000000..8a127b3a2 --- /dev/null +++ b/distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_completing_upgrade.py @@ -0,0 +1,90 @@ +# +# Copyright (c) 2020 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +import mock + +from dcmanager.common import consts + +from dcmanager.tests.unit.manager.states.upgrade.test_base import FakeUpgrade +from dcmanager.tests.unit.manager.states.upgrade.test_base \ + import TestSwUpgradeState + +VALID_UPGRADE = FakeUpgrade(state='activation-complete') +INVALID_UPGRADE = FakeUpgrade(state='aborting') + + +class TestSwUpgradeCompletingStage(TestSwUpgradeState): + + def setUp(self): + super(TestSwUpgradeCompletingStage, self).setUp() + + # next state after completing an upgrade is 'complete' + self.on_success_state = consts.STRATEGY_STATE_COMPLETE + + # Add the strategy_step state being processed by this unit test + self.strategy_step = \ + self.setup_strategy_step(consts.STRATEGY_STATE_COMPLETING_UPGRADE) + + # Add mock API endpoints for sysinv client calls invcked by this state + self.sysinv_client.upgrade_complete = mock.MagicMock() + self.sysinv_client.get_upgrades = mock.MagicMock() + + def test_upgrade_subcloud_completing_upgrade_failure(self): + """Test the completing upgrade API call fails.""" + + # upgrade_complete will only be called if an appropriate upgrade exists + self.sysinv_client.get_upgrades.return_value = [VALID_UPGRADE, ] + + # API call raises an exception when it is rejected + self.sysinv_client.upgrade_complete.side_effect = \ + Exception("upgrade complete failed for some reason") + + # invoke the strategy state operation on the orch thread + self.worker.perform_state_action(self.strategy_step) + + # verify the expected API call was invoked + self.sysinv_client.upgrade_complete.assert_called() + + # Verify the state moves to 'failed' + self.assert_step_updated(self.strategy_step.subcloud_id, + consts.STRATEGY_STATE_FAILED) + + def test_upgrade_subcloud_completing_upgrade_success(self): + """Test the completing upgrade step succeeds.""" + + # upgrade_complete will only be called if an appropriate upgrade exists + self.sysinv_client.get_upgrades.return_value = [VALID_UPGRADE, ] + + # API call will not raise an exception. It will delete the upgrade + self.sysinv_client.upgrade_complete.return_value = None + + # invoke the strategy state operation on the orch thread + self.worker.perform_state_action(self.strategy_step) + + # verify the API cvall was invoked + self.sysinv_client.upgrade_complete.assert_called() + + # On success, the state should be updated to the next state + self.assert_step_updated(self.strategy_step.subcloud_id, + self.on_success_state) + + def test_upgrade_subcloud_completing_upgrade_skip_already_completed(self): + """Test the completing upgrade step skipped if already completed.""" + + # upgrade_complete will only be called if an appropriate upgrade exists + # If the upgrade has been deleted, there is nothing to complete + self.sysinv_client.get_upgrades.return_value = [] + + # API call will not be invoked, so no need to mock it + + # invoke the strategy state operation on the orch thread + self.worker.perform_state_action(self.strategy_step) + + # upgrade is already in one of the completing states so skip completing + self.sysinv_client.upgrade_complete.assert_not_called() + + # On success, the state is set to the next state + self.assert_step_updated(self.strategy_step.subcloud_id, + self.on_success_state) diff --git a/distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_importing_load.py b/distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_importing_load.py new file mode 100644 index 000000000..96c20e6a8 --- /dev/null +++ b/distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_importing_load.py @@ -0,0 +1,155 @@ +# +# Copyright (c) 2020 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +import mock + +from dcmanager.common import consts +from dcmanager.common.exceptions import VaultLoadMissingError + +from dcmanager.tests.unit.manager.states.upgrade.test_base import FakeLoad +from dcmanager.tests.unit.manager.states.upgrade.test_base import FakeSystem +from dcmanager.tests.unit.manager.states.upgrade.test_base \ + import PREVIOUS_VERSION +from dcmanager.tests.unit.manager.states.upgrade.test_base \ + import TestSwUpgradeState +from dcmanager.tests.unit.manager.states.upgrade.test_base \ + import UPGRADED_VERSION + +PREVIOUS_LOAD = FakeLoad(1, software_version=PREVIOUS_VERSION) +UPGRADED_LOAD = FakeLoad(2, + compatible_version=PREVIOUS_VERSION, + software_version=UPGRADED_VERSION) + +DEST_LOAD_EXISTS = [PREVIOUS_LOAD, UPGRADED_LOAD, ] +DEST_LOAD_MISSING = [PREVIOUS_LOAD, ] + +FAKE_ISO = '/opt/dc-vault/loads/' + UPGRADED_VERSION + '/bootimage.iso' +FAKE_SIG = '/opt/dc-vault/loads/' + UPGRADED_VERSION + '/bootimage.sig' + +FAILED_IMPORT_RESPONSE = 'kaboom' +SUCCESS_IMPORT_RESPONSE = { + 'new_load': { + 'id': 2, + 'uuid': 'aaa4b4c6-8536-41f6-87ea-211d208a723b', + 'compatible_version': PREVIOUS_VERSION, + 'required_patches': '', + 'software_version': UPGRADED_VERSION, + 'state': 'importing', + 'created_at': '2020-06-01 12:12:12+00:00', + 'updated_at': None + } +} + + +class TestSwUpgradeImportingLoadStage(TestSwUpgradeState): + + def setUp(self): + super(TestSwUpgradeImportingLoadStage, self).setUp() + + # next state after 'importing load' is 'starting upgrade' + self.on_success_state = consts.STRATEGY_STATE_STARTING_UPGRADE + + # Add the strategy_step state being processed by this unit test + self.strategy_step = \ + self.setup_strategy_step(consts.STRATEGY_STATE_IMPORTING_LOAD) + + # Add mock API endpoints for sysinv client calls invcked by this state + self.sysinv_client.get_system = mock.MagicMock() + self.sysinv_client.get_system.return_value = FakeSystem() + self.sysinv_client.get_loads = mock.MagicMock() + self.sysinv_client.import_load = mock.MagicMock() + + @mock.patch('dcmanager.manager.states.upgrade.utils.get_vault_load_files') + def test_upgrade_subcloud_importing_load_failure(self, mock_vault_files): + """Test importing load step where the import_load API call fails.""" + + # simulate determine_matching_load finding the iso and sig in the vault + mock_vault_files.return_value = (FAKE_ISO, FAKE_SIG) + + # Simulate the target load has not been imported yet on the subcloud + self.sysinv_client.get_loads.return_value = DEST_LOAD_MISSING + + # Simulate an API failure on the subcloud. + self.sysinv_client.import_load.return_value = FAILED_IMPORT_RESPONSE + + # invoke the strategy state operation on the orch thread + self.worker.perform_state_action(self.strategy_step) + + # verify the import load API call was invoked + self.sysinv_client.import_load.assert_called() + + # Verify a failure leads to a state failure + self.assert_step_updated(self.strategy_step.subcloud_id, + consts.STRATEGY_STATE_FAILED) + + @mock.patch('dcmanager.manager.states.upgrade.utils.get_vault_load_files') + def test_upgrade_subcloud_importing_load_success(self, mock_vault_files): + """Test the importing load step succeeds. + + The load will be imported on the subcloud when the subcloud does not + have the load already imported, and the API call succeeds to import it. + """ + # simulate determine_matching_load finding the iso and sig in the vault + mock_vault_files.return_value = (FAKE_ISO, FAKE_SIG) + + # Simulate the target load has not been imported yet on the subcloud + self.sysinv_client.get_loads.return_value = DEST_LOAD_MISSING + + # Simulate an API success on the subcloud. + self.sysinv_client.import_load.return_value = SUCCESS_IMPORT_RESPONSE + + # invoke the strategy state operation on the orch thread + self.worker.perform_state_action(self.strategy_step) + + # verify the import load API call was invoked + self.sysinv_client.import_load.assert_called() + + # On success, should have moved to the next state + self.assert_step_updated(self.strategy_step.subcloud_id, + self.on_success_state) + + @mock.patch('dcmanager.manager.states.upgrade.utils.get_vault_load_files') + def test_upgrade_subcloud_importing_load_fails_missing_vault_files( + self, + mock_determine_matching_load): + """Test importing load fails when files are not in the vault.""" + + mock_determine_matching_load.side_effect = \ + VaultLoadMissingError(file_type='.iso', vault_dir='/mock/vault/') + + # Simulate the target load has not been imported yet on the subcloud + self.sysinv_client.get_loads.return_value = DEST_LOAD_MISSING + + # Simulate an API success on the subcloud. It should not get here. + self.sysinv_client.import_load.return_value = SUCCESS_IMPORT_RESPONSE + + # invoke the strategy state operation on the orch thread + self.worker.perform_state_action(self.strategy_step) + + # verify the import load API call was never invoked + self.sysinv_client.import_load.assert_not_called() + + # Verify a failure leads to a state failure + self.assert_step_updated(self.strategy_step.subcloud_id, + consts.STRATEGY_STATE_FAILED) + + def test_upgrade_subcloud_importing_load_skip_existing(self): + """Test the importing load step skipped due to load already there""" + + # Simulate the target load has been previously imported on the subcloud + self.sysinv_client.get_loads.return_value = DEST_LOAD_EXISTS + + # Simulate an API failure for import_load. It should not be called. + self.sysinv_client.import_load.return_value = FAILED_IMPORT_RESPONSE + + # invoke the strategy state operation on the orch thread + self.worker.perform_state_action(self.strategy_step) + + # The import_load should not have been attempted + self.sysinv_client.import_load.assert_not_called() + + # On success, should have moved to the next state + self.assert_step_updated(self.strategy_step.subcloud_id, + self.on_success_state) diff --git a/distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_installing_license.py b/distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_installing_license.py new file mode 100644 index 000000000..c550bfa14 --- /dev/null +++ b/distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_installing_license.py @@ -0,0 +1,158 @@ +# +# Copyright (c) 2020 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +import mock + +from dcmanager.common import consts + +from dcmanager.tests.unit.manager.states.upgrade.test_base \ + import TestSwUpgradeState + +MISSING_LICENSE_RESPONSE = { + u'content': u'', + u'error': u'License file not found. A license may not have been installed.' +} + +LICENSE_VALID_RESPONSE = { + u'content': u'A valid license', + u'error': u'' +} + +ALTERNATE_LICENSE_RESPONSE = { + u'content': u'A different valid license', + u'error': u'' +} + + +class TestSwUpgradeInstallingLicenseStage(TestSwUpgradeState): + + def setUp(self): + super(TestSwUpgradeInstallingLicenseStage, self).setUp() + + # next state after installing a license is 'importing load' + self.on_success_state = consts.STRATEGY_STATE_IMPORTING_LOAD + + # Add the strategy_step state being processed by this unit test + self.strategy_step = \ + self.setup_strategy_step(consts.STRATEGY_STATE_INSTALLING_LICENSE) + + # Add mock API endpoints for sysinv client calls invcked by this state + self.sysinv_client.get_license = mock.MagicMock() + self.sysinv_client.install_license = mock.MagicMock() + + def test_upgrade_subcloud_license_install_failure(self): + """Test the installing license step where the install fails. + + The system controller has a license, but the API call to install on the + subcloud fails. + """ + + # Order of get_license calls: + # first license query is to system controller + # second license query is to subcloud (should be missing) + self.sysinv_client.get_license.side_effect = [LICENSE_VALID_RESPONSE, + MISSING_LICENSE_RESPONSE] + + # Simulate a license install failure on the subcloud + self.sysinv_client.install_license.return_value = \ + MISSING_LICENSE_RESPONSE + + # invoke the strategy state operation on the orch thread + self.worker.perform_state_action(self.strategy_step) + + # verify the license install was invoked + self.sysinv_client.install_license.assert_called() + + # Verify a install_license failure leads to a state failure + self.assert_step_updated(self.strategy_step.subcloud_id, + consts.STRATEGY_STATE_FAILED) + + def test_upgrade_subcloud_license_install_success(self): + """Test the install license step succeeds. + + The license will be installed on the subcloud when system controller + has a license, the subcloud does not have a license, and the API call + succeeds. + """ + + # Order of get_license calls: + # first license query is to system controller + # second license query is to subcloud (should be missing) + self.sysinv_client.get_license.side_effect = [LICENSE_VALID_RESPONSE, + MISSING_LICENSE_RESPONSE] + + # A license install should return a success + self.sysinv_client.install_license.return_value = \ + LICENSE_VALID_RESPONSE + + # invoke the strategy state operation on the orch thread + self.worker.perform_state_action(self.strategy_step) + + # verify the license install was invoked + self.sysinv_client.install_license.assert_called() + + # On success, the next state after installing license is importing load + self.assert_step_updated(self.strategy_step.subcloud_id, + self.on_success_state) + + def test_upgrade_subcloud_license_skip_existing(self): + """Test the install license step skipped due to license up to date""" + + # Order of get_license calls: + # first license query is to system controller + # second license query is to subcloud + self.sysinv_client.get_license.side_effect = [LICENSE_VALID_RESPONSE, + LICENSE_VALID_RESPONSE] + + # invoke the strategy state operation on the orch thread + self.worker.perform_state_action(self.strategy_step) + + # A license install should not have been attempted due to the license + # already being up to date + self.sysinv_client.install_license.assert_not_called() + + # On success, the next state after installing license is importing load + self.assert_step_updated(self.strategy_step.subcloud_id, + self.on_success_state) + + def test_upgrade_subcloud_license_overrides_mismatched_license(self): + """Test the install license overrides a mismatched license""" + + # Order of get_license calls: + # first license query is to system controller + # second license query is to subcloud (should be valid but different) + self.sysinv_client.get_license.side_effect = \ + [LICENSE_VALID_RESPONSE, + ALTERNATE_LICENSE_RESPONSE] + + # A license install should return a success + self.sysinv_client.install_license.return_value = \ + LICENSE_VALID_RESPONSE + + # invoke the strategy state operation on the orch thread + self.worker.perform_state_action(self.strategy_step) + + # verify the license install was invoked + self.sysinv_client.install_license.assert_called() + + # Verify it successfully moves to the next step + self.assert_step_updated(self.strategy_step.subcloud_id, + self.on_success_state) + + def test_upgrade_subcloud_license_skip_when_no_sys_controller_lic(self): + """Test license install skipped when no license on system controller.""" + + # Only makes one query: to system controller + self.sysinv_client.get_license.return_value = MISSING_LICENSE_RESPONSE + + # invoke the strategy state operation on the orch thread + self.worker.perform_state_action(self.strategy_step) + + # Should skip install_license API call + self.sysinv_client.install_license.assert_not_called() + + # Verify it successfully moves to the next step + self.assert_step_updated(self.strategy_step.subcloud_id, + self.on_success_state) diff --git a/distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_lock_controller.py b/distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_lock_controller.py index 4df13d311..96df6df7c 100644 --- a/distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_lock_controller.py +++ b/distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_lock_controller.py @@ -56,8 +56,8 @@ class TestSwUpgradeLockControllerStage(TestSwUpgradeState): # mock the API call as failed on the subcloud self.sysinv_client.lock_host.return_value = CONTROLLER_0_LOCKING - # invoke the strategy state operation - self.worker.lock_subcloud_controller(self.strategy_step) + # invoke the strategy state operation on the orch thread + self.worker.perform_state_action(self.strategy_step) # verify the lock command was actually attempted self.sysinv_client.lock_host.assert_called() @@ -72,8 +72,8 @@ class TestSwUpgradeLockControllerStage(TestSwUpgradeState): # mock the controller host query as being already locked self.sysinv_client.get_host.return_value = CONTROLLER_0_LOCKED - # invoke the strategy state operation - self.worker.lock_subcloud_controller(self.strategy_step) + # invoke the strategy state operation on the orch thread + self.worker.perform_state_action(self.strategy_step) # verify the lock command was never attempted self.sysinv_client.lock_host.assert_not_called() @@ -95,8 +95,8 @@ class TestSwUpgradeLockControllerStage(TestSwUpgradeState): # mock the API call as successful on the subcloud self.sysinv_client.lock_host.return_value = CONTROLLER_0_LOCKING - # invoke the strategy state operation - self.worker.lock_subcloud_controller(self.strategy_step) + # invoke the strategy state operation on the orch thread + self.worker.perform_state_action(self.strategy_step) # verify the lock command was actually attempted self.sysinv_client.lock_host.assert_called() @@ -118,8 +118,8 @@ class TestSwUpgradeLockControllerStage(TestSwUpgradeState): # mock the API call as failed on the subcloud self.sysinv_client.lock_host.return_value = CONTROLLER_0_LOCKING_FAILED - # invoke the strategy state operation - self.worker.lock_subcloud_controller(self.strategy_step) + # invoke the strategy state operation on the orch thread + self.worker.perform_state_action(self.strategy_step) # verify the lock command was actually attempted self.sysinv_client.lock_host.assert_called() @@ -135,8 +135,8 @@ class TestSwUpgradeLockControllerStage(TestSwUpgradeState): self.sysinv_client.get_host.side_effect = \ Exception("Unable to find host controller-0") - # invoke the strategy state operation - self.worker.lock_subcloud_controller(self.strategy_step) + # invoke the strategy state operation on the orch thread + self.worker.perform_state_action(self.strategy_step) # verify the lock command was never attempted self.sysinv_client.lock_host.assert_not_called() diff --git a/distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_starting_upgrade.py b/distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_starting_upgrade.py new file mode 100644 index 000000000..f32c6da8e --- /dev/null +++ b/distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_starting_upgrade.py @@ -0,0 +1,124 @@ +# +# Copyright (c) 2020 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +import mock + +from dcmanager.common import consts + +from dcmanager.tests.unit.manager.states.upgrade.test_base import FakeUpgrade +from dcmanager.tests.unit.manager.states.upgrade.test_base \ + import TestSwUpgradeState + +UPGRADE_ABORTING = [FakeUpgrade(state='aborting'), ] +UPGRADE_STARTED = [FakeUpgrade(state='started'), ] +SUCCESS_UPGRADE_START = 'I do not know what this looks like yet' + + +class TestSwUpgradeStartingUpgradeStage(TestSwUpgradeState): + + def setUp(self): + super(TestSwUpgradeStartingUpgradeStage, self).setUp() + + # next state after 'starting upgrade' is 'migrating data' + self.on_success_state = consts.STRATEGY_STATE_LOCKING_CONTROLLER + + # Add the strategy_step state being processed by this unit test + self.strategy_step = \ + self.setup_strategy_step(consts.STRATEGY_STATE_STARTING_UPGRADE) + + # Add mock API endpoints for sysinv client calls invcked by this state + self.sysinv_client.upgrade_start = mock.MagicMock() + self.sysinv_client.get_upgrades = mock.MagicMock() + + def test_upgrade_subcloud_upgrade_start_failure(self): + """Test the upgrade_start where the API call fails. + + The upgrade_start call fails due to a validation check such as from + the health-query check. + """ + + # No upgrades should yet exist in the DB / API + self.sysinv_client.get_upgrades.return_value = [] + + # Simulate an upgrade_start failure on the subcloud. + # The API throws an exception rather than returning an error response + self.sysinv_client.upgrade_start.side_effect = \ + Exception("HTTPBadRequest: upgrade-start rejected: " + "System is not in a valid state for upgrades. " + "Run system health-query-upgrade for more details.") + + # invoke the strategy state operation on the orch thread + self.worker.perform_state_action(self.strategy_step) + + # verify the API call that failed was invoked + self.sysinv_client.upgrade_start.assert_called() + + # Verify the API failure leads to a state failure + self.assert_step_updated(self.strategy_step.subcloud_id, + consts.STRATEGY_STATE_FAILED) + + def test_upgrade_subcloud_upgrade_start_success(self): + """Test upgrade_start where the API call succeeds. + + This will result in an upgrade being created with the appropriate + state. + """ + + # No upgrades should yet exist in the DB / API + self.sysinv_client.get_upgrades.return_value = [] + + # Simulate an upgrade_start succeeds on the subcloud + self.sysinv_client.upgrade_start.return_value = SUCCESS_UPGRADE_START + + # invoke the strategy state operation on the orch thread + self.worker.perform_state_action(self.strategy_step) + + # verify the API call that succeeded was actually invoked + self.sysinv_client.upgrade_start.assert_called() + + # On success, the state should transition to the next state + self.assert_step_updated(self.strategy_step.subcloud_id, + self.on_success_state) + + def test_upgrade_subcloud_upgrade_start_skip_already_started(self): + """Test upgrade_start where the upgrade is already started.""" + + # An already started upgrade exists in the DB""" + self.sysinv_client.get_upgrades.return_value = [UPGRADE_STARTED, ] + + # upgrade_start should not be invoked, so can be mocked as 'failed' + # by raising an exception + self.sysinv_client.upgrade_start.side_effect = \ + Exception("HTTPBadRequest: this is a fake exception") + + # invoke the strategy state operation on the orch thread + self.worker.perform_state_action(self.strategy_step) + + # upgrade_start API call should not have been attempted due to the + # existing upgrade already in started state. + self.sysinv_client.upgrade_start.assert_not_called() + + # On success, the state should transition to the next state + self.assert_step_updated(self.strategy_step.subcloud_id, + self.on_success_state) + + def test_upgrade_subcloud_upgrade_start_fails_bad_existing_upgrade(self): + """Test the upgrade_start fails due to a bad existing upgrade.""" + + # An already started upgrade exists in the DB but is in bad shape.""" + self.sysinv_client.get_upgrades.return_value = [UPGRADE_ABORTING, ] + + # upgrade_start will NOT be invoked. No need to mock it. + + # invoke the strategy state operation on the orch thread + self.worker.perform_state_action(self.strategy_step) + + # upgrade_start API call should not have been attempted due to the + # invalid existing upgrade that needs to be cleaned up. + self.sysinv_client.upgrade_start.assert_not_called() + + # Verify it failed and moves to the next step + self.assert_step_updated(self.strategy_step.subcloud_id, + self.on_success_state) diff --git a/distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_unlock_controller.py b/distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_unlock_controller.py index d6a9c8230..14db98ede 100644 --- a/distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_unlock_controller.py +++ b/distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_unlock_controller.py @@ -31,7 +31,7 @@ class TestSwUpgradeUnlockControllerStage(TestSwUpgradeState): super(TestSwUpgradeUnlockControllerStage, self).setUp() # next state after a successful unlock is 'activating' - self.on_success_state = consts.STRATEGY_STATE_ACTIVATING + self.on_success_state = consts.STRATEGY_STATE_ACTIVATING_UPGRADE # Add the strategy_step state being processed by this unit test self.strategy_step = self.setup_strategy_step( @@ -56,8 +56,8 @@ class TestSwUpgradeUnlockControllerStage(TestSwUpgradeState): # mock the API call as failed on the subcloud self.sysinv_client.unlock_host.return_value = CONTROLLER_0_UNLOCKING - # invoke the strategy state operation - self.worker.unlock_subcloud_controller(self.strategy_step) + # invoke the strategy state operation on the orch thread + self.worker.perform_state_action(self.strategy_step) # verify the unlock command was actually attempted self.sysinv_client.unlock_host.assert_called() @@ -72,8 +72,8 @@ class TestSwUpgradeUnlockControllerStage(TestSwUpgradeState): # mock the controller host query as being already unlocked self.sysinv_client.get_host.return_value = CONTROLLER_0_UNLOCKED - # invoke the strategy state operation - self.worker.unlock_subcloud_controller(self.strategy_step) + # invoke the strategy state operation on the orch thread + self.worker.perform_state_action(self.strategy_step) # verify the unlock command was never attempted self.sysinv_client.unlock_host.assert_not_called() @@ -95,8 +95,8 @@ class TestSwUpgradeUnlockControllerStage(TestSwUpgradeState): # mock the API call as successful on the subcloud self.sysinv_client.unlock_host.return_value = CONTROLLER_0_UNLOCKING - # invoke the strategy state operation - self.worker.unlock_subcloud_controller(self.strategy_step) + # invoke the strategy state operation on the orch thread + self.worker.perform_state_action(self.strategy_step) # verify the lock command was actually attempted self.sysinv_client.unlock_host.assert_called() @@ -119,8 +119,8 @@ class TestSwUpgradeUnlockControllerStage(TestSwUpgradeState): self.sysinv_client.unlock_host.return_value = \ CONTROLLER_0_UNLOCKING_FAILED - # invoke the strategy state operation - self.worker.unlock_subcloud_controller(self.strategy_step) + # invoke the strategy state operation on the orch thread + self.worker.perform_state_action(self.strategy_step) # verify the unlock command was actually attempted self.sysinv_client.unlock_host.assert_called() @@ -136,8 +136,8 @@ class TestSwUpgradeUnlockControllerStage(TestSwUpgradeState): self.sysinv_client.get_host.side_effect = \ Exception("Unable to find host controller-0") - # invoke the strategy state operation - self.worker.unlock_subcloud_controller(self.strategy_step) + # invoke the strategy state operation on the orch thread + self.worker.perform_state_action(self.strategy_step) # verify the unlock command was never attempted self.sysinv_client.unlock_host.assert_not_called() diff --git a/distributedcloud/dcmanager/tests/unit/manager/test_sw_upgrade.py b/distributedcloud/dcmanager/tests/unit/manager/test_sw_upgrade.py index 0567de5ef..444e9b524 100644 --- a/distributedcloud/dcmanager/tests/unit/manager/test_sw_upgrade.py +++ b/distributedcloud/dcmanager/tests/unit/manager/test_sw_upgrade.py @@ -33,7 +33,6 @@ from dcmanager.tests.unit.manager.test_sw_update_manager \ import Subcloud from dcmanager.tests import utils - CONF = cfg.CONF FAKE_ID = '1' FAKE_SW_UPDATE_DATA = { @@ -53,29 +52,6 @@ FAKE_STRATEGY_STEP_DATA = { "subcloud": None } -MISSING_LICENSE_RESPONSE = { - u'content': u'', - u'error': u'License file not found. A license may not have been installed.' -} - -LICENSE_VALID_RESPONSE = { - u'content': u'A valid license', - u'error': u'' -} - -ALTERNATE_LICENSE_RESPONSE = { - u'content': u'A different valid license', - u'error': u'' -} - - -class FakeSysinvClient(object): - - def __init__(self): - super(FakeSysinvClient, self).__init__() - self.get_license = mock.MagicMock() - self.install_license = mock.MagicMock() - class TestSwUpgrade(base.DCManagerTestCase): def setUp(self): @@ -99,14 +75,6 @@ class TestSwUpgrade(base.DCManagerTestCase): self.fake_patch_orch_thread self.addCleanup(p.stop) - # Mock the sysinv client - self.fake_sysinv_client = FakeSysinvClient() - p = mock.patch.object(sw_upgrade_orch_thread.SwUpgradeOrchThread, - 'get_sysinv_client') - self.mock_sysinv_client = p.start() - self.mock_sysinv_client.return_value = self.fake_sysinv_client - self.addCleanup(p.stop) - # Mock db_api p = mock.patch.object(sw_upgrade_orch_thread, 'db_api') self.mock_db_api = p.start() @@ -128,7 +96,6 @@ class TestSwUpgrade(base.DCManagerTestCase): mock_dcmanager_audit_api = mock.Mock() worker = sw_update_manager.SwUpgradeOrchThread(mock_strategy_lock, mock_dcmanager_audit_api) - worker.get_ks_client = mock.Mock() return worker def assert_step_updated(self, subcloud_id, update_state): @@ -140,168 +107,3 @@ class TestSwUpgrade(base.DCManagerTestCase): started_at=mock.ANY, finished_at=mock.ANY, ) - - -class TestSwUpgradeLicenseStage(TestSwUpgrade): - - def setUp(self): - super(TestSwUpgradeLicenseStage, self).setUp() - self.strategy_step = \ - self.setup_strategy_step(consts.STRATEGY_STATE_INSTALLING_LICENSE) - - def test_upgrade_subcloud_license_install_failure(self): - # Test the install subcloud license step where the system controller - # license is valid, and the subcloud license install fails - - # Order of get_license calls: - # first license query is to system controller - # second license query is to subcloud (should be missing) - self.fake_sysinv_client.get_license.side_effect = \ - [LICENSE_VALID_RESPONSE, - MISSING_LICENSE_RESPONSE] - - # Simulate a license install failure on the subcloud - self.fake_sysinv_client.install_license.return_value = \ - MISSING_LICENSE_RESPONSE - - self.worker.install_subcloud_license(self.strategy_step) - - # verify the license install was invoked - self.fake_sysinv_client.install_license.assert_called() - - # Verify a install_license failure leads to a state failure - self.assert_step_updated(self.strategy_step.subcloud_id, - consts.STRATEGY_STATE_FAILED) - - def test_upgrade_subcloud_license_install_success(self): - # Test the install subcloud license step where the system controller - # license is valid, and the subcloud installation succeeds - - # Order of get_license calls: - # first license query is to system controller - # second license query is to subcloud (should be missing) - self.fake_sysinv_client.get_license.side_effect = \ - [LICENSE_VALID_RESPONSE, - MISSING_LICENSE_RESPONSE] - - # A license install should return a success - self.fake_sysinv_client.install_license.return_value = \ - LICENSE_VALID_RESPONSE - - self.worker.install_subcloud_license(self.strategy_step) - - # verify the license install was invoked - self.fake_sysinv_client.install_license.assert_called() - - # On success, the next state after installing license is importing load - self.assert_step_updated(self.strategy_step.subcloud_id, - consts.STRATEGY_STATE_IMPORTING_LOAD) - - def test_upgrade_subcloud_license_skip_existing(self): - # Test the install subcloud license step where the system controller - # license is valid, and the subcloud already has the same license - - # Order of get_license calls: - # first license query is to system controller - # second license query is to subcloud - self.fake_sysinv_client.get_license.side_effect = \ - [LICENSE_VALID_RESPONSE, - LICENSE_VALID_RESPONSE] - self.worker.install_subcloud_license(self.strategy_step) - - # A license install should not have been attempted due to the license - # already being up to date - self.fake_sysinv_client.install_license.assert_not_called() - # On success, the next state after installing license is importing load - self.assert_step_updated(self.strategy_step.subcloud_id, - consts.STRATEGY_STATE_IMPORTING_LOAD) - - def test_upgrade_subcloud_license_overrides_mismatched_license(self): - # Test the install subcloud license step where the system controller - # license is valid, and the subcloud has a differnt license which - # should be overridden - - # Order of get_license calls: - # first license query is to system controller - # second license query is to subcloud (should be valid but different) - self.fake_sysinv_client.get_license.side_effect = \ - [LICENSE_VALID_RESPONSE, - ALTERNATE_LICENSE_RESPONSE] - - # A license install should return a success - self.fake_sysinv_client.install_license.return_value = \ - LICENSE_VALID_RESPONSE - - self.worker.install_subcloud_license(self.strategy_step) - - # verify the license install was invoked - self.fake_sysinv_client.install_license.assert_called() - - # On success, the next state after installing license is importing load - self.assert_step_updated(self.strategy_step.subcloud_id, - consts.STRATEGY_STATE_IMPORTING_LOAD) - - def test_upgrade_subcloud_license_skip_when_no_sys_controller_lic(self): - # Test the install subcloud license step is skipped and proceeds - # to the next state when there is no license on system controller - - # Only makes one query to system controller - self.fake_sysinv_client.get_license.side_effect = \ - [MISSING_LICENSE_RESPONSE, ] - # Test the install subcloud license stage - self.worker.install_subcloud_license(self.strategy_step) - - # A license install should proceed to the next state without - # calling a license install - self.fake_sysinv_client.install_license.assert_not_called() - # Skip license install and move to next state - self.assert_step_updated(self.strategy_step.subcloud_id, - consts.STRATEGY_STATE_IMPORTING_LOAD) - - def test_upgrade_subcloud_license_handle_failure(self): - # Test the install subcloud license step where the system controller - # license is valid, and the subcloud license install fails - - # Order of get_license calls: - # first license query is to system controller - # second license query is to subcloud (should be missing) - self.fake_sysinv_client.get_license.side_effect = \ - [LICENSE_VALID_RESPONSE, - MISSING_LICENSE_RESPONSE] - - # Simulate a license install failure on the subcloud - self.fake_sysinv_client.install_license.return_value = \ - MISSING_LICENSE_RESPONSE - - self.worker.install_subcloud_license(self.strategy_step) - - # verify the license install was invoked - self.fake_sysinv_client.install_license.assert_called() - - # Verify a install_license failure leads to a state failure - self.assert_step_updated(self.strategy_step.subcloud_id, - consts.STRATEGY_STATE_FAILED) - - def test_upgrade_subcloud_license_installs(self): - # Test the install subcloud license step where the system controller - # license is valid, and the subcloud installation succeeds - - # Order of get_license calls: - # first license query is to system controller - # second license query is to subcloud (should be missing) - self.fake_sysinv_client.get_license.side_effect = \ - [LICENSE_VALID_RESPONSE, - MISSING_LICENSE_RESPONSE] - - # A license install should return a success - self.fake_sysinv_client.install_license.return_value = \ - LICENSE_VALID_RESPONSE - - self.worker.install_subcloud_license(self.strategy_step) - - # verify the license install was invoked - self.fake_sysinv_client.install_license.assert_called() - - # On success, the next state after installing license is importing load - self.assert_step_updated(self.strategy_step.subcloud_id, - consts.STRATEGY_STATE_IMPORTING_LOAD)