diff --git a/software/setup.cfg b/software/setup.cfg index 1730668c..a7fd6ccd 100644 --- a/software/setup.cfg +++ b/software/setup.cfg @@ -40,6 +40,7 @@ console_scripts = software-deploy-activate = software.utilities.activate:activate software-deploy-activate-rollback = software.utilities.activate_rollback:activate_rollback software-deploy-set-failed = software.utilities.deploy_set_failed:deploy_set_failed + software-deploy-delete = software.utilities.deploy_delete:deploy_delete [wheel] diff --git a/software/software/plugin.py b/software/software/plugin.py new file mode 100644 index 00000000..8a10aa2b --- /dev/null +++ b/software/software/plugin.py @@ -0,0 +1,129 @@ +""" +Copyright (c) 2025 Wind River Systems, Inc. + +SPDX-License-Identifier: Apache-2.0 + +""" +import logging +from packaging.version import Version +import re +import os +import tempfile +import shutil +import subprocess + +from software import constants +from software import utils +from software.utilities.utils import SOFTWARE_LOG_FILE + + +LOG = logging.getLogger('main_logger') +USM_PLUGIN_PATH = "/usr/local/share/upgrade.d" + + +# run deploy plugin such as deploy delete, with pre-acquired +# auth token, deploy context +# the plugin runner will automatically determine the higher sw-version from +# the deploy entity, and run the higher sw-version (to-release on deploy) +# plugin, unless plugin_path is specified. +# in the case that the higher sw-version plugin is not on the file system, +# the runner will pull the feed repo and run it. +# TODO(bqian) at this point the patching behaviour of delete action is not +# defined. When determined, the ostree pull command should include proper +# commit-id in order to pull the right release. +class DeployPluginRunner(object): + def __init__(self, deploy, plugin_path=None): + self._deploy = deploy + self._temp_plugin_path = None + self._bin_path = plugin_path + if plugin_path is None: + sw_version = DeployPluginRunner.get_higher_version(deploy) + self._source_repo = f"/var/www/pages/feed/rel-{sw_version}/ostree_repo" + if constants.SW_VERSION != sw_version: + # create temp directory to pull and run usm-plugin from N+1 release + self._temp_plugin_path = os.path.join(tempfile.mkdtemp(prefix="usm-plugin")) + self._bin_path = os.path.join(self._temp_plugin_path, "upgrade.d") + else: + self._bin_path = USM_PLUGIN_PATH + + self._env = None + + def __del__(self): + if self._temp_plugin_path: + shutil.rmtree(self._temp_plugin_path, ignore_errors=True) + if os.path.isdir(self._temp_plugin_path): + LOG.warning(f"Temporary directory {self._temp_plugin_path} could not be deleted") + + @staticmethod + def get_higher_version(deploy): + from_release = deploy.get("from_release") + to_release = deploy.get("to_release") + higher_release = from_release + if Version(to_release) > Version(from_release): + higher_release = to_release + + return utils.get_major_release_version(higher_release) + + @property + def plugin_path(self): + return self._bin_path + + def set_auth_token(self): + token, endpoint = utils.get_endpoints_token() + self._env["ANSIBLE_LOG_PATH"] = SOFTWARE_LOG_FILE + self._env["OS_AUTH_TOKEN"] = token + self._env["SYSTEM_URL"] = re.sub('/v[1,9]$', '', endpoint) # remove ending /v1 + + def set_deploy_options(self): + options = self._deploy.get("options", None) + if not options: + options = {} + for k, v in options.items(): + self._env[k] = v + + def set_execution_context(self, context): + self._env["from_release"] = self._deploy.get("from_release") + self._env["to_release"] = self._deploy.get("to_release") + + for k, v in context: + if k in self._env: + LOG.warning(f"context {k} overwrites deploy option value: {self._env[k]}") + self._env[k] = v + + def execute(self, cmd, context=None): + if not context: + context = {} + + if self._temp_plugin_path: + checkout_cmd = f"ostree --repo={self._source_repo} checkout " + \ + f"--subpath=/usr/local/share/upgrade.d {constants.OSTREE_REF} " + \ + f"{self._bin_path}" + try: + res = subprocess.run(checkout_cmd, check=True, shell=True, stderr=subprocess.STDOUT) + LOG.info(f"Checkout deploy plugins to {self._bin_path} completed successfully") + LOG.info(f"{res.stdout}") + except subprocess.CalledProcessError as e: + LOG.error(f"Failed to checkout deploy plugins {checkout_cmd}. Error output:") + LOG.error(f"{e.output}") + raise + except subprocess.SubprocessError: + LOG.error(f"Checkout deploy plugins has timeout. {checkout_cmd}") + raise + + self._env = os.environ.copy() + # option comes from API, is the least priority, can be overwritten + # by any system internal context + self.set_deploy_options() + + self.set_auth_token() + self.set_execution_context(context) + + plugin_cmd = ' '.join(["source", "/etc/platform/openrc;", cmd]) + + try: + LOG.info("starting subprocess %s" % plugin_cmd) + subprocess.Popen(plugin_cmd, start_new_session=True, shell=True, env=self._env) + LOG.info("subprocess started") + except subprocess.CalledProcessError as e: + LOG.error("Failed to start command: %s. Error %s" % (plugin_cmd, e)) + raise diff --git a/software/software/software_controller.py b/software/software/software_controller.py index f5ce8c97..74bdafeb 100644 --- a/software/software/software_controller.py +++ b/software/software/software_controller.py @@ -64,6 +64,7 @@ from software.exceptions import HostAgentUnreachable from software.exceptions import HostIpNotFound from software.exceptions import MaxReleaseExceeded from software.exceptions import ServiceParameterNotFound +from software.plugin import DeployPluginRunner from software.release_data import reload_release_data from software.release_data import get_SWReleaseCollection from software.software_functions import collect_current_load_for_hosts @@ -1404,7 +1405,6 @@ class PatchController(PatchService): return dict(info=msg_info, warning=msg_warning, error=msg_error) if os.path.isfile(INSTALL_LOCAL_FLAG) and delete: - # Remove install local flag if enabled if os.path.isfile(INSTALL_LOCAL_FLAG): try: os.remove(INSTALL_LOCAL_FLAG) @@ -3495,6 +3495,16 @@ class PatchController(PatchService): tree = ET.tostring(root) outfile.write(tree) + def execute_delete_actions(self): + deploy = self.db_api_instance.get_current_deploy() + to_release = deploy.get("to_release") + from_release = deploy.get("from_release") + + delete_cmd = f"/usr/bin/software-deploy-delete {from_release} {to_release} --is_major_release" + + runner = DeployPluginRunner(deploy) + runner.execute(delete_cmd) + @require_deploy_state([DEPLOY_STATES.HOST_ROLLBACK_DONE, DEPLOY_STATES.COMPLETED, DEPLOY_STATES.START_DONE, DEPLOY_STATES.START_FAILED], "Deploy must be in the following states to be able to delete: %s, %s, %s, %s" % ( @@ -3546,7 +3556,7 @@ class PatchController(PatchService): if is_major_release and self.hostname != constants.CONTROLLER_0_HOSTNAME: raise SoftwareServiceError("Deploy delete can only be performed on controller-0.") - if DEPLOY_STATES.COMPLETED == deploy_state_instance.get_deploy_state(): + if DEPLOY_STATES.COMPLETED == deploy_state: if is_applying: major_release = utils.get_major_release_version(from_release) # In case of a major release deployment set all the releases related to from_release to unavailable @@ -3563,12 +3573,14 @@ class PatchController(PatchService): removing_release_state = ReleaseState(release_state=states.REMOVING) removing_release_state.available() - elif DEPLOY_STATES.HOST_ROLLBACK_DONE == deploy_state_instance.get_deploy_state(): + elif DEPLOY_STATES.HOST_ROLLBACK_DONE == deploy_state: major_release = utils.get_major_release_version(from_release) release_state = ReleaseState(release_state=states.DEPLOYING) release_state.available() - elif deploy_state_instance.get_deploy_state() in [DEPLOY_STATES.START_DONE, DEPLOY_STATES.START_FAILED]: + elif deploy_state in [DEPLOY_STATES.START_DONE, DEPLOY_STATES.START_FAILED]: + # TODO(bqian), this check is redundant. there should be no host deployed/deploying + # when deploy in START_DONE or START_FAILED states hosts_states = [] for host in self.db_api_instance.get_deploy_host(): hosts_states.append(host.get("state")) @@ -3582,6 +3594,7 @@ class PatchController(PatchService): if is_major_release: try: + # TODO(bqian) Move below function to a delete action run_remove_temporary_data_script(to_release) except subprocess.CalledProcessError as e: msg_error = "Failed to delete deploy" @@ -3602,8 +3615,8 @@ class PatchController(PatchService): LOG.error(msg_error) raise SoftwareServiceError(msg_error) - # Remove install local flag if enabled if os.path.isfile(INSTALL_LOCAL_FLAG): + # Remove install local flag if enabled try: os.remove(INSTALL_LOCAL_FLAG) except Exception: @@ -3613,7 +3626,6 @@ class PatchController(PatchService): LOG.info("Software deployment in local installation mode is stopped") if is_major_release: - if SW_VERSION == major_release: msg_error = ( f"Deploy {major_release} can't be deleted as it is still the" @@ -3621,12 +3633,13 @@ class PatchController(PatchService): LOG.error(msg_error) raise SoftwareServiceError(msg_error) + # TODO(bqian) Move below function to a delete action clean_up_deployment_data(major_release) # Send message to agents cleanup their ostree environment # if the deployment has completed or rolled-back successfully finished_deploy_states = [DEPLOY_STATES.COMPLETED, DEPLOY_STATES.HOST_ROLLBACK_DONE] - if deploy_state_instance.get_deploy_state() in finished_deploy_states: + if deploy_state in finished_deploy_states: cleanup_req = SoftwareMessageDeployDeleteCleanupReq() cleanup_req.major_release = utils.get_major_release_version(to_release) cleanup_req.encode() @@ -3637,12 +3650,19 @@ class PatchController(PatchService): self.manage_software_alarm(fm_constants.FM_ALARM_ID_USM_CLEANUP_DEPLOYMENT_DATA, fm_constants.FM_ALARM_STATE_CLEAR, "%s=%s" % (fm_constants.FM_ENTITY_TYPE_HOST, constants.CONTROLLER_FLOATING_HOSTNAME)) + + # execute deploy delete plugins + # NOTE(bqian) implement for major release deploy delete only as deleting action + # for patching is undefined, i.e, in the case of patch is applied, both from and + # to releases are applied. + self.execute_delete_actions() else: self.delete_all_patch_activate_scripts() msg_info += "Deploy deleted with success" self.db_api_instance.delete_deploy_host_all() self.db_api_instance.delete_deploy() + LOG.info("Deploy is deleted") return dict(info=msg_info, warning=msg_warning, error=msg_error) diff --git a/software/software/utilities/deploy_delete.py b/software/software/utilities/deploy_delete.py new file mode 100644 index 00000000..46e8cb8e --- /dev/null +++ b/software/software/utilities/deploy_delete.py @@ -0,0 +1,70 @@ +# +# Copyright (c) 2025 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# This script neeeds to be running on both N-1 runtime from +# a temporary directory and N runtime from /usr/ directory + +import argparse +import logging + +from software.utilities import utils + + +LOG = logging.getLogger('main_logger') + + +def do_deploy_delete(from_release, to_release, plugin_path, is_major_release): + # This is a "best effort" operation. Failing steps will be logged + # and move on. + + if not is_major_release: + return + + res = True + try: + if plugin_path: + utils.execute_migration_scripts(from_release, to_release, + utils.ACTION_DELETE, + migration_script_dir=plugin_path) + else: + utils.execute_migration_scripts(from_release, to_release, + utils.ACTION_DELETE) + except Exception: + res = False + finally: + if res: + LOG.info("Deploy delete completed successfully") + else: + LOG.info("Errors occored in deploy delete.") + + +def deploy_delete(): + # this is the entry point to deploy delete plugin + utils.configure_logging() + parser = argparse.ArgumentParser(add_help=False) + + parser.add_argument("from_release", + default=False, + help="From release") + + parser.add_argument("to_release", + default=False, + help="To release") + + # Optional flag --is_major_release + parser.add_argument("--is_major_release", + action="store_true", + help="Specify if this is a major release") + + # Optional flag --plugin-path + parser.add_argument("--plugin_path", + dest="plugin_path", + default=None, + help="Specify the path of action plugins") + + args = parser.parse_args() + + do_deploy_delete(args.from_release, args.to_release, args.plugin_path, args.is_major_release) diff --git a/software/software/utilities/utils.py b/software/software/utilities/utils.py index 63e4badc..8d99294c 100644 --- a/software/software/utilities/utils.py +++ b/software/software/utilities/utils.py @@ -39,6 +39,7 @@ ACTION_START = "start" ACTION_MIGRATE = "migrate" ACTION_ACTIVATE = "activate" ACTION_ACTIVATE_ROLLBACK = "activate-rollback" +ACTION_DELETE = "delete" def configure_logging():