From e63a5d4794f8a9e7024890c46da71c65fd460143 Mon Sep 17 00:00:00 2001 From: Bin Qian Date: Fri, 30 May 2025 16:53:06 +0000 Subject: [PATCH] deploy delete action This commit add support of new deploy delete plug. This commit also add new DeployPluginRunner class in order to unify execution of deploy actions, with API credential, CLI environment, and deploy context (options from APIs). Story: 2011357 Task: 52340 Test Plan: passed: execute delete action w/ deploy complete, deploy delete passed: execute delete action w/ deploy start, deploy delete. passed: execute delete action w/ deploy start, host, abort, rollback delete passed: complete deploy for upgrade. Signed-off-by: Bin Qian Change-Id: I52aeb3669a4fc61a0941553c1f40c52acc87e868 --- software/setup.cfg | 1 + software/software/plugin.py | 129 +++++++++++++++++++ software/software/software_controller.py | 34 ++++- software/software/utilities/deploy_delete.py | 70 ++++++++++ software/software/utilities/utils.py | 1 + 5 files changed, 228 insertions(+), 7 deletions(-) create mode 100644 software/software/plugin.py create mode 100644 software/software/utilities/deploy_delete.py diff --git a/software/setup.cfg b/software/setup.cfg index 1730668c..a7fd6ccd 100644 --- a/software/setup.cfg +++ b/software/setup.cfg @@ -40,6 +40,7 @@ console_scripts = software-deploy-activate = software.utilities.activate:activate software-deploy-activate-rollback = software.utilities.activate_rollback:activate_rollback software-deploy-set-failed = software.utilities.deploy_set_failed:deploy_set_failed + software-deploy-delete = software.utilities.deploy_delete:deploy_delete [wheel] diff --git a/software/software/plugin.py b/software/software/plugin.py new file mode 100644 index 00000000..8a10aa2b --- /dev/null +++ b/software/software/plugin.py @@ -0,0 +1,129 @@ +""" +Copyright (c) 2025 Wind River Systems, Inc. + +SPDX-License-Identifier: Apache-2.0 + +""" +import logging +from packaging.version import Version +import re +import os +import tempfile +import shutil +import subprocess + +from software import constants +from software import utils +from software.utilities.utils import SOFTWARE_LOG_FILE + + +LOG = logging.getLogger('main_logger') +USM_PLUGIN_PATH = "/usr/local/share/upgrade.d" + + +# run deploy plugin such as deploy delete, with pre-acquired +# auth token, deploy context +# the plugin runner will automatically determine the higher sw-version from +# the deploy entity, and run the higher sw-version (to-release on deploy) +# plugin, unless plugin_path is specified. +# in the case that the higher sw-version plugin is not on the file system, +# the runner will pull the feed repo and run it. +# TODO(bqian) at this point the patching behaviour of delete action is not +# defined. When determined, the ostree pull command should include proper +# commit-id in order to pull the right release. +class DeployPluginRunner(object): + def __init__(self, deploy, plugin_path=None): + self._deploy = deploy + self._temp_plugin_path = None + self._bin_path = plugin_path + if plugin_path is None: + sw_version = DeployPluginRunner.get_higher_version(deploy) + self._source_repo = f"/var/www/pages/feed/rel-{sw_version}/ostree_repo" + if constants.SW_VERSION != sw_version: + # create temp directory to pull and run usm-plugin from N+1 release + self._temp_plugin_path = os.path.join(tempfile.mkdtemp(prefix="usm-plugin")) + self._bin_path = os.path.join(self._temp_plugin_path, "upgrade.d") + else: + self._bin_path = USM_PLUGIN_PATH + + self._env = None + + def __del__(self): + if self._temp_plugin_path: + shutil.rmtree(self._temp_plugin_path, ignore_errors=True) + if os.path.isdir(self._temp_plugin_path): + LOG.warning(f"Temporary directory {self._temp_plugin_path} could not be deleted") + + @staticmethod + def get_higher_version(deploy): + from_release = deploy.get("from_release") + to_release = deploy.get("to_release") + higher_release = from_release + if Version(to_release) > Version(from_release): + higher_release = to_release + + return utils.get_major_release_version(higher_release) + + @property + def plugin_path(self): + return self._bin_path + + def set_auth_token(self): + token, endpoint = utils.get_endpoints_token() + self._env["ANSIBLE_LOG_PATH"] = SOFTWARE_LOG_FILE + self._env["OS_AUTH_TOKEN"] = token + self._env["SYSTEM_URL"] = re.sub('/v[1,9]$', '', endpoint) # remove ending /v1 + + def set_deploy_options(self): + options = self._deploy.get("options", None) + if not options: + options = {} + for k, v in options.items(): + self._env[k] = v + + def set_execution_context(self, context): + self._env["from_release"] = self._deploy.get("from_release") + self._env["to_release"] = self._deploy.get("to_release") + + for k, v in context: + if k in self._env: + LOG.warning(f"context {k} overwrites deploy option value: {self._env[k]}") + self._env[k] = v + + def execute(self, cmd, context=None): + if not context: + context = {} + + if self._temp_plugin_path: + checkout_cmd = f"ostree --repo={self._source_repo} checkout " + \ + f"--subpath=/usr/local/share/upgrade.d {constants.OSTREE_REF} " + \ + f"{self._bin_path}" + try: + res = subprocess.run(checkout_cmd, check=True, shell=True, stderr=subprocess.STDOUT) + LOG.info(f"Checkout deploy plugins to {self._bin_path} completed successfully") + LOG.info(f"{res.stdout}") + except subprocess.CalledProcessError as e: + LOG.error(f"Failed to checkout deploy plugins {checkout_cmd}. Error output:") + LOG.error(f"{e.output}") + raise + except subprocess.SubprocessError: + LOG.error(f"Checkout deploy plugins has timeout. {checkout_cmd}") + raise + + self._env = os.environ.copy() + # option comes from API, is the least priority, can be overwritten + # by any system internal context + self.set_deploy_options() + + self.set_auth_token() + self.set_execution_context(context) + + plugin_cmd = ' '.join(["source", "/etc/platform/openrc;", cmd]) + + try: + LOG.info("starting subprocess %s" % plugin_cmd) + subprocess.Popen(plugin_cmd, start_new_session=True, shell=True, env=self._env) + LOG.info("subprocess started") + except subprocess.CalledProcessError as e: + LOG.error("Failed to start command: %s. Error %s" % (plugin_cmd, e)) + raise diff --git a/software/software/software_controller.py b/software/software/software_controller.py index f5ce8c97..74bdafeb 100644 --- a/software/software/software_controller.py +++ b/software/software/software_controller.py @@ -64,6 +64,7 @@ from software.exceptions import HostAgentUnreachable from software.exceptions import HostIpNotFound from software.exceptions import MaxReleaseExceeded from software.exceptions import ServiceParameterNotFound +from software.plugin import DeployPluginRunner from software.release_data import reload_release_data from software.release_data import get_SWReleaseCollection from software.software_functions import collect_current_load_for_hosts @@ -1404,7 +1405,6 @@ class PatchController(PatchService): return dict(info=msg_info, warning=msg_warning, error=msg_error) if os.path.isfile(INSTALL_LOCAL_FLAG) and delete: - # Remove install local flag if enabled if os.path.isfile(INSTALL_LOCAL_FLAG): try: os.remove(INSTALL_LOCAL_FLAG) @@ -3495,6 +3495,16 @@ class PatchController(PatchService): tree = ET.tostring(root) outfile.write(tree) + def execute_delete_actions(self): + deploy = self.db_api_instance.get_current_deploy() + to_release = deploy.get("to_release") + from_release = deploy.get("from_release") + + delete_cmd = f"/usr/bin/software-deploy-delete {from_release} {to_release} --is_major_release" + + runner = DeployPluginRunner(deploy) + runner.execute(delete_cmd) + @require_deploy_state([DEPLOY_STATES.HOST_ROLLBACK_DONE, DEPLOY_STATES.COMPLETED, DEPLOY_STATES.START_DONE, DEPLOY_STATES.START_FAILED], "Deploy must be in the following states to be able to delete: %s, %s, %s, %s" % ( @@ -3546,7 +3556,7 @@ class PatchController(PatchService): if is_major_release and self.hostname != constants.CONTROLLER_0_HOSTNAME: raise SoftwareServiceError("Deploy delete can only be performed on controller-0.") - if DEPLOY_STATES.COMPLETED == deploy_state_instance.get_deploy_state(): + if DEPLOY_STATES.COMPLETED == deploy_state: if is_applying: major_release = utils.get_major_release_version(from_release) # In case of a major release deployment set all the releases related to from_release to unavailable @@ -3563,12 +3573,14 @@ class PatchController(PatchService): removing_release_state = ReleaseState(release_state=states.REMOVING) removing_release_state.available() - elif DEPLOY_STATES.HOST_ROLLBACK_DONE == deploy_state_instance.get_deploy_state(): + elif DEPLOY_STATES.HOST_ROLLBACK_DONE == deploy_state: major_release = utils.get_major_release_version(from_release) release_state = ReleaseState(release_state=states.DEPLOYING) release_state.available() - elif deploy_state_instance.get_deploy_state() in [DEPLOY_STATES.START_DONE, DEPLOY_STATES.START_FAILED]: + elif deploy_state in [DEPLOY_STATES.START_DONE, DEPLOY_STATES.START_FAILED]: + # TODO(bqian), this check is redundant. there should be no host deployed/deploying + # when deploy in START_DONE or START_FAILED states hosts_states = [] for host in self.db_api_instance.get_deploy_host(): hosts_states.append(host.get("state")) @@ -3582,6 +3594,7 @@ class PatchController(PatchService): if is_major_release: try: + # TODO(bqian) Move below function to a delete action run_remove_temporary_data_script(to_release) except subprocess.CalledProcessError as e: msg_error = "Failed to delete deploy" @@ -3602,8 +3615,8 @@ class PatchController(PatchService): LOG.error(msg_error) raise SoftwareServiceError(msg_error) - # Remove install local flag if enabled if os.path.isfile(INSTALL_LOCAL_FLAG): + # Remove install local flag if enabled try: os.remove(INSTALL_LOCAL_FLAG) except Exception: @@ -3613,7 +3626,6 @@ class PatchController(PatchService): LOG.info("Software deployment in local installation mode is stopped") if is_major_release: - if SW_VERSION == major_release: msg_error = ( f"Deploy {major_release} can't be deleted as it is still the" @@ -3621,12 +3633,13 @@ class PatchController(PatchService): LOG.error(msg_error) raise SoftwareServiceError(msg_error) + # TODO(bqian) Move below function to a delete action clean_up_deployment_data(major_release) # Send message to agents cleanup their ostree environment # if the deployment has completed or rolled-back successfully finished_deploy_states = [DEPLOY_STATES.COMPLETED, DEPLOY_STATES.HOST_ROLLBACK_DONE] - if deploy_state_instance.get_deploy_state() in finished_deploy_states: + if deploy_state in finished_deploy_states: cleanup_req = SoftwareMessageDeployDeleteCleanupReq() cleanup_req.major_release = utils.get_major_release_version(to_release) cleanup_req.encode() @@ -3637,12 +3650,19 @@ class PatchController(PatchService): self.manage_software_alarm(fm_constants.FM_ALARM_ID_USM_CLEANUP_DEPLOYMENT_DATA, fm_constants.FM_ALARM_STATE_CLEAR, "%s=%s" % (fm_constants.FM_ENTITY_TYPE_HOST, constants.CONTROLLER_FLOATING_HOSTNAME)) + + # execute deploy delete plugins + # NOTE(bqian) implement for major release deploy delete only as deleting action + # for patching is undefined, i.e, in the case of patch is applied, both from and + # to releases are applied. + self.execute_delete_actions() else: self.delete_all_patch_activate_scripts() msg_info += "Deploy deleted with success" self.db_api_instance.delete_deploy_host_all() self.db_api_instance.delete_deploy() + LOG.info("Deploy is deleted") return dict(info=msg_info, warning=msg_warning, error=msg_error) diff --git a/software/software/utilities/deploy_delete.py b/software/software/utilities/deploy_delete.py new file mode 100644 index 00000000..46e8cb8e --- /dev/null +++ b/software/software/utilities/deploy_delete.py @@ -0,0 +1,70 @@ +# +# Copyright (c) 2025 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# This script neeeds to be running on both N-1 runtime from +# a temporary directory and N runtime from /usr/ directory + +import argparse +import logging + +from software.utilities import utils + + +LOG = logging.getLogger('main_logger') + + +def do_deploy_delete(from_release, to_release, plugin_path, is_major_release): + # This is a "best effort" operation. Failing steps will be logged + # and move on. + + if not is_major_release: + return + + res = True + try: + if plugin_path: + utils.execute_migration_scripts(from_release, to_release, + utils.ACTION_DELETE, + migration_script_dir=plugin_path) + else: + utils.execute_migration_scripts(from_release, to_release, + utils.ACTION_DELETE) + except Exception: + res = False + finally: + if res: + LOG.info("Deploy delete completed successfully") + else: + LOG.info("Errors occored in deploy delete.") + + +def deploy_delete(): + # this is the entry point to deploy delete plugin + utils.configure_logging() + parser = argparse.ArgumentParser(add_help=False) + + parser.add_argument("from_release", + default=False, + help="From release") + + parser.add_argument("to_release", + default=False, + help="To release") + + # Optional flag --is_major_release + parser.add_argument("--is_major_release", + action="store_true", + help="Specify if this is a major release") + + # Optional flag --plugin-path + parser.add_argument("--plugin_path", + dest="plugin_path", + default=None, + help="Specify the path of action plugins") + + args = parser.parse_args() + + do_deploy_delete(args.from_release, args.to_release, args.plugin_path, args.is_major_release) diff --git a/software/software/utilities/utils.py b/software/software/utilities/utils.py index 63e4badc..8d99294c 100644 --- a/software/software/utilities/utils.py +++ b/software/software/utilities/utils.py @@ -39,6 +39,7 @@ ACTION_START = "start" ACTION_MIGRATE = "migrate" ACTION_ACTIVATE = "activate" ACTION_ACTIVATE_ROLLBACK = "activate-rollback" +ACTION_DELETE = "delete" def configure_logging():