From 1235fa5fc541cbb6a3b3eca327a0d00a5d0b9f14 Mon Sep 17 00:00:00 2001 From: Luis Eduardo Bonatti Date: Thu, 25 Apr 2024 20:13:09 -0300 Subject: [PATCH] Delete deployment This commit adds the function to delete a deployment. There are some validations and actions to be made regarding each state of the deployment. Deploy state completed: Delete the from_release data, the deploy host entity and deploy entity. Deploy state aborted: Delete the to_release data generated, the deploy host entity and the deploy entity. Deploy state start_failed or start_done: Check if deploy host state is one of: [pending, failed] and if all nodes are running from release software. Delete the to_release data generated, the deploy host entity and the deploy entity. If the release is major it will call the deploy-cleanup scripts to delete the /sysroot/upgrade/ostree_repo and /sysroot/upgrade/sysroot. The data generated that was mentioned are the folders [armada, config, deploy, fluxcd, helm, .keyring, puppet, sysinv, nfv/vim] under /opt/platform// which will be deleted in case of a major release for all the mentioned states [aborted, completed, start_done, start_failed]. Also will be deleted /var/lib/postgres/, /var/lib/postgres/upgrade, /opt/etcd/, /var/lib/rabbitmq/ in case of major release deployment. Test Plan: PASS: Software deploy delete for patch release deployment. PASS: Cleanup of staging data. PASS: Failed to attempt deletion with a deployed host N+1 release PASS: CLI command works with success. PASS: Software deploy delete for a GA release deployment. Story: 2010676 Task: 49979 Change-Id: I1789172edc730e6c94fa6bec7f5881c0bdfd7eab Signed-off-by: Luis Eduardo Bonatti --- software-client/software_client/v1/deploy.py | 9 +++ .../software_client/v1/deploy_cmd.py | 1 + .../software_client/v1/deploy_shell.py | 12 +++ .../software/api/controllers/v1/deploy.py | 9 +++ software/software/constants.py | 19 +++++ software/software/software_controller.py | 73 ++++++++++++++++++- software/software/software_functions.py | 39 ++++++++++ 7 files changed, 161 insertions(+), 1 deletion(-) diff --git a/software-client/software_client/v1/deploy.py b/software-client/software_client/v1/deploy.py index 9d490476..a90fb9ba 100644 --- a/software-client/software_client/v1/deploy.py +++ b/software-client/software_client/v1/deploy.py @@ -82,6 +82,15 @@ class DeployManager(base.Manager): return self._create(path, body={}) + def delete(self, args): + # Ignore interrupts during this function + signal.signal(signal.SIGINT, signal.SIG_IGN) + + # Issue deploy delete request + path = "/v1/deploy" + + return self._delete(path) + def host_list(self): path = '/v1/deploy_host' return self._list(path, "") diff --git a/software-client/software_client/v1/deploy_cmd.py b/software-client/software_client/v1/deploy_cmd.py index 20cd482f..7851b73c 100644 --- a/software-client/software_client/v1/deploy_cmd.py +++ b/software-client/software_client/v1/deploy_cmd.py @@ -19,6 +19,7 @@ DEPLOY_COMMAND_MODULES = [ # - host # - activate # - complete +# - delete # non root/sudo users can run: # - host-list # - show diff --git a/software-client/software_client/v1/deploy_shell.py b/software-client/software_client/v1/deploy_shell.py index 295f8cdd..4ef63419 100644 --- a/software-client/software_client/v1/deploy_shell.py +++ b/software-client/software_client/v1/deploy_shell.py @@ -152,3 +152,15 @@ def do_complete(cc, args): utils.display_info(resp) return utils.check_rc(resp, data) + + +def do_delete(cc, args): + """Delete the software deployment""" + resp, data = cc.deploy.delete(args) + + if args.debug: + utils.print_result_debug(resp, data) + + utils.display_info(resp) + + return utils.check_rc(resp, data) diff --git a/software/software/api/controllers/v1/deploy.py b/software/software/api/controllers/v1/deploy.py index e000e3b0..c2a48a64 100644 --- a/software/software/api/controllers/v1/deploy.py +++ b/software/software/api/controllers/v1/deploy.py @@ -23,6 +23,7 @@ class DeployController(RestController): 'precheck': ['POST'], 'start': ['POST'], 'complete': ['POST'], + 'delete': ['DELETE'], 'software_upgrade': ['GET'], } @@ -42,6 +43,14 @@ class DeployController(RestController): sc.software_sync() return result + @expose(method='DELETE', template='json') + def delete(self): + reload_release_data() + + result = sc.software_deploy_delete_api() + sc.software_sync() + return result + @expose(method='POST', template='json') def precheck(self, release, force=None, region_name=None): _force = force is not None diff --git a/software/software/constants.py b/software/software/constants.py index 41f64086..95d25b19 100644 --- a/software/software/constants.py +++ b/software/software/constants.py @@ -36,6 +36,7 @@ RC_UNHEALTHY = 3 DEPLOY_PRECHECK_SCRIPT = "deploy-precheck" UPGRADE_UTILS_SCRIPT = "upgrade_utils.py" DEPLOY_START_SCRIPT = "software-deploy-start" +DEPLOY_CLEANUP_SCRIPT = "deploy-cleanup" SEMANTICS_DIR = "%s/semantics" % SOFTWARE_STORAGE_DIR @@ -59,6 +60,24 @@ DEBIAN_RELEASE = "bullseye" STARLINGX_RELEASE = SW_VERSION PATCH_SCRIPTS_STAGING_DIR = "/var/www/pages/updates/software-scripts" SYSROOT_OSTREE = "/sysroot/ostree/repo" +STAGING_DIR = "/sysroot/upgrade" +ROOT_DIR = "%s/sysroot" % STAGING_DIR +POSTGRES_PATH = "/var/lib/postgresql" +PLATFORM_PATH = "/opt/platform" +RABBIT_PATH = '/var/lib/rabbitmq' +ETCD_PATH = "/opt/etcd" +ARMADA = "armada" +CONFIG = "config" +DEPLOY = "deploy" +FLUXCD = "fluxcd" +HELM = "helm" +KEYRING = ".keyring" +PUPPET = "puppet" +SYSINV = "sysinv" +UPGRADE = "upgrade" +VIM = "nfv/vim" + +DEPLOY_CLEANUP_FOLDERS_NAME = [ARMADA, CONFIG, DEPLOY, FLUXCD, HELM, KEYRING, PUPPET, SYSINV, VIM] LOOPBACK_INTERFACE_NAME = "lo" diff --git a/software/software/software_controller.py b/software/software/software_controller.py index 6cf99ee8..8e124e2b 100644 --- a/software/software/software_controller.py +++ b/software/software/software_controller.py @@ -27,7 +27,6 @@ from wsgiref import simple_server from fm_api import fm_api from fm_api import constants as fm_constants - from oslo_config import cfg as oslo_cfg import software.apt_utils as apt_utils @@ -75,6 +74,8 @@ from software.software_functions import repo_root_dir from software.software_functions import is_deploy_state_in_sync from software.software_functions import is_deployment_in_progress from software.software_functions import get_release_from_patch +from software.software_functions import clean_up_deployment_data +from software.software_functions import run_deploy_clean_up_script from software.release_state import ReleaseState from software.deploy_host_state import DeployHostState from software.deploy_state import DeployState @@ -2661,6 +2662,76 @@ class PatchController(PatchService): return dict(info=msg_info, warning=msg_warning, error=msg_error) + @require_deploy_state([DEPLOY_STATES.ABORT_DONE, DEPLOY_STATES.COMPLETED, DEPLOY_STATES.START_DONE, + DEPLOY_STATES.START_FAILED], + "Deploy must be in the following states to be able to delete: %s, %s, %s, %s" % ( + DEPLOY_STATES.ABORT_DONE.value, DEPLOY_STATES.COMPLETED.value, + DEPLOY_STATES.START_DONE.value, DEPLOY_STATES.START_FAILED.value)) + def software_deploy_delete_api(self) -> dict: + """ + Delete deployment and the data generated during the deploy. + + :return: dict of info, warning and error messages + """ + msg_info = "" + msg_warning = "" + msg_error = "" + deploy = self.db_api_instance.get_current_deploy() + to_release = deploy.get("to_release") + from_release = deploy.get("from_release") + to_release_deployment = constants.RELEASE_GA_NAME % to_release + from_release_deployment = constants.RELEASE_GA_NAME % from_release + deploy_state_instance = DeployState.get_instance() + is_major_release = False + + if DEPLOY_STATES.COMPLETED == deploy_state_instance.get_deploy_state(): + major_release = utils.get_major_release_version(from_release) + # Try except in case there is no deploy in the class i.e. after unlock in RR deployment. + try: + is_major_release = ReleaseState().is_major_release_deployment() + except AttributeError: + release = self.release_collection.get_release_by_id(from_release_deployment) + is_major_release = ReleaseState(release_ids=[release.id]).is_major_release_deployment() + + elif DEPLOY_STATES.ABORT_DONE == deploy_state_instance.get_deploy_state(): + major_release = utils.get_major_release_version(to_release) + try: + is_major_release = ReleaseState().is_major_release_deployment() + except AttributeError: + release = self.release_collection.get_release_by_id(to_release_deployment) + is_major_release = ReleaseState(release_ids=[release.id]).is_major_release_deployment() + + elif deploy_state_instance.get_deploy_state() in [DEPLOY_STATES.START_DONE, DEPLOY_STATES.START_FAILED]: + hosts_states = [] + for host in self.db_api_instance.get_deploy_host(): + hosts_states.append(host.get("state")) + if (DEPLOY_HOST_STATES.DEPLOYED.value in hosts_states or + DEPLOY_HOST_STATES.DEPLOYING.value in hosts_states): + raise SoftwareServiceError(f"There are hosts already {DEPLOY_HOST_STATES.DEPLOYED.value} " + f"or in {DEPLOY_HOST_STATES.DEPLOYING.value} process") + + major_release = utils.get_major_release_version(to_release) + try: + is_major_release = ReleaseState().is_major_release_deployment() + except AttributeError: + release = self.release_collection.get_release_by_id(to_release_deployment) + is_major_release = ReleaseState(release_ids=[release.id]).is_major_release_deployment() + + if is_major_release: + try: + run_deploy_clean_up_script(to_release) + except subprocess.CalledProcessError as e: + msg_error = "Failed to delete deploy" + LOG.error("%s: %s" % (msg_error, e)) + raise SoftwareServiceError(msg_error) + + if is_major_release: + clean_up_deployment_data(major_release) + msg_info += "Deploy deleted with success" + self.db_api_instance.delete_deploy_host_all() + self.db_api_instance.delete_deploy() + return dict(info=msg_info, warning=msg_warning, error=msg_error) + def _deploy_complete(self): is_all_hosts_in_deployed_state = all(host_state.get("state") == DEPLOY_HOST_STATES.DEPLOYED.value for host_state in self.db_api_instance.get_deploy_host()) diff --git a/software/software/software_functions.py b/software/software/software_functions.py index b88176c4..b02144f9 100644 --- a/software/software/software_functions.py +++ b/software/software/software_functions.py @@ -1425,3 +1425,42 @@ def mount_remote_directory(remote_dir, local_dir): subprocess.check_call(["/bin/umount", local_dir]) except subprocess.CalledProcessError as e: LOG.error("Error unmounting %s: %s" % (local_dir, str(e))) + + +def clean_up_deployment_data(major_release): + """ + Clean up all data generated during deployment. + + :param major_release: Major release to be deleted. + """ + # Delete the data inside /opt/platform// + for folder in constants.DEPLOY_CLEANUP_FOLDERS_NAME: + path = os.path.join(constants.PLATFORM_PATH, folder, major_release, "") + shutil.rmtree(path, ignore_errors=True) + # TODO(lbonatti): These folders should be revisited on software deploy abort/rollback + # to check additional folders that might be needed to delete. + upgrade_folders = [ + os.path.join(constants.POSTGRES_PATH, constants.UPGRADE), + os.path.join(constants.POSTGRES_PATH, major_release), + os.path.join(constants.RABBIT_PATH, major_release), + os.path.join(constants.ETCD_PATH, major_release), + ] + for folder in upgrade_folders: + shutil.rmtree(folder, ignore_errors=True) + + +def run_deploy_clean_up_script(release): + """ + Runs the deploy-cleanup script for the given release. + + :param release: Release to be cleaned. + """ + cmd_path = utils.get_software_deploy_script(release, constants.DEPLOY_CLEANUP_SCRIPT) + if (os.path.exists(f"{constants.STAGING_DIR}/{constants.OSTREE_REPO}") and + os.path.exists(constants.ROOT_DIR)): + try: + subprocess.check_output([cmd_path, f"{constants.STAGING_DIR}/{constants.OSTREE_REPO}", + constants.ROOT_DIR, "all"], stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + LOG.exception("Error running deploy-cleanup script: %s" % str(e)) + raise