diff --git a/software/pylint.rc b/software/pylint.rc index e6516404..274d001e 100644 --- a/software/pylint.rc +++ b/software/pylint.rc @@ -473,7 +473,7 @@ disable= C0103,C0114,C0115,C0116,C0201,C0202,C0206,C0209,C2801, R1715,R1722,R1724,R1725,R1732,R1735, W0107,W0231,W0602,W0603,W0621,W0622, W0703,W0707,W0719,W1201,W1514,W3101, - E0605 + E0605,W1203 # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option diff --git a/software/software/tests/test_software_migrate_utils.py b/software/software/tests/test_software_migrate_utils.py new file mode 100644 index 00000000..98c88b8e --- /dev/null +++ b/software/software/tests/test_software_migrate_utils.py @@ -0,0 +1,48 @@ +# +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (c) 2025 Wind River Systems, Inc. +# +import unittest + +from software.utilities.utils import sort_migration_scripts + + +class TestSoftwareMigration(unittest.TestCase): + + def setUp(self): + pass + + def tearDown(self): + pass + + def test_sort_migration_scripts(self): + scripts = ['01-script1.sh', + '02-script2.py', + '09-script9.sh', + '06-script77.py', + '123-script-9.py'] + + migrate_exp = ['01-script1.sh', + '02-script2.py', + '06-script77.py', + '09-script9.sh', + '123-script-9.py'] + migrate = sort_migration_scripts(scripts, 'migrate') + assert migrate_exp == migrate + + activate_exp = ['01-script1.sh', + '02-script2.py', + '06-script77.py', + '09-script9.sh', + '123-script-9.py'] + activate = sort_migration_scripts(scripts, 'activate') + assert activate_exp == activate + + rollback_exp = ['123-script-9.py', + '09-script9.sh', + '06-script77.py', + '02-script2.py', + '01-script1.sh'] + rollback = sort_migration_scripts(scripts, 'activate-rollback') + assert rollback_exp == rollback diff --git a/software/software/utilities/utils.py b/software/software/utilities/utils.py index bb48c7b8..459bdfab 100644 --- a/software/software/utilities/utils.py +++ b/software/software/utilities/utils.py @@ -45,96 +45,113 @@ def configure_logging(): logging.basicConfig(filename=SOFTWARE_LOG_FILE, format=log_format, level=logging.INFO, datefmt=log_datefmt) -def execute_migration_scripts(from_release, to_release, action, port=None, - migration_script_dir="/usr/local/share/upgrade.d"): - """Execute deployment scripts with an action: - start: Prepare for upgrade on release N side. Called during - "system upgrade-start". - migrate: Perform data migration on release N+1 side. Called while - system data migration is taking place. - activate: Activates the deployment. Called during "software deploy activate". - activate-rollback: Rolls back the activate deployment. Called during - "software deploy activate". - """ - - LOG.info("Executing deployment scripts from: %s with from_release: %s, to_release: %s, " - "action: %s" % (migration_script_dir, from_release, to_release, action)) - - ignore_errors = os.environ.get("IGNORE_ERRORS", 'False').upper() == 'TRUE' - +def get_migration_scripts(migration_script_dir): if not os.path.isdir(migration_script_dir): msg = "Folder %s does not exist" % migration_script_dir LOG.exception(msg) raise Exception(msg) - # Get a sorted list of all the migration scripts - # Exclude any files that can not be executed, including .pyc and .pyo files files = [f for f in os.listdir(migration_script_dir) if os.path.isfile(os.path.join(migration_script_dir, f)) and os.access(os.path.join(migration_script_dir, f), os.X_OK)] + return files + + +def sort_migration_scripts(scripts, action): + reversed_actions = ['activate-rollback'] # From file name, get the number to sort the calling sequence, # abort when the file name format does not follow the pattern # "nnn-*.*", where "nnn" string shall contain only digits, corresponding # to a valid unsigned integer (first sequence of characters before "-") try: - files.sort(key=lambda x: int(x.split("-")[0])) + scripts.sort(key=lambda x: int(x.split("-")[0])) + if action in reversed_actions: + scripts = scripts[::-1] + LOG.info(f"Executing deployment scripts for {action} in reversed order") except Exception: LOG.exception("Deployment script sequence validation failed, invalid " "file name format") raise + return scripts + + +def execute_script(script, from_release, to_release, action, port): MSG_SCRIPT_FAILURE = "Deployment script %s failed with return code %d" \ "\nScript output:\n%s" + result = (0, f'Deployment script {script} completed successfully') + try: + LOG.info("Executing deployment script %s" % script) + cmdline = [script, from_release, to_release, action] + if port is not None: + cmdline.append(port) + + # Let subprocess.run handle non-zero exit codes via check=True + subprocess.run(cmdline, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + check=True) + + except subprocess.CalledProcessError as e: + # Deduplicate output lines using set and create error message + unique_output = "\n".join(e.output.splitlines()) + "\n" + error = MSG_SCRIPT_FAILURE % (script, e.returncode, unique_output) + result = (1, error) + except Exception as e: + # Log exception but continue processing + error = f"Unexpected error executing {script}: {str(e)}" + result = (1, error) + + return result + + +def initialize_deploy_failure_log(): + if not DEPLOY_SCRIPTS_FAILURES_LOG.handlers: + log_format = ('%(asctime)s: %(message)s') + log_datefmt = "%FT%T" + DEPLOY_SCRIPTS_FAILURES_LOG.setLevel(logging.INFO) + log_file_handler = logging.FileHandler(DEPLOY_SCRIPTS_FAILURES_LOG_FILE) + log_file_handler.setFormatter(logging.Formatter( + fmt=log_format, datefmt=log_datefmt)) + DEPLOY_SCRIPTS_FAILURES_LOG.addHandler(log_file_handler) + + +def execute_scripts(scripts, from_release, to_release, action, port, migration_script_dir): # Execute each migration script and collect errors errors = [] - for f in files: + for f in scripts: migration_script = os.path.join(migration_script_dir, f) - try: - LOG.info("Executing deployment script %s" % migration_script) - cmdline = [migration_script, from_release, to_release, action] - if port is not None: - cmdline.append(port) - - # Let subprocess.run handle non-zero exit codes via check=True - subprocess.run(cmdline, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - check=True) - - except subprocess.CalledProcessError as e: - # Deduplicate output lines using set and create error message - unique_output = "\n".join(e.output.splitlines()) + "\n" - msg = MSG_SCRIPT_FAILURE % (migration_script, e.returncode, unique_output) + ret_code, msg = execute_script(migration_script, from_release, to_release, action, port) + if ret_code: errors.append(msg) - except Exception as e: - # Log exception but continue processing - error_msg = f"Unexpected error executing {migration_script}: {str(e)}" - errors.append(error_msg) if errors: - LOG.exception( + initialize_deploy_failure_log() + LOG.Error( "%d deployment scripts failed.\n See details in %s.\n", len(errors), DEPLOY_SCRIPTS_FAILURES_LOG_FILE) - if not DEPLOY_SCRIPTS_FAILURES_LOG.handlers: - log_format = ('%(asctime)s: %(message)s') - log_datefmt = "%FT%T" - DEPLOY_SCRIPTS_FAILURES_LOG.setLevel(logging.INFO) - log_file_handler = logging.FileHandler(DEPLOY_SCRIPTS_FAILURES_LOG_FILE) - log_file_handler.setFormatter(logging.Formatter( - fmt=log_format, datefmt=log_datefmt)) - DEPLOY_SCRIPTS_FAILURES_LOG.addHandler(log_file_handler) - - # Log the errors to the dedicated failure log + # initialize_deploy_failure_log Log the errors to the dedicated failure log DEPLOY_SCRIPTS_FAILURES_LOG.info("%s action partially failed. " % action) DEPLOY_SCRIPTS_FAILURES_LOG.info("\n".join(errors)) + ignore_errors = os.environ.get("IGNORE_ERRORS", 'False').upper() == 'TRUE' # After processing all files, raise any accumulated errors if errors and (not ignore_errors): raise # pylint: disable=misplaced-bare-raise +def execute_migration_scripts(from_release, to_release, action, port=None, + migration_script_dir="/usr/local/share/upgrade.d"): + LOG.info("Executing deployment scripts from: %s with from_release: %s, to_release: %s, " + "action: %s" % (migration_script_dir, from_release, to_release, action)) + scripts = get_migration_scripts(migration_script_dir) + scripts = sort_migration_scripts(scripts, action) + + execute_scripts(scripts, from_release, to_release, action, port, migration_script_dir) + + def get_db_connection(hiera_db_records, database): username = hiera_db_records[database]['username'] password = hiera_db_records[database]['password']