Support remote install for previous release centos-based subcloud

This commit adds support for remote installation of previous
centos-based subclouds.

The subcloud install logic now invokes either the debian-based
gen-bootloader-iso.sh script for current release subcloud installs, or
the gen-bootloader-iso-centos.sh script for the installing the previous
release subcloud.

We also clarify the log output of the gen-bootloader-iso command
shell invocation, to include all stdout/stderr output in the main
dcmanager.log file (via subprocess.run).

Includes a minor refactor of the utils naming in subcloud_install.py
to clarify which utils module is being used.

Test Plan
PASS
- Verify successful subcloud add which includes remote install with
  specified previous release (21.12). Verify that the centos-based
  miniboot bootimage.iso file is generated and installed via rvmc.
- Verify successful subcloud install with the specified current
  release. This includes the proper miniboot invocation for debian.
- Verify that any gen-bootloader-iso install issues/errors are now
  reflected in the dcmanager.log file

Story: 2010611
Task: 47784

Signed-off-by: Kyle MacLeod <kyle.macleod@windriver.com>
Change-Id: I2fa2bc56cc79ca8d17905337a8d81ad5c9a908ac

distributedcloud/dccommon/subcloud_install.py

@@ -29,7 +29,7 @@ from dccommon.drivers.openstack.keystone_v3 import KeystoneClient
 from dccommon.drivers.openstack.sysinv_v1 import SysinvClient
 from dccommon import exceptions
 from dccommon import install_consts
-from dccommon import utils as common_utils
+from dccommon import utils as dccommon_utils
 from dcmanager.common import consts as common_consts
 from dcmanager.common import utils
 
@@ -286,8 +286,10 @@ class SubcloudInstall(object):
     def update_iso(self, override_path, values):
         if not os.path.isdir(self.www_root):
             os.mkdir(self.www_root, 0o755)
-
+        LOG.debug("update_iso: www_root: %s, values: %s, override_path: %s",
+                  self.www_root, str(values), override_path)
         path = None
+        software_version = str(values['software_version'])
         try:
             if parse.urlparse(values['image']).scheme:
                 url = values['image']
@@ -297,7 +299,7 @@ class SubcloudInstall(object):
             filename = os.path.join(override_path, 'bootimage.iso')
 
             if path and path.startswith(consts.LOAD_VAULT_DIR +
-                                        '/' + str(values['software_version'])):
+                                        '/' + software_version):
                 if os.path.exists(path):
                     # Reference known load in vault
                     LOG.info("Setting input_iso to load vault path %s" % path)
@@ -324,7 +326,9 @@ class SubcloudInstall(object):
                 resource=self.name,
                 msg=msg)
 
-        if common_utils.is_debian():
+        is_subcloud_debian = dccommon_utils.is_debian(software_version)
+
+        if is_subcloud_debian:
             update_iso_cmd = [
                 GEN_ISO_COMMAND,
                 "--input", self.input_iso,
@@ -341,6 +345,7 @@ class SubcloudInstall(object):
                 "--id", self.name,
                 "--boot-hostname", self.name,
                 "--timeout", BOOT_MENU_TIMEOUT,
+                "--patches-from-iso",
             ]
         for key in GEN_ISO_OPTIONS:
             if key in values:
@@ -381,7 +386,11 @@ class SubcloudInstall(object):
                 else:
                     update_iso_cmd += [GEN_ISO_OPTIONS[key], str(values[key])]
 
-        if common_utils.is_centos():
+        if is_subcloud_debian:
+            # Get the base URL. ostree_repo is located within this path
+            base_url = os.path.join(self.get_image_base_url(), 'iso',
+                                    software_version)
+        else:
             # create ks-addon.cfg
             addon_cfg = os.path.join(override_path, 'ks-addon.cfg')
             self.create_ks_conf_file(addon_cfg, values)
@@ -390,22 +399,21 @@ class SubcloudInstall(object):
 
             # Get the base URL
             base_url = os.path.join(self.get_image_base_url(), 'iso',
-                                    str(values['software_version']))
+                                    software_version)
-        else:
-            # Get the base URL. ostree_repo is located within this path
-            base_url = os.path.join(self.get_image_base_url(), 'iso',
-                                    str(values['software_version']))
 
         update_iso_cmd += ['--base-url', base_url]
 
         str_cmd = ' '.join(x for x in update_iso_cmd)
         LOG.info("Running update_iso_cmd: %s", str_cmd)
-        try:
+        result = subprocess.run(update_iso_cmd,
-            with open(os.devnull, "w") as fnull:
+                                stdout=subprocess.PIPE,
-                subprocess.check_call(  # pylint: disable=E1102
+                                stderr=subprocess.STDOUT)
-                    update_iso_cmd, stdout=fnull, stderr=fnull)
+        if result.returncode != 0:
-        except subprocess.CalledProcessError:
+            msg = f'Failed to update iso: {str_cmd}'
-            msg = "Failed to update iso %s, " % str(update_iso_cmd)
+            LOG.error("%s returncode: %s, output: %s",
+                      msg,
+                      result.returncode,
+                      result.stdout.decode('utf-8').replace('\n', ', '))
             raise Exception(msg)
 
     def cleanup(self):
@@ -416,7 +424,7 @@ class SubcloudInstall(object):
             os.remove(self.input_iso)
 
         if (self.www_root is not None and os.path.isdir(self.www_root)):
-            if common_utils.is_debian():
+            if dccommon_utils.is_debian():
                 cleanup_cmd = [
                     GEN_ISO_COMMAND,
                     "--id", self.name,
@@ -430,7 +438,6 @@ class SubcloudInstall(object):
                     "--www-root", self.www_root,
                     "--delete"
                 ]
-
             try:
                 with open(os.devnull, "w") as fnull:
                     subprocess.check_call(  # pylint: disable=E1102
@@ -538,19 +545,21 @@ class SubcloudInstall(object):
             if k in payload:
                 iso_values[k] = payload.get(k)
 
+        software_version = str(payload['software_version'])
+        iso_values['software_version'] = payload['software_version']
+        iso_values['image'] = payload['image']
+
         override_path = os.path.join(override_path, self.name)
         if not os.path.isdir(override_path):
             os.mkdir(override_path, 0o755)
 
-        self.www_root = os.path.join(SUBCLOUD_ISO_PATH,
+        self.www_root = os.path.join(SUBCLOUD_ISO_PATH, software_version)
-                                     str(iso_values['software_version']))
-
-        software_version = str(payload['software_version'])
 
         feed_path_rel_version = os.path.join(SUBCLOUD_FEED_PATH,
                                              "rel-{version}".format(
                                                  version=software_version))
-        if common_utils.is_debian():
+
+        if dccommon_utils.is_debian(software_version):
             self.check_ostree_mount(feed_path_rel_version)
 
         # Clean up iso directory if it already exists
@@ -584,7 +593,11 @@ class SubcloudInstall(object):
             if k in payload:
                 del payload[k]
 
-        if common_utils.is_centos():
+        # Only applicable for 22.06:
+        if (
+            dccommon_utils.is_centos(software_version)
+            and software_version == dccommon_utils.LAST_SW_VERSION_IN_CENTOS
+        ):
             # when adding a new subcloud, the subcloud will pull
             # the file "packages_list" from the controller.
             # The subcloud pulls from /var/www/pages/iso/<version>/.
@@ -624,7 +637,7 @@ class SubcloudInstall(object):
         try:
             # Since this is a long-running task we want to register
             # for cleanup on process restart/SWACT.
-            common_utils.run_playbook(log_file, install_command)
+            dccommon_utils.run_playbook(log_file, install_command)
         except exceptions.PlaybookExecutionFailed:
             msg = ("Failed to install %s, check individual "
                    "log at %s or run %s for details"

distributedcloud/dccommon/utils.py

@@ -46,6 +46,8 @@ STALE_TOKEN_DURATION_STEP = 20
 # Exitcode from 'timeout' command on timeout:
 TIMEOUT_EXITCODE = 124
 
+LAST_SW_VERSION_IN_CENTOS = "22.06"
+
 
 class memoized(object):
     """Decorator.
@@ -253,11 +255,27 @@ def get_os_type(release_file=consts.OS_RELEASE_FILE):
     return get_os_release(release_file)[0]
 
 
-def is_debian():
+def is_debian(software_version=None):
+    """Check target version or underlying OS type.
+
+    Check either the given software_version (e.g. for checking a subcloud,
+    or prestaging operation), or the underlying OS type (for this running
+    instance)
+    """
+    if software_version:
+        return not is_centos(software_version)
     return get_os_type() == consts.OS_DEBIAN
 
 
-def is_centos():
+def is_centos(software_version=None):
+    """Check target version or underlying OS type.
+
+    Check either the given software_version (e.g. for checking a subcloud,
+    or prestaging operation), or the underlying OS type (for this running
+    instance)
+    """
+    if software_version:
+        return software_version <= LAST_SW_VERSION_IN_CENTOS
     return get_os_type() == consts.OS_CENTOS
 
 

distributedcloud/dcmanager/common/prestage.py

@@ -34,6 +34,7 @@ from dccommon.drivers.openstack.sdk_platform import OpenStackDriver
 from dccommon.drivers.openstack.sysinv_v1 import SysinvClient
 from dccommon.exceptions import PlaybookExecutionFailed
 from dccommon.exceptions import PlaybookExecutionTimeout
+from dccommon.utils import LAST_SW_VERSION_IN_CENTOS
 from dccommon.utils import run_playbook
 
 from dcmanager.common import consts
@@ -58,8 +59,6 @@ ANSIBLE_PRESTAGE_SUBCLOUD_IMAGES_PLAYBOOK = \
     "/usr/share/ansible/stx-ansible/playbooks/prestage_images.yml"
 ANSIBLE_PRESTAGE_INVENTORY_SUFFIX = '_prestage_inventory.yml'
 
-LAST_SW_VERSION_IN_CENTOS = "22.06"
-
 
 def is_deploy_status_prestage(deploy_status):
     return deploy_status in (consts.PRESTAGE_STATE_PREPARE,

distributedcloud/dcmanager/manager/subcloud_manager.py

@@ -474,7 +474,7 @@ class SubcloudManager(manager.Manager):
                     ansible_subcloud_inventory_file,
                     subcloud.software_version)
                 apply_thread = threading.Thread(
-                    target=self.run_deploy,
+                    target=self.run_deploy_thread,
                     args=(subcloud, payload, context,
                           None, None, None, rehome_command))
             else:
@@ -489,7 +489,7 @@ class SubcloudManager(manager.Manager):
                     ansible_subcloud_inventory_file,
                     subcloud.software_version)
                 apply_thread = threading.Thread(
-                    target=self.run_deploy,
+                    target=self.run_deploy_thread,
                     args=(subcloud, payload, context,
                           install_command, apply_command, deploy_command))
 
@@ -536,7 +536,7 @@ class SubcloudManager(manager.Manager):
 
             del payload['sysadmin_password']
             apply_thread = threading.Thread(
-                target=self.run_deploy,
+                target=self.run_deploy_thread,
                 args=(subcloud, payload, context, None, None, deploy_command))
             apply_thread.start()
             return db_api.subcloud_db_model_to_dict(subcloud)
@@ -612,7 +612,7 @@ class SubcloudManager(manager.Manager):
             management_subnet = utils.get_management_subnet(payload)
             network_reconfig = management_subnet != subcloud.management_subnet
             apply_thread = threading.Thread(
-                target=self.run_deploy,
+                target=self.run_deploy_thread,
                 args=(subcloud, payload, context,
                       install_command, apply_command, deploy_command,
                       None, network_reconfig))
@@ -1261,16 +1261,27 @@ class SubcloudManager(manager.Manager):
         except Exception as e:
             LOG.exception(e)
 
-    # TODO(kmacleod) add outer try/except here to catch and log unexpected
+    def run_deploy_thread(self, subcloud, payload, context,
-    # exception. As this stands, any uncaught exception is a silent (unlogged)
+                          install_command=None, apply_command=None,
-    # failure
+                          deploy_command=None, rehome_command=None,
-    def run_deploy(self, subcloud, payload, context,
+                          network_reconfig=None):
-                   install_command=None, apply_command=None,
+        try:
-                   deploy_command=None, rehome_command=None,
+            self._run_deploy(subcloud, payload, context,
-                   network_reconfig=None):
+                             install_command, apply_command,
+                             deploy_command, rehome_command,
+                             network_reconfig)
+        except Exception as ex:
+            LOG.exception("run_deploy failed")
+            raise ex
 
-        log_file = os.path.join(consts.DC_ANSIBLE_LOG_DIR, subcloud.name) + \
+    def _run_deploy(self, subcloud, payload, context,
-            '_playbook_output.log'
+                    install_command, apply_command,
+                    deploy_command, rehome_command,
+                    network_reconfig):
+        log_file = (
+            os.path.join(consts.DC_ANSIBLE_LOG_DIR, subcloud.name)
+            + "_playbook_output.log"
+        )
         if install_command:
             install_success = self._run_subcloud_install(
                 context, subcloud, install_command,
@@ -1285,9 +1296,10 @@ class SubcloudManager(manager.Manager):
                     context, subcloud.id,
                     deploy_status=consts.DEPLOY_STATE_BOOTSTRAPPING,
                     error_description=consts.ERROR_DESC_EMPTY)
-            except Exception as e:
+            except Exception:
-                LOG.exception(e)
+                LOG.error("DB subcloud_update failed")
-                raise e
+                # exception is logged above
+                raise
 
             # Run the ansible boostrap-subcloud playbook
             LOG.info("Starting bootstrap of %s" % subcloud.name)