Merge "Add automated test for subcloud addition"

config/deployment_assets/files/default.json5

@@ -5,23 +5,23 @@
 	// All the files are assumed to be stored on the Central Cloud in the case of a DC system.
 
 	"controller" : {
-		deployment_config_file: "/home/sysadmin/deployment-config.yaml",
+		"deployment_config_file": "/home/sysadmin/deployment-config.yaml",
 	},
-
 	"subclouds" : {
-
 		"subcloud1": {
-			bootstrap_file: "/home/sysadmin/subcloud1/subcloud1-bootstrap-values.yaml",
+			"bootstrap_file": "/home/sysadmin/subcloud-1/subcloud1-bootstrap-values.yaml",
-			deployment_config_file: "/home/sysadmin/subcloud1/subcloud1-deploy-standard.yaml",
+			"deployment_config_file": "/home/sysadmin/subcloud-1/subcloud1-deploy-standard.yaml",
-			install_file: "/home/sysadmin/subcloud1/subcloud1-install-values.yaml"
+			"install_file": "/home/sysadmin/subcloud-1/subcloud1-install-values.yaml"
 		},
-
 		"subcloud2": {
-			bootstrap_file: "/home/sysadmin/subcloud2/subcloud2-bootstrap-values.yaml",
+			"bootstrap_file": "/home/sysadmin/subcloud-2/subcloud2-bootstrap-values.yaml",
-			deployment_config_file: "/home/sysadmin/subcloud2/subcloud2-deploy-standard.yaml",
+			"deployment_config_file": "/home/sysadmin/subcloud-2/subcloud2-deploy-standard.yaml",
-			install_file: "/home/sysadmin/subcloud2/subcloud2-install-values.yaml"
+			"install_file": "/home/sysadmin/subcloud-2/subcloud2-install-values.yaml"
+		},
+        "subcloud3": {
+			"bootstrap_file": "/home/sysadmin/subcloud-3/subcloud3-bootstrap-values.yaml",
+			"deployment_config_file": "/home/sysadmin/subcloud-3/subcloud3-deploy-standard.yaml",
+			"install_file": "/home/sysadmin/subcloud-3/subcloud3-install-values.yaml"
 		},
-
 	},
-
 }

config/deployment_assets/objects/deployment_assets.py

@@ -32,7 +32,7 @@ class DeploymentAssets:
         Returns (str): boostrap_file
 
         """
-        return self.bootstrap_file
+        return self.bootstrap_file.strip()
 
     def get_deployment_config_file(self) -> str:
         """
@@ -41,7 +41,7 @@ class DeploymentAssets:
         Returns (str): deployment_config_file
 
         """
-        return self.deployment_config_file
+        return self.deployment_config_file.strip()
 
     def get_install_file(self) -> str:
         """
@@ -50,4 +50,4 @@ class DeploymentAssets:
         Returns (str): install_file
 
         """
-        return self.install_file
+        return self.install_file.strip()

keywords/cloud_platform/dcmanager/dcmanager_subcloud_add_keywords.py

@@ -0,0 +1,49 @@
+from config.configuration_manager import ConfigurationManager
+from framework.ssh.ssh_connection import SSHConnection
+from keywords.base_keyword import BaseKeyword
+from keywords.cloud_platform.command_wrappers import source_openrc
+from keywords.cloud_platform.dcmanager.dcmanager_subcloud_list_keywords import DcManagerSubcloudListKeywords
+
+
+class DcManagerSubcloudAddKeywords(BaseKeyword):
+    """
+    This class contains all the keywords related to the 'dcmanager subcloud Add' commands.
+    """
+
+    def __init__(self, ssh_connection: SSHConnection):
+        """Constructor
+
+        Args:
+            ssh_connection (SSHConnection): ssh for the active controller
+        """
+        self.ssh_connection = ssh_connection
+
+    def dcmanager_subcloud_add(self, subcloud_name: str):
+        """Adds the subcloud using 'dcmanager subcloud add '.
+
+        Args:
+            subcloud_name (str): a str name for the subcloud.
+
+        """
+        # Get the subcloud config
+        sc_config = ConfigurationManager.get_lab_config().get_subcloud(subcloud_name)
+
+        # Get the subcloud deployment assets
+        deployment_assets_config = ConfigurationManager.get_deployment_assets_config()
+        sc_assets = deployment_assets_config.get_subcloud_deployment_assets(subcloud_name)
+        bootstrap_file = sc_assets.get_bootstrap_file()
+        deploy_file = sc_assets.get_deployment_config_file()
+        install_file = sc_assets.get_install_file()
+
+        # Get the subcloud bootstrap address
+        boot_add = sc_config.get_nodes()[0].get_ip()
+        admin_creds = sc_config.get_admin_credentials()
+
+        # Execute the command
+        cmd = f"dcmanager subcloud add --bootstrap-address {boot_add} --bootstrap-values {bootstrap_file} --deploy-config {deploy_file} --sysadmin-password {admin_creds.get_password()} --bmc-password {sc_config.get_bm_password()} --install-values {install_file}"
+        self.ssh_connection.send(source_openrc(cmd))
+        self.validate_success_return_code(self.ssh_connection)
+
+        # validate subcloud status until complete
+        dc_manager_sc_list_kw = DcManagerSubcloudListKeywords(self.ssh_connection)
+        dc_manager_sc_list_kw.validate_subcloud_status(subcloud_name, "complete")

keywords/cloud_platform/dcmanager/dcmanager_subcloud_list_keywords.py

@@ -1,3 +1,7 @@
+import time
+
+from framework.logging.automation_logger import get_logger
+from framework.ssh.ssh_connection import SSHConnection
 from keywords.base_keyword import BaseKeyword
 from keywords.cloud_platform.command_wrappers import source_openrc
 from keywords.cloud_platform.dcmanager.objects.dcmanager_subcloud_list_output import DcManagerSubcloudListOutput
@@ -8,27 +12,59 @@ class DcManagerSubcloudListKeywords(BaseKeyword):
     This class contains all the keywords related to the 'dcmanager subcloud list' commands.
     """
 
-    def __init__(self, ssh_connection):
+    def __init__(self, ssh_connection: SSHConnection):
-        """
+        """Constructor
-        Constructor
+
         Args:
-            ssh_connection:
+            ssh_connection (SSHConnection): ssh object
 
         """
         self.ssh_connection = ssh_connection
 
     def get_dcmanager_subcloud_list(self) -> DcManagerSubcloudListOutput:
-        """
+        """Gets the 'dcmanager subcloud list' output.
-        Gets the 'dcmanager subcloud list' output.
+
         Args: None
 
         Returns:
-            dcmanager subcloud list (DcManagerSubcloudListOutput): a DcManagerSubcloudListOutput object representing
+            DcManagerSubcloudListOutput: a DcManagerSubcloudListOutput object representing
             the output of the command 'dcmanager subcloud list'.
 
         """
-        output = self.ssh_connection.send(source_openrc('dcmanager subcloud list'))
+        output = self.ssh_connection.send(source_openrc("dcmanager subcloud list"))
         self.validate_success_return_code(self.ssh_connection)
         dcmanager_subcloud_list_output = DcManagerSubcloudListOutput(output)
 
         return dcmanager_subcloud_list_output
+
+    def validate_subcloud_status(self, subcloud_name: str, status: str) -> bool:
+        """Validates the status of specified subcloud until reaches the desired status.
+
+        Args:
+            subcloud_name (str): a str name for the subcloud.
+            status (str): a str status for the subcloud.
+
+        Returns:
+            bool: True if the subcloud reaches the desired status, False otherwise.
+
+        Raises:
+            Exception: if the subcloud is in a failed state.
+
+        """
+        failed_status = ["bootstrap-failed", "install-failed", "create-failed"]
+        time_out = 3600
+        polling_sleep_time = 60
+        end_time = time.time() + time_out
+
+        while time.time() < end_time:
+            sc_list_out = self.get_dcmanager_subcloud_list().get_subcloud_by_name(subcloud_name)
+            sc_status = sc_list_out.get_deploy_status()
+            msg = f"Subcloud {subcloud_name} is in {sc_status} state"
+            if sc_status == status:
+                return True
+            else:
+                if sc_status in failed_status:
+                    raise Exception(msg)
+            time.sleep(polling_sleep_time)
+        get_logger().log_error(msg)
+        raise TimeoutError(msg)

keywords/cloud_platform/ssh/lab_connection_keywords.py

@@ -13,8 +13,7 @@ class LabConnectionKeywords(BaseKeyword):
     """
 
     def get_active_controller_ssh(self) -> SSHConnection:
-        """
+        """Gets the active controller ssh
-        Gets the active controller ssh
 
         Returns:
             SSHConnection: the ssh for the active controller
@@ -118,8 +117,8 @@ class LabConnectionKeywords(BaseKeyword):
         return connection
 
     def get_subcloud_ssh(self, subcloud_name: str) -> SSHConnection:
-        """
+        """Gets an SSH connection to the 'Subcloud' node whose name is specified by the argument 'subcloud_name'.
-        Gets an SSH connection to the 'Subcloud' node whose name is specified by the argument 'subcloud_name'.
+
         Args:
              subcloud_name (str): The name of the 'subcloud' node.
 

keywords/files/file_keywords.py

@@ -1,5 +1,6 @@
 import time
 
+from config.configuration_manager import ConfigurationManager
 from framework.exceptions.keyword_exception import KeywordException
 from framework.logging.automation_logger import get_logger
 from framework.ssh.ssh_connection import SSHConnection
@@ -179,3 +180,16 @@ class FileKeywords(BaseKeyword):
         """
         self.ssh_connection.send_as_sudo(f"rm -r -f {folder_path}")
         return self.validate_file_exists_with_sudo(folder_path)
+
+    def execute_rsync(self, source_path: str, remote_path: str):
+        """Execute rsync command to copy files from source (active controller) to destination
+
+        Args:
+            source_path (str): The source path in active controller
+            remote_path (str): The destination path
+        """
+        pasw = ConfigurationManager.get_lab_config().get_admin_credentials().get_password()
+
+        # active_controller_ssh.send(f"sshpass -p '{pasw}' rsync -avz {source} {user}@{destination}")
+        self.ssh_connection.send(f"sshpass -p '{pasw}' rsync -avz {source_path} {remote_path}")
+        self.validate_success_return_code(self.ssh_connection)

testcases/cloud_platform/sanity/test_dc_sanity.py

@@ -0,0 +1,118 @@
+import os
+
+from pytest import mark
+
+from config.configuration_manager import ConfigurationManager
+from framework.logging.automation_logger import get_logger
+from framework.validation.validation import validate_equals
+from keywords.cloud_platform.dcmanager.dcmanager_subcloud_add_keywords import DcManagerSubcloudAddKeywords
+from keywords.cloud_platform.dcmanager.dcmanager_subcloud_manager_keywords import DcManagerSubcloudManagerKeywords
+from keywords.cloud_platform.ssh.lab_connection_keywords import LabConnectionKeywords
+from keywords.cloud_platform.system.host.system_host_list_keywords import SystemHostListKeywords
+from keywords.cloud_platform.system.host.system_host_swact_keywords import SystemHostSwactKeywords
+from keywords.files.file_keywords import FileKeywords
+
+
+def sanity_pre_requisite():
+    """
+    Sanity pre-requisite for the test case
+    """
+    # Sync the lab configuration between active and standby controller
+    lab_config = ConfigurationManager.get_lab_config()
+    active_controller_ssh = LabConnectionKeywords().get_active_controller_ssh()
+    user = lab_config.get_admin_credentials().get_user_name()
+
+    # get the standby controller
+    standby_controller = SystemHostListKeywords(active_controller_ssh).get_standby_controller()
+    if not standby_controller:
+        raise Exception("System does not have a standby controller")
+    standby_host_name = standby_controller.get_host_name()
+    deployment_assets_config = ConfigurationManager.get_deployment_assets_config()
+    file_kw = FileKeywords(active_controller_ssh)
+    # sync all subclouds files
+    for sc_assets in deployment_assets_config.subclouds_deployment_assets.values():
+        # get base path of the file
+        for file in [sc_assets.get_deployment_config_file(), sc_assets.get_install_file(), sc_assets.get_bootstrap_file()]:
+            # get the base path of the file
+            base_file_path = os.path.join(os.path.dirname(file), "")
+            # prepare remote path
+            remote_path = f"{user}@{standby_host_name}:{base_file_path}"
+            file_kw.execute_rsync(file, remote_path)
+
+
+def subcloud_add(subcloud_name: str):
+    """Add a subcloud to the system.
+
+    Args:
+        subcloud_name (str): name of the subcloud to be added
+    """
+    # Gets the SSH connection to the active controller of the central cloud.
+    change_state_timeout = 60
+
+    ssh_connection = LabConnectionKeywords().get_active_controller_ssh()
+    dcm_sc_add_kw = DcManagerSubcloudAddKeywords(ssh_connection)
+    dcm_sc_add_kw.dcmanager_subcloud_add(subcloud_name)
+    dcmanager_subcloud_manage_keywords = DcManagerSubcloudManagerKeywords(ssh_connection)
+    dcmanager_subcloud_manage_output = dcmanager_subcloud_manage_keywords.get_dcmanager_subcloud_manage(subcloud_name, change_state_timeout)
+    manage_status = dcmanager_subcloud_manage_output.get_dcmanager_subcloud_manage_object().get_management()
+    get_logger().log_info(f"The management state of the subcloud {subcloud_name} {manage_status}")
+
+
+@mark.p0
+@mark.lab_has_min_2_subclouds
+def test_dc_subcloud_add_simplex():
+    """Verify subcloud Add works as expected
+
+    Test Steps:
+        - log onto system controller
+        - add The subcloud
+        - validate that the subcloud is added
+    """
+    sanity_pre_requisite()
+    # read the config file for subcloud
+    subcloud_name = "subcloud1"
+    subcloud_add(subcloud_name)
+
+
+@mark.p0
+@mark.lab_has_subcloud
+def test_dc_swact():
+    """Test swact Host
+
+    Test Steps:
+    - Swact the host.
+    - Verify that the host is changed.
+    """
+    # Gets the SSH connection to the active controller of the central cloud.
+    ssh_connection = LabConnectionKeywords().get_active_controller_ssh()
+
+    # Swact the host
+    system_host_list_keywords = SystemHostListKeywords(ssh_connection)
+    active_controller = system_host_list_keywords.get_active_controller()
+    standby_controller = system_host_list_keywords.get_standby_controller()
+    get_logger().log_info(f"A 'swact' operation is about to be executed in {ssh_connection}. Current controllers' configuration before this operation: Active controller = {active_controller.get_host_name()}, Standby controller = {standby_controller.get_host_name()}.")
+    system_host_swact_keywords = SystemHostSwactKeywords(ssh_connection)
+    system_host_swact_keywords.host_swact()
+
+    # Gets the controllers after the execution of the swact operation.
+    active_controller_after_swact = system_host_list_keywords.get_active_controller()
+    standby_controller_after_swact = system_host_list_keywords.get_standby_controller()
+
+    validate_equals(active_controller.get_id(), standby_controller_after_swact.get_id(), "Validate that active controller is now standby")
+    validate_equals(standby_controller.get_id(), active_controller_after_swact.get_id(), "Validate that standby controller is now active")
+
+
+@mark.p0
+@mark.lab_has_min_2_subclouds
+def test_dc_subcloud_add_duplex():
+    """Verify subcloud Add works as expected
+
+    Test Steps:
+        - log onto system controller
+        - add The subcloud
+        - validate that the subcloud is added
+    """
+    sanity_pre_requisite()
+    # read the config file for subcloud
+    subcloud_name = "subcloud2"
+    subcloud_add(subcloud_name)

testcases/cloud_platform/sanity/test_sanity.py

@@ -682,7 +682,6 @@ def test_dc_swact_host(request):
         _ Reestablishes the active/standby host configuration.
 
     """
-    get_logger().log_info("Starting 'test_dc_swact_host' test case.")
 
     # Time in seconds for a subcloud to change its state from 'managed' to 'unmanaged' and vice versa.
     change_state_timeout = 60