Fix and improve transfer_file

- Fixed issue with destination not handling sub-folders
- Refactored transfer_file to use validate_equals_with_retry

Change-Id: I357620bde7b792c50bb7e52ade52e9ef7191f70d

framework/ssh/secure_transfer_file/secure_transfer_file.py

@@ -1,9 +1,9 @@
 import os
-import time
 
 from framework.logging.automation_logger import get_logger
 from framework.ssh.secure_transfer_file.secure_transfer_file_enum import TransferDirection
 from framework.ssh.secure_transfer_file.secure_transfer_file_input_object import SecureTransferFileInputObject
+from framework.validation.validation import validate_equals_with_retry
 from keywords.python.string import String
 
 
@@ -42,22 +42,13 @@ class SecureTransferFile:
             otherwise.
 
         """
-        file_was_transferred = False
-
         # Origin path.
         origin_path = self.secure_transfer_file_input_object.get_origin_path()
 
         # Destination path.
         destination_path = self.secure_transfer_file_input_object.get_destination_path()
-
-        # Try to extract the destination file name from destination_path. If that is not possible, attempt to extract it
-        # from origin_path. If this also fails, terminate the process.
         destination_file_name = os.path.basename(destination_path)
-        if String.is_empty(destination_file_name):
+        destination_folder_name = os.path.dirname(destination_path)
-            destination_file_name = os.path.basename(origin_path)
-            if String.is_empty(destination_file_name):
-                get_logger().log_error("The file name must be specified in the property 'origin_path'.")
-                return False
 
         # Gets the SFTP client
         sftp_client = self.secure_transfer_file_input_object.get_sftp_client()
@@ -67,16 +58,15 @@ class SecureTransferFile:
 
         # Transfers the file from remote to local destination.
         if transfer_direction == TransferDirection.FROM_REMOTE_TO_LOCAL:
-            # Verifies is the file already exists at the destination path.
+
-            file_already_exists = os.path.isfile(destination_path)
+            def is_file_at_destination():
+                return os.path.isfile(destination_path)
 
             # Manages forced file transfer.
             force_transfer = self.secure_transfer_file_input_object.get_force()
-            if file_already_exists:
+            if is_file_at_destination():
                 if not force_transfer:
-                    get_logger().log_info(
+                    get_logger().log_info(f"The file {destination_path} already exists. Try setting the 'force' property of the SecureTransferFileInputObject instance to 'True' if you want to overwrite it.")
-                        f"The file {destination_path} already exists. Try setting the 'force' property of the SecureTransferFileInputObject instance to 'True' if you want to overwrite it."
-                    )
                     return False
 
             # Transfers the file from remote to local destination.
@@ -86,36 +76,21 @@ class SecureTransferFile:
                 get_logger().log_exception(f"Error when trying to transfer the file {origin_path} to destination {destination_path}. Exception: {ex}")
                 raise ex
 
-            # Monitors the file transfer from remote to the local destination.
+            # Validate that the file was transferred successfully
-            check_interval = 3
+            validate_equals_with_retry(is_file_at_destination, True, f"File {origin_path} is transferred to {destination_path}.", timeout=self.secure_transfer_file_input_object.get_timeout_in_seconds())
-            timeout_seconds = self.secure_transfer_file_input_object.get_timeout_in_seconds()
-            end_time = time.time() + timeout_seconds
-            while time.time() < end_time:
-                try:
-                    file_was_transferred = os.path.isfile(destination_path)
-                except Exception as ex:
-                    get_logger().log_exception(f"Error while trying to transfer the file {destination_path}. Exception: {ex}")
-                    raise ex
-                if file_was_transferred:
-                    break
-                time.sleep(check_interval)
-                get_logger().log_exception(
-                    f"Waiting for {check_interval} more seconds to the file {origin_path} be transferred to {destination_path}. Maximum total time to wait: {timeout_seconds} s. Remaining time: {end_time - time.time():.3f} s."
-                )
 
         # Transfers to remote destination.
         elif transfer_direction == TransferDirection.FROM_LOCAL_TO_REMOTE:
-            # Verifies is the file already exists at the destination path.
+
-            file_list = sftp_client.listdir()
+            def is_file_at_destination():
-            file_already_exists = any(destination_file_name == file for file in file_list)
+                file_list = sftp_client.listdir(destination_folder_name)
+                return any(destination_file_name == file for file in file_list)
 
             # Manages forced file transfer.
             force_transfer = self.secure_transfer_file_input_object.get_force()
-            if file_already_exists:
+            if is_file_at_destination():
                 if not force_transfer:
-                    get_logger().log_info(
+                    get_logger().log_info(f"The file {destination_path} already exists. Try setting the 'force' property of the SecureTransferFileInputObject instance to 'True' if you want to overwrite it.")
-                        f"The file {destination_path} already exists. Try setting the 'force' property of the SecureTransferFileInputObject instance to 'True' if you want to overwrite it."
-                    )
                     return False
 
             # Transfers the file to remote destination.
@@ -125,37 +100,11 @@ class SecureTransferFile:
                 get_logger().log_exception(f"Error when trying to transfer the file {origin_path} to destination {destination_path}. Exception: {ex}")
                 raise ex
 
-            # Monitors the file transfer to the remote destination.
+            # Validate that the file was transferred successfully
-            file_was_transferred = False
+            validate_equals_with_retry(is_file_at_destination, True, f"File {origin_path} is transferred to {destination_path}.", timeout=self.secure_transfer_file_input_object.get_timeout_in_seconds())
-            check_interval = 3
-            timeout_seconds = self.secure_transfer_file_input_object.get_timeout_in_seconds()
-            end_time = time.time() + timeout_seconds
-            while time.time() < end_time:
-                try:
-                    file_list = sftp_client.listdir()
-                except Exception as ex:
-                    get_logger().log_exception(f"Error while trying to transfer the file {destination_path}. Exception: {ex}")
-                    raise ex
-                file_was_transferred = any(destination_file_name == file for file in file_list)
-                if file_was_transferred:
-                    break
-                time.sleep(check_interval)
-                get_logger().log_exception(
-                    f"Waiting for {check_interval} more seconds to the file {origin_path} be transferred to {destination_path}. Maximum total time to wait: {timeout_seconds} s. Remaining time: {end_time - time.time()} s."
-                )
 
         # The property 'transfer_direction' was not properly specified.
         else:
             message = f"Error when trying to transfer the file {origin_path} to destination {destination_path}. Property 'transfer_direction' must be specified as either TransferDirection.FROM_LOCAL_TO_REMOTE or TransferDirection.FROM_REMOTE_TO_LOCAL"
             get_logger().log_exception(message)
             raise ValueError(message)
-
-        # If we've reached this point, the file was either transferred successfully or the transfer failed due to a timeout.
-        if file_was_transferred:
-            message = f"File {origin_path} successfully transferred to {destination_path}."
-            get_logger().log_info(message)
-            return True
-        else:
-            message = f"Error while trying to transfer the file {destination_path}. Time to transfer ({self.secure_transfer_file_input_object.get_timeout_in_seconds()} s) exceeded."
-            get_logger().log_error(message)
-            return False

testcases/cloud_platform/regression/containers/test_k8s_dashboard.py

@@ -46,10 +46,7 @@ def copy_k8s_files(ssh_connection):
 
         # Transfers the dashboard file from local path to remote path.
         secure_transfer_file = SecureTransferFile(secure_transfer_file_input_object)
-        file_transfer_succeeded = secure_transfer_file.transfer_file()
+        secure_transfer_file.transfer_file()
-
-        # Asserts the file was really transferred.
-        assert file_transfer_succeeded
 
 
 def create_k8s_dashboard(request, namespace, con_ssh):

testcases/cloud_platform/sanity/test_sanity.py

@@ -533,10 +533,7 @@ def test_dc_install_custom_app():
 
     # Transfers the app file from local path to remote path.
     secure_transfer_file = SecureTransferFile(secure_transfer_file_input_object)
-    file_transfer_succeeded = secure_transfer_file.transfer_file()
+    secure_transfer_file.transfer_file()
-
-    # Asserts the file was really transferred.
-    assert file_transfer_succeeded
 
     # Step 2: Upload the app file to the active controller
 
@@ -618,10 +615,7 @@ def test_dc_install_custom_app():
 
         # Transfer the app file to the current subcloud.
         secure_transfer_file = SecureTransferFile(secure_transfer_file_input_object)
-        file_transfer_succeeded = secure_transfer_file.transfer_file()
+        secure_transfer_file.transfer_file()
-
-        # Asserts the file was really transferred to the current subcloud.
-        assert file_transfer_succeeded
 
         # Step 6: Uploads the app file to the current subcloud.