diff --git a/ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/ClientInstallationRequestListener.java b/ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/ClientInstallationRequestListener.java index 3cf92d2..b8399ea 100644 --- a/ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/ClientInstallationRequestListener.java +++ b/ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/ClientInstallationRequestListener.java @@ -130,6 +130,7 @@ public class ClientInstallationRequestListener implements InitializingBean { switch (requestType) { case "DIAGNOSTICS" -> processDiagnosticsRequest(request); case "VM" -> processOnboardingRequest(request); + case "REMOVE" -> processRemoveRequest(request); default -> throw new IllegalArgumentException("Unsupported request type: "+requestType); }; @@ -174,6 +175,7 @@ public class ClientInstallationRequestListener implements InitializingBean { // Create client installation task ClientInstallationTask newTask = ClientInstallationTask.builder() .id(request.get("requestId")) + .taskType(ClientInstallationTask.TASK_TYPE.DIAGNOSTIC) .requestId(request.get("requestId")) .type(request.get("requestType")) .nodeId(request.get("deviceId")) @@ -265,4 +267,26 @@ public class ClientInstallationRequestListener implements InitializingBean { log.trace("InstallationEventListener.convertToNodeInfoMap(): END: nodeMap: {}", nodeMap); return nodeMap; } + + private void processRemoveRequest(Map request) throws Exception { + String requestId = request.getOrDefault("requestId", "").trim(); + String nodeAddress = request.getOrDefault("deviceIpAddress", "").trim(); + log.info("InstallationEventListener: New node REMOVE request with Id: {}, address={}", requestId, nodeAddress); + if (StringUtils.isBlank(requestId)) { + clientInstaller.sendErrorClientInstallationReport("MISSING-REQUEST-ID", "INVALID REQUEST. MISSING REQUEST ID"); + return; + } + if (StringUtils.isBlank(nodeAddress)) { + clientInstaller.sendErrorClientInstallationReport(requestId, "INVALID REQUEST. MISSING IP ADDRESS"); + return; + } + + try { + log.debug("InstallationEventListener: Off-boarding node due to REMOVE request with Id: {}", requestId); + nodeRegistration.unregisterNode(nodeAddress, new TranslationContext(requestId)); + } catch (Exception e) { + log.warn("InstallationEventListener: EXCEPTION while executing REMOVE request with Id: {}\n", requestId, e); + clientInstaller.sendErrorClientInstallationReport(requestId, "ERROR: "+e.getMessage()); + } + } } diff --git a/ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/ClientInstallationTask.java b/ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/ClientInstallationTask.java index 1d6085f..cfee0bc 100644 --- a/ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/ClientInstallationTask.java +++ b/ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/ClientInstallationTask.java @@ -16,6 +16,7 @@ import lombok.Builder; import lombok.Data; import java.util.List; +import java.util.concurrent.Callable; /** * Client installation task @@ -23,17 +24,22 @@ import java.util.List; @Data @Builder public class ClientInstallationTask { + public enum TASK_TYPE { INSTALL, UNINSTALL, DIAGNOSTIC, OTHER } + private final String id; + private final TASK_TYPE taskType; private final String nodeId; private final String requestId; private final String name; private final String os; private final String address; - private final String type; + private final String type; // Node type (VM, baremetal etc) private final String provider; private final SshConfig ssh; private final NodeRegistryEntry nodeRegistryEntry; private final List instructionSets; private final TranslationContext translationContext; private boolean nodeMustBeInRegistry = true; + + private Callable callback; } diff --git a/ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/ClientInstaller.java b/ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/ClientInstaller.java index 48543f5..bd20ce2 100644 --- a/ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/ClientInstaller.java +++ b/ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/ClientInstaller.java @@ -10,6 +10,7 @@ package gr.iccs.imu.ems.baguette.client.install; import gr.iccs.imu.ems.baguette.server.BaguetteServer; +import gr.iccs.imu.ems.baguette.server.ClientShellCommand; import gr.iccs.imu.ems.baguette.server.NodeRegistryEntry; import gr.iccs.imu.ems.brokercep.BrokerCepService; import gr.iccs.imu.ems.common.plugin.PluginManager; @@ -69,26 +70,49 @@ public class ClientInstaller implements InitializingBean { executorService.submit(() -> { long taskCnt = taskCounter.getAndIncrement(); String resultStr = ""; + String callbackStr = ""; String errorStr = ""; // Execute task + boolean result = false; try { log.info("ClientInstaller: Executing Client installation Task #{}: task-id={}, node-id={}, name={}, type={}, address={}", taskCnt, task.getId(), task.getNodeId(), task.getName(), task.getType(), task.getAddress()); long startTm = System.currentTimeMillis(); - boolean result = executeTask(task, taskCnt); + result = executeTask(task, taskCnt); long endTm = System.currentTimeMillis(); resultStr = result ? "SUCCESS" : "FAILED"; log.info("ClientInstaller: Client installation Task #{}: result={}, duration={}ms", taskCnt, resultStr, endTm - startTm); } catch (Throwable t) { - log.info("ClientInstaller: Exception caught in Client installation Task #{}: Exception: ", taskCnt, t); - errorStr = t.getMessage(); + log.error("ClientInstaller: Exception caught in Client installation Task #{}: Exception: ", taskCnt, t); + errorStr = "EXCEPTION " + t.getMessage(); + } + + // Run callback (if any) + if (result && task.getCallback()!=null) { + try { + log.debug("ClientInstaller: CALLBACK started: Task #{}: task-id={}", taskCnt, task.getId()); + long startTm = System.currentTimeMillis(); + callbackStr = task.getCallback().call(); + long endTm = System.currentTimeMillis(); + log.info("ClientInstaller: CALLBACK completed: Task #{}: callback-result={}, duration={}ms", taskCnt, callbackStr, endTm - startTm); + if (! "OK".equalsIgnoreCase(callbackStr)) resultStr = "FAILED"; + } catch (Throwable t) { + log.error("ClientInstaller: CALLBACK: Exception caught while running callback of Client installation Task #{}: Exception: ", taskCnt, t); + callbackStr = "CALLBACK-EXCEPTION " + t.getMessage(); + resultStr = "FAILED"; + } + } else { + if (result) + log.debug("ClientInstaller: No CALLBACK found for Task #{}", taskCnt); + else + log.debug("ClientInstaller: Skipped CALLBACK because execution failed for Task #{}", taskCnt); } // Send execution report to local broker try { - resultStr = StringUtils.defaultIfBlank(resultStr, "ERROR: " + errorStr); + resultStr = StringUtils.defaultIfBlank(resultStr, "ERROR: " + errorStr + " " + callbackStr); sendSuccessClientInstallationReport(taskCnt, task, resultStr); } catch (Throwable t) { log.info("ClientInstaller: EXCEPTION while sending Client installation report for Task #{}: Exception: ", taskCnt, t); @@ -103,7 +127,8 @@ public class ClientInstaller implements InitializingBean { return executeVmOrBaremetalTask(task, taskCounter); } else - if ("DIAGNOSTICS".equalsIgnoreCase(task.getType())) { + //if ("DIAGNOSTICS".equalsIgnoreCase(task.getType())) { + if (task.getTaskType()==ClientInstallationTask.TASK_TYPE.DIAGNOSTIC) { return executeDiagnosticsTask(task, taskCounter); } else { log.error("ClientInstaller: UNSUPPORTED TASK TYPE: {}", task.getType()); @@ -121,30 +146,55 @@ public class ClientInstaller implements InitializingBean { if (! task.isNodeMustBeInRegistry()) entry = task.getNodeRegistryEntry(); - entry.nodeInstalling(task); - - // Call InstallationContextPlugin's before installation - log.debug("ClientInstaller: PRE-INSTALLATION: Calling installation context processors: {}", properties.getInstallationContextProcessorPlugins()); - pluginManager.getActivePlugins(InstallationContextProcessorPlugin.class) - .forEach(plugin->((InstallationContextProcessorPlugin)plugin).processBeforeInstallation(task, taskCounter)); - - log.debug("ClientInstaller: INSTALLATION: Executing installation task: task-counter={}, task={}", taskCounter, task); - boolean success = executeVmTask(task, taskCounter); - log.debug("ClientInstaller: NODE_REGISTRY_ENTRY after installation execution: \n{}", task.getNodeRegistryEntry()); - - if (entry.getState()==NodeRegistryEntry.STATE.INSTALLING) { - log.warn("ClientInstaller: NODE_REGISTRY_ENTRY status is still INSTALLING after executing client installation. Changing to INSTALL_ERROR"); - entry.nodeInstallationError(null); + // Check if node is being removed or have been archived + if (task.getNodeRegistryEntry().isArchived()) { + log.warn("ClientInstaller: Node is being removed or has been archived: {}", properties.getInstallationContextProcessorPlugins()); + throw new IllegalStateException("Node is being removed or has been archived: Node IP address: "+ task.getAddress()); } - // Call InstallationContextPlugin's after installation - log.debug("ClientInstaller: POST-INSTALLATION: Calling installation context processors: {}", properties.getInstallationContextProcessorPlugins()); - pluginManager.getActivePlugins(InstallationContextProcessorPlugin.class) - .forEach(plugin->((InstallationContextProcessorPlugin)plugin).processAfterInstallation(task, taskCounter, success)); + boolean success; + if (! task.getInstructionSets().isEmpty()) { + // Starting installation + entry.nodeInstalling(task); + + // Call InstallationContextPlugin's before installation + log.debug("ClientInstaller: PRE-INSTALLATION: Calling installation context processors: {}", properties.getInstallationContextProcessorPlugins()); + pluginManager.getActivePlugins(InstallationContextProcessorPlugin.class) + .forEach(plugin -> ((InstallationContextProcessorPlugin) plugin).processBeforeInstallation(task, taskCounter)); + + log.debug("ClientInstaller: INSTALLATION: Executing installation task: task-counter={}, task={}", taskCounter, task); + success = executeVmTask(task, taskCounter); + log.debug("ClientInstaller: NODE_REGISTRY_ENTRY after installation execution: \n{}", task.getNodeRegistryEntry()); + + if (entry.getState() == NodeRegistryEntry.STATE.INSTALLING) { + log.warn("ClientInstaller: NODE_REGISTRY_ENTRY status is still INSTALLING after executing client installation. Changing to INSTALL_ERROR"); + entry.nodeInstallationError(null); + } + + // Call InstallationContextPlugin's after installation + log.debug("ClientInstaller: POST-INSTALLATION: Calling installation context processors: {}", properties.getInstallationContextProcessorPlugins()); + pluginManager.getActivePlugins(InstallationContextProcessorPlugin.class) + .forEach(plugin -> ((InstallationContextProcessorPlugin) plugin).processAfterInstallation(task, taskCounter, success)); + } else { + log.debug("ClientInstaller: SKIP INSTALLATION: Task has no instructions sets. Skipping execution: Node IP address: "+ task.getAddress()); + success = true; + } // Pre-register Node to baguette Server Coordinator - log.debug("ClientInstaller: POST-INSTALLATION: Node is being pre-registered: {}", entry); - baguetteServer.getNodeRegistry().getCoordinator().preregister(entry); + if (task.getTaskType()==ClientInstallationTask.TASK_TYPE.INSTALL) { + log.debug("ClientInstaller: POST-INSTALLATION: Node is being pre-registered: {}", entry); + baguetteServer.getNodeRegistry().getCoordinator().preregister(entry); + } + + // Un-register Node from baguette Server Coordinator + if (task.getTaskType()==ClientInstallationTask.TASK_TYPE.UNINSTALL) { + ClientShellCommand csc = ClientShellCommand.getActiveByIpAddress(entry.getIpAddress()); + log.debug("ClientInstaller: POST-INSTALLATION: CSC of node to be unregistered: {}", csc); + if (csc!=null) { + log.debug("ClientInstaller: POST-INSTALLATION: Node is going to be unregistered: {}", entry); + baguetteServer.getNodeRegistry().getCoordinator().unregister(csc); + } + } log.debug("ClientInstaller: Installation outcome: {}", success ? "Success" : "Error"); return success; diff --git a/ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/api/INodeRegistration.java b/ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/api/INodeRegistration.java index aff0e94..b70d2e4 100644 --- a/ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/api/INodeRegistration.java +++ b/ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/api/INodeRegistration.java @@ -15,4 +15,5 @@ import java.util.Map; public interface INodeRegistration { String registerNode(String baseUrl, Map nodeInfo, TranslationContext translationContext) throws Exception; + String unregisterNode(String nodeAddress, TranslationContext translationContext) throws Exception; } diff --git a/ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/helper/AbstractInstallationHelper.java b/ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/helper/AbstractInstallationHelper.java index 265a7c4..d0936ad 100644 --- a/ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/helper/AbstractInstallationHelper.java +++ b/ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/helper/AbstractInstallationHelper.java @@ -242,6 +242,21 @@ public abstract class AbstractInstallationHelper implements InitializingBean, Ap return instructionsSetList; } + public List prepareUninstallInstructionsForOs(NodeRegistryEntry entry) throws IOException { + if (! entry.getBaguetteServer().isServerRunning()) throw new RuntimeException("Baguette Server is not running"); + log.trace("AbstractInstallationHelper.prepareUninstallInstructionsForOs(): node-map={}", entry.getPreregistration()); + + String osFamily = entry.getPreregistration().get("operatingSystem"); + List instructionsSetList = null; + if (matchesOsFamily(osFamily, LINUX_OS_FAMILY)) + instructionsSetList = prepareUninstallInstructionsForLinux(entry); + else if (matchesOsFamily(osFamily, WINDOWS_OS_FAMILY)) + instructionsSetList = prepareUninstallInstructionsForWin(entry); + else + log.warn("AbstractInstallationHelper.prepareUninstallInstructionsForOs(): Unsupported OS family: {}", osFamily); + return instructionsSetList; + } + public boolean matchesOsFamily(@NonNull String lookup, @NonNull String osFamily) { lookup = lookup.trim().toUpperCase(); List familyList = properties.getOsFamilies().get(osFamily); diff --git a/ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/helper/InstallationHelper.java b/ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/helper/InstallationHelper.java index 420016a..8f4a868 100644 --- a/ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/helper/InstallationHelper.java +++ b/ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/helper/InstallationHelper.java @@ -25,8 +25,13 @@ public interface InstallationHelper { List prepareInstallationInstructionsForWin(NodeRegistryEntry entry); List prepareInstallationInstructionsForLinux(NodeRegistryEntry entry) throws IOException; + List prepareUninstallInstructionsForOs(NodeRegistryEntry entry) throws IOException; + List prepareUninstallInstructionsForWin(NodeRegistryEntry entry); + List prepareUninstallInstructionsForLinux(NodeRegistryEntry entry) throws IOException; + default ClientInstallationTask createClientInstallationTask(NodeRegistryEntry entry) throws Exception { return createClientInstallationTask(entry, null); } ClientInstallationTask createClientInstallationTask(NodeRegistryEntry entry, TranslationContext translationContext) throws Exception; + ClientInstallationTask createClientUninstallTask(NodeRegistryEntry entry, TranslationContext translationContext) throws Exception; } diff --git a/ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/helper/VmInstallationHelper.java b/ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/helper/VmInstallationHelper.java index 2f67860..4857738 100644 --- a/ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/helper/VmInstallationHelper.java +++ b/ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/helper/VmInstallationHelper.java @@ -22,6 +22,7 @@ import gr.iccs.imu.ems.baguette.server.NodeRegistryEntry; import gr.iccs.imu.ems.translate.TranslationContext; import gr.iccs.imu.ems.util.CredentialsMap; import gr.iccs.imu.ems.util.NetUtil; +import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.text.StringSubstitutor; @@ -60,7 +61,31 @@ public class VmInstallationHelper extends AbstractInstallationHelper { @Override public ClientInstallationTask createClientInstallationTask(NodeRegistryEntry entry, TranslationContext translationContext) throws IOException { - Map nodeMap = entry.getPreregistration(); + // Get EMS client installation instructions for VM node + List instructionsSetList = + prepareInstallationInstructionsForOs(entry); + + return createClientTask(ClientInstallationTask.TASK_TYPE.INSTALL, entry, translationContext, instructionsSetList); + } + + @Override + public ClientInstallationTask createClientUninstallTask(NodeRegistryEntry entry, TranslationContext translationContext) throws Exception { + // Clear any cached 'instruction-files' override (from a previous run) + entry.getPreregistration().remove("instruction-files"); + + // Get EMS client uninstall instructions for VM node + List instructionsSetList = + prepareUninstallInstructionsForOs(entry); + + return createClientTask(ClientInstallationTask.TASK_TYPE.UNINSTALL, entry, translationContext, instructionsSetList); + } + + private ClientInstallationTask createClientTask(@NonNull ClientInstallationTask.TASK_TYPE taskType, + NodeRegistryEntry entry, + TranslationContext translationContext, + List instructionsSetList) + { + Map nodeMap = initializeNodeMap(entry); String baseUrl = nodeMap.get("BASE_URL"); String clientId = nodeMap.get("CLIENT_ID"); @@ -95,13 +120,10 @@ public class VmInstallationHelper extends AbstractInstallationHelper { if (StringUtils.isEmpty(password) && StringUtils.isBlank(privateKey)) throw new IllegalArgumentException("Missing SSH password or private key for Node"); - // Get EMS client installation instructions for VM node - List instructionsSetList = - prepareInstallationInstructionsForOs(entry); - // Create Installation Task for VM node - ClientInstallationTask installationTask = ClientInstallationTask.builder() + ClientInstallationTask task = ClientInstallationTask.builder() .id(clientId) + .taskType(taskType) .nodeId(nodeId) .requestId(requestId) .name(nodeName) @@ -121,29 +143,21 @@ public class VmInstallationHelper extends AbstractInstallationHelper { .nodeRegistryEntry(entry) .translationContext(translationContext) .build(); - log.debug("VmInstallationHelper.createClientInstallationTask(): Created client installation task: {}", installationTask); - - return installationTask; + log.debug("VmInstallationHelper.createClientTask(): Created client task: {}", task); + return task; } - @Override - public List prepareInstallationInstructionsForWin(NodeRegistryEntry entry) { - log.warn("VmInstallationHelper.prepareInstallationInstructionsForWin(): NOT YET IMPLEMENTED"); - throw new IllegalArgumentException("VmInstallationHelper.prepareInstallationInstructionsForWin(): NOT YET IMPLEMENTED"); - } - - @Override - public List prepareInstallationInstructionsForLinux(NodeRegistryEntry entry) throws IOException { + private Map initializeNodeMap(NodeRegistryEntry entry) { Map nodeMap = entry.getPreregistration(); BaguetteServer baguette = entry.getBaguetteServer(); String baseUrl = StringUtils.removeEnd(nodeMap.get("BASE_URL"), "/"); String clientId = nodeMap.get("CLIENT_ID"); String ipSetting = nodeMap.get("IP_SETTING"); - log.debug("VmInstallationHelper.prepareInstallationInstructionsForLinux(): Invoked: base-url={}", baseUrl); + log.debug("VmInstallationHelper.initializeNodeMap(): Invoked: base-url={}", baseUrl); // Get parameters - log.trace("VmInstallationHelper.prepareInstallationInstructionsForLinux(): properties: {}", properties); + log.trace("VmInstallationHelper.initializeNodeMap(): properties: {}", properties); String rootCmd = properties.getRootCmd(); String baseDir = properties.getBaseDir(); String checkInstallationFile = properties.getCheckInstalledFile(); @@ -170,7 +184,7 @@ public class VmInstallationHelper extends AbstractInstallationHelper { : "NODE_" + e.getKey().toUpperCase(), Map.Entry::getValue, (v1, v2) -> { - log.warn("VmInstallationHelper.prepareInstallationInstructionsForLinux(): DUPLICATE KEY FOUND: key={}, old-value={}, new-value={}", + log.warn("VmInstallationHelper.initializeNodeMap(): DUPLICATE KEY FOUND: key={}, old-value={}, new-value={}", k, v1, v2); return v2; } @@ -182,12 +196,12 @@ public class VmInstallationHelper extends AbstractInstallationHelper { ? "NODE_SSH_" + k.substring(4).toUpperCase() : "NODE_" + k.toUpperCase(); if (additionalKeysMap.containsKey(k)) { - log.warn("VmInstallationHelper.prepareInstallationInstructionsForLinux(): DUPLICATE KEY FOUND: key={}, old-value={}, new-value={}", + log.warn("VmInstallationHelper.initializeNodeMap(): DUPLICATE KEY FOUND: key={}, old-value={}, new-value={}", k, additionalKeysMap.get(k), v); } additionalKeysMap.put(k, v); } catch (Exception ex) { - log.error("VmInstallationHelper.prepareInstallationInstructionsForLinux(): EXCEPTION in additional keys copy loop: key={}, value={}, additionalKeysMap={}, Exception:\n", + log.error("VmInstallationHelper.initializeNodeMap(): EXCEPTION in additional keys copy loop: key={}, value={}, additionalKeysMap={}, Exception:\n", k, v, additionalKeysMap, ex); } }); @@ -229,7 +243,7 @@ public class VmInstallationHelper extends AbstractInstallationHelper { nodeMap.putAll(clientInstallationProperties.getParameters()); nodeMap.put("EMS_PUBLIC_DIR", System.getProperty("PUBLIC_DIR", System.getenv("PUBLIC_DIR"))); - log.trace("VmInstallationHelper.prepareInstallationInstructionsForLinux: value-map: {}", nodeMap); + log.trace("VmInstallationHelper.initializeNodeMap: value-map: {}", nodeMap); /* // Clear EMS server certificate (PEM) file, if not secure if (!isServerSecure) { @@ -251,26 +265,65 @@ public class VmInstallationHelper extends AbstractInstallationHelper { } }*/ + return nodeMap; + } + + @Override + public List prepareInstallationInstructionsForWin(NodeRegistryEntry entry) { + log.warn("VmInstallationHelper.prepareInstallationInstructionsForWin(): NOT YET IMPLEMENTED"); + throw new IllegalArgumentException("VmInstallationHelper.prepareInstallationInstructionsForWin(): NOT YET IMPLEMENTED"); + } + + @Override + public List prepareUninstallInstructionsForWin(NodeRegistryEntry entry) { + log.warn("VmInstallationHelper.prepareUninstallInstructionsForWin(): NOT YET IMPLEMENTED"); + throw new IllegalArgumentException("VmInstallationHelper.prepareUninstallInstructionsForWin(): NOT YET IMPLEMENTED"); + } + + @Override + public List prepareInstallationInstructionsForLinux(NodeRegistryEntry entry) throws IOException { + return prepareInstructionsForLinux(entry, "LINUX"); + } + + @Override + public List prepareUninstallInstructionsForLinux(NodeRegistryEntry entry) throws IOException { + return prepareInstructionsForLinux(entry, "REMOVE_LINUX"); + } + + private List prepareInstructionsForLinux(@NonNull NodeRegistryEntry entry, String instructionsScenarioName) throws IOException { + Map nodeMap = entry.getPreregistration(); + List instructionsSetList = new ArrayList<>(); try { // Read installation instructions from JSON file - List instructionSetFileList = null; + List instructionSetFileList; if (nodeMap.containsKey("instruction-files")) { + log.trace("VmInstallationHelper.prepareInstructionsForLinux: FOUND instruction-files override: value={}", nodeMap.getOrDefault("instruction-files", null)); instructionSetFileList = Arrays.stream(nodeMap.getOrDefault("instruction-files", "").split(",")) .filter(StringUtils::isNotBlank) .map(String::trim) .collect(Collectors.toList()); + log.debug("VmInstallationHelper.prepareInstructionsForLinux: FOUND instruction-files override: list={}", instructionSetFileList); if (instructionSetFileList.isEmpty()) - log.warn("VmInstallationHelper.prepareInstallationInstructionsForLinux: Context map contains 'instruction-files' entry with no contents"); + log.warn("VmInstallationHelper.prepareInstructionsForLinux: Context map contains 'instruction-files' entry with no contents"); } else { - instructionSetFileList = properties.getInstructions().get("LINUX"); + log.trace("VmInstallationHelper.prepareInstructionsForLinux: NOT FOUND instruction-files override. Using configured instructions sets: instructionsScenarioName={}", instructionsScenarioName); + instructionSetFileList = properties.getInstructions().get(instructionsScenarioName); + log.trace("VmInstallationHelper.prepareInstructionsForLinux: NOT FOUND instruction-files override. Using configured instructions sets: list={}", instructionSetFileList); + if (instructionSetFileList==null || instructionSetFileList.isEmpty()) { + log.warn("VmInstallationHelper.prepareInstructionsForLinux: No instructions set files provided in configuration with name: {}", instructionsScenarioName); + instructionSetFileList = Collections.emptyList(); + } } + log.debug("VmInstallationHelper.prepareInstructionsForLinux: Instructions sets list: {}", instructionSetFileList); + + // Load instructions set from file for (String instructionSetFile : instructionSetFileList) { // Load instructions set from file - log.debug("VmInstallationHelper.prepareInstallationInstructionsForLinux: Installation instructions file for LINUX: {}", instructionSetFile); + log.debug("VmInstallationHelper.prepareInstructionsForLinux: Installation instructions file for LINUX: {}", instructionSetFile); InstructionsSet instructionsSet = InstructionsService.getInstance().loadInstructionsFile(instructionSetFile); - log.debug("VmInstallationHelper.prepareInstallationInstructionsForLinux: Instructions set loaded from file: {}\n{}", instructionSetFile, instructionsSet); + log.debug("VmInstallationHelper.prepareInstructionsForLinux: Instructions set loaded from file: {}\n{}", instructionSetFile, instructionsSet); // Pretty print instructionsSet JSON if (log.isTraceEnabled()) { @@ -279,7 +332,7 @@ public class VmInstallationHelper extends AbstractInstallationHelper { try (PrintWriter writer = new PrintWriter(stringWriter)) { gson.toJson(instructionsSet, writer); } - log.trace("VmInstallationHelper.prepareInstallationInstructionsForLinux: Installation instructions for LINUX: json:\n{}", stringWriter); + log.trace("VmInstallationHelper.prepareInstructionsForLinux: Installation instructions for LINUX: json:\n{}", stringWriter); } instructionsSetList.add(instructionsSet); @@ -287,7 +340,7 @@ public class VmInstallationHelper extends AbstractInstallationHelper { return instructionsSetList; } catch (Exception ex) { - log.error("VmInstallationHelper.prepareInstallationInstructionsForLinux: Exception while reading Installation instructions for LINUX: ", ex); + log.error("VmInstallationHelper.prepareInstructionsForLinux: Exception while reading Installation instructions for LINUX: ", ex); throw ex; } } diff --git a/ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/selfhealing/ClientRecoveryPlugin.java b/ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/selfhealing/ClientRecoveryPlugin.java index 25b2232..4cf113b 100644 --- a/ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/selfhealing/ClientRecoveryPlugin.java +++ b/ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/selfhealing/ClientRecoveryPlugin.java @@ -48,8 +48,9 @@ public class ClientRecoveryPlugin implements InitializingBean, EventBus.EventCon private long clientRecoveryDelay; private String recoveryInstructionsFile; - private final static String CLIENT_EXIT_TOPIC = "BAGUETTE_SERVER_CLIENT_EXITED"; - private final static String CLIENT_REGISTERED_TOPIC = "BAGUETTE_SERVER_CLIENT_REGISTERED"; + public final static String CLIENT_EXIT_TOPIC = "BAGUETTE_SERVER_CLIENT_EXITED"; + public final static String CLIENT_REGISTERED_TOPIC = "BAGUETTE_SERVER_CLIENT_REGISTERED"; + public final static String CLIENT_REMOVED_TOPIC = "BAGUETTE_SERVER_CLIENT_REMOVED"; @Override public void afterPropertiesSet() throws Exception { @@ -61,6 +62,8 @@ public class ClientRecoveryPlugin implements InitializingBean, EventBus.EventCon log.debug("ClientRecoveryPlugin: Subscribed for BAGUETTE_SERVER_CLIENT_EXITED events"); eventBus.subscribe(CLIENT_REGISTERED_TOPIC, this); log.debug("ClientRecoveryPlugin: Subscribed for BAGUETTE_SERVER_CLIENT_REGISTERED events"); + eventBus.subscribe(CLIENT_REMOVED_TOPIC, this); + log.debug("ClientRecoveryPlugin: Subscribed for CLIENT_REMOVED_TOPIC events"); log.trace("ClientRecoveryPlugin: clientInstallationProperties: {}", clientInstallationProperties); log.trace("ClientRecoveryPlugin: baguetteServer: {}", baguetteServer); @@ -80,18 +83,28 @@ public class ClientRecoveryPlugin implements InitializingBean, EventBus.EventCon } // Only process messages of ClientShellCommand type are accepted (sent by CSC instances) - if (! (message instanceof ClientShellCommand)) { - log.warn("ClientRecoveryPlugin: onMessage(): Message is not a {} object. Will ignore it.", ClientShellCommand.class.getSimpleName()); + if (! (message instanceof NodeRegistryEntry) && ! (message instanceof ClientShellCommand)) { + log.warn("ClientRecoveryPlugin: onMessage(): Message is neither a {} or a {} object. Will ignore it.", + NodeRegistryEntry.class.getSimpleName(), ClientShellCommand.class.getSimpleName()); return; } - // Get NodeRegistryEntry from ClientShellCommand passed with event - ClientShellCommand csc = (ClientShellCommand)message; - String clientId = csc.getId(); - String address = csc.getClientIpAddress(); - log.debug("ClientRecoveryPlugin: onMessage(): client-id={}, client-address={}", clientId, address); + NodeRegistryEntry nodeInfo; + String clientId; + String address; + if (message instanceof NodeRegistryEntry entry) { + nodeInfo = entry; + clientId = entry.getClientId(); + address = entry.getIpAddress(); + } else { + // Get NodeRegistryEntry from ClientShellCommand passed with event + ClientShellCommand csc = (ClientShellCommand) message; + clientId = csc.getId(); + address = csc.getClientIpAddress(); + log.debug("ClientRecoveryPlugin: onMessage(): client-id={}, client-address={}", clientId, address); - NodeRegistryEntry nodeInfo = csc.getNodeRegistryEntry(); //or = nodeRegistry.getNodeByAddress(address); + nodeInfo = csc.getNodeRegistryEntry(); //or = nodeRegistry.getNodeByAddress(address); + } log.debug("ClientRecoveryPlugin: onMessage(): client-node-info={}", nodeInfo); log.trace("ClientRecoveryPlugin: onMessage(): node-registry.node-addresses={}", nodeRegistry.getNodeAddresses()); log.trace("ClientRecoveryPlugin: onMessage(): node-registry.nodes={}", nodeRegistry.getNodes()); @@ -108,14 +121,25 @@ public class ClientRecoveryPlugin implements InitializingBean, EventBus.EventCon processExitEvent(nodeInfo); } if (CLIENT_REGISTERED_TOPIC.equals(topic)) { - log.debug("ClientRecoveryPlugin: onMessage(): CLIENT REGISTERED_TOPIC: message={}", message); + log.debug("ClientRecoveryPlugin: onMessage(): CLIENT REGISTERED: message={}", message); processRegisteredEvent(nodeInfo); } + if (CLIENT_REMOVED_TOPIC.equals(topic)) { + log.debug("ClientRecoveryPlugin: onMessage(): CLIENT REMOVED: message={}", message); + processRemovedEvent(nodeInfo); + } } private void processExitEvent(NodeRegistryEntry nodeInfo) { log.debug("ClientRecoveryPlugin: processExitEvent(): BEGIN: client-id={}, client-address={}", nodeInfo.getClientId(), nodeInfo.getIpAddress()); + // Check if node can be recovered (based on its Life-Cycle state) + if (! nodeInfo.canRecover()) { + log.warn("ClientRecoveryPlugin: processExitEvent(): Node will not be recovered because its state is {}: client-id={}, client-address={}", + nodeInfo.getState(), nodeInfo.getClientId(), nodeInfo.getIpAddress()); + return; + } + // Set node state to DOWN baguetteServer.getSelfHealingManager().setNodeSelfHealingState(nodeInfo, SelfHealingManager.NODE_STATE.DOWN); @@ -156,10 +180,29 @@ public class ClientRecoveryPlugin implements InitializingBean, EventBus.EventCon baguetteServer.getSelfHealingManager().setNodeSelfHealingState(nodeInfo, SelfHealingManager.NODE_STATE.UP); } + private void processRemovedEvent(NodeRegistryEntry nodeInfo) { + log.debug("ClientRecoveryPlugin: processRemovedEvent(): BEGIN: client-id={}, client-address={}", nodeInfo.getClientId(), nodeInfo.getIpAddress()); + + // Cancel any pending recovery task (for the node) + ScheduledFuture future = pendingTasks.remove(nodeInfo); + log.debug("ClientRecoveryPlugin: processRemovedEvent(): Recovery task: task={}, client-id={}, client-address={}", future, nodeInfo.getClientId(), nodeInfo.getIpAddress()); + if (future!=null && ! future.isDone() && ! future.isCancelled()) { + log.warn("ClientRecoveryPlugin: processRemovedEvent(): Cancelled recovery task: client-id={}, client-address={}", nodeInfo.getClientId(), nodeInfo.getIpAddress()); + future.cancel(false); + } + } + public void runClientRecovery(NodeRegistryEntry entry) throws Exception { log.debug("ClientRecoveryPlugin: runClientRecovery(): node-info={}", entry); if (entry==null) return; + if (! entry.canRecover()) { + log.info("ClientRecoveryPlugin: runClientRecovery(): Cannot recover node. Will not attempt recovery again: node-state={}, client-id={}, client-address={}", + entry.getState(), entry.getClientId(), entry.getIpAddress()); + pendingTasks.remove(entry); + return; + } + log.trace("ClientRecoveryPlugin: runClientRecovery(): recoveryInstructionsFile={}", recoveryInstructionsFile); entry.getPreregistration().put("instruction-files", recoveryInstructionsFile); diff --git a/ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/BaguetteServer.java b/ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/BaguetteServer.java index b3227df..9b39da6 100644 --- a/ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/BaguetteServer.java +++ b/ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/BaguetteServer.java @@ -550,4 +550,13 @@ public class BaguetteServer implements InitializingBean, EventBus.EventConsumer< } return clientId; } + + public NodeRegistryEntry unregisterClient(NodeRegistryEntry entry) throws UnknownHostException { + log.debug("BaguetteServer.unregisterClient(): entry={}", entry); + + // Archive node entry in node registry + nodeRegistry.archiveNode(entry); + + return entry; + } } diff --git a/ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/ClientShellCommand.java b/ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/ClientShellCommand.java index e873f28..e36664b 100644 --- a/ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/ClientShellCommand.java +++ b/ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/ClientShellCommand.java @@ -265,8 +265,16 @@ public class ClientShellCommand implements Command, Runnable, ServerSessionAware } } // Client connection closed - eventBus.send("BAGUETTE_SERVER_CLIENT_EXITING", this); - getNodeRegistryEntry().nodeExiting(null); + try { + eventBus.send("BAGUETTE_SERVER_CLIENT_EXITING", this); + NodeRegistryEntry entry = getNodeRegistryEntry(); + if (! entry.isArchived()) + entry.nodeExiting(null); + else + log.warn("{}==> Node is archived", id); + } catch (Exception e) { + log.warn("{}==> EXCEPTION: ", id, e); + } log.info("{}==> Signaling client to exit", id); out.println("EXIT"); diff --git a/ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/NodeRegistry.java b/ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/NodeRegistry.java index 154d780..05a7b8e 100644 --- a/ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/NodeRegistry.java +++ b/ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/NodeRegistry.java @@ -10,6 +10,7 @@ package gr.iccs.imu.ems.baguette.server; import lombok.Getter; +import lombok.NonNull; import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -17,9 +18,7 @@ import org.springframework.stereotype.Service; import java.net.InetAddress; import java.net.UnknownHostException; -import java.util.Collection; -import java.util.LinkedHashMap; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; /** @@ -29,6 +28,7 @@ import java.util.stream.Collectors; @Service public class NodeRegistry { private final Map registry = new LinkedHashMap<>(); + private final List archived = new LinkedList<>(); @Getter @Setter private ServerCoordinator coordinator; @@ -134,4 +134,27 @@ public class NodeRegistry { public Collection getNodeReferences() { return registry.values().stream().map(NodeRegistryEntry::getReference).collect(Collectors.toList()); } + + public synchronized void archiveNode(@NonNull NodeRegistryEntry entry) { + log.debug("NodeRegistry.archiveNode(): Archiving node: entry={}", entry); + + // Run checks + if (StringUtils.isBlank(entry.getIpAddress())) { + log.warn("NodeRegistry.archiveNode(): Node entry does not have an IP address. Ignoring archiving operation: entry={}", entry); + return; + } + if (! registry.containsKey(entry.getIpAddress())) { + log.warn("NodeRegistry.archiveNode(): Node with IP address {} not found in registry. Ignoring archiving operation: entry={}", entry.getIpAddress(), entry); + return; + } + if (registry.get(entry.getIpAddress())!=entry) { + log.warn("NodeRegistry.archiveNode(): Node passed does not match with node registered to IP address {}. Ignoring archiving operation: entry={}", entry.getIpAddress(), entry); + return; + } + + // Archive node + archived.add(entry); + registry.remove(entry.getIpAddress()); + entry.nodeArchived(null); + } } diff --git a/ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/NodeRegistryEntry.java b/ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/NodeRegistryEntry.java index 1248050..b033514 100644 --- a/ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/NodeRegistryEntry.java +++ b/ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/NodeRegistryEntry.java @@ -21,8 +21,12 @@ import java.util.*; @AllArgsConstructor @RequiredArgsConstructor public class NodeRegistryEntry { - public enum STATE { PREREGISTERED, IGNORE_NODE, INSTALLING, NOT_INSTALLED, INSTALLED, INSTALL_ERROR, - WAITING_REGISTRATION, REGISTERING, REGISTERED, REGISTRATION_ERROR, DISCONNECTED, EXITING, EXITED, NODE_FAILED + public enum STATE { + PREREGISTERED, IGNORE_NODE, + INSTALLING, NOT_INSTALLED, INSTALLED, INSTALL_ERROR, + WAITING_REGISTRATION, REGISTERING, REGISTERED, REGISTRATION_ERROR, + DISCONNECTED, EXITING, EXITED, NODE_FAILED, + REMOVING, REMOVED, REMOVE_ERROR, ARCHIVED }; @Getter private final String ipAddress; @Getter private final String clientId; @@ -40,6 +44,8 @@ public class NodeRegistryEntry { @JsonIgnore @Getter private transient Map registration = new LinkedHashMap<>(); @JsonIgnore + @Getter private transient Map removal = new LinkedHashMap<>(); + @JsonIgnore @Getter @Setter private transient IClusterZone clusterZone; @JsonIgnore @@ -70,100 +76,118 @@ public class NodeRegistryEntry { public void refreshReference() { reference = UUID.randomUUID().toString(); } - public NodeRegistryEntry nodePreregistration(Map nodeInfo) { - preregistration.clear(); - preregistration.putAll(StrUtil.deepFlattenMap(nodeInfo)); - setState(STATE.PREREGISTERED); + public boolean canRecover() { + return state != null && switch (state) { + case PREREGISTERED, IGNORE_NODE -> false; + case REMOVING, REMOVED, REMOVE_ERROR, ARCHIVED -> false; + default -> true; + }; + } + + public boolean isArchived() { + return state!=null && state==STATE.ARCHIVED; + } + + public boolean canChangeStateTo(@NonNull STATE newState) { + return ! isArchived(); + } + + private void _canUpdateEntry(@NonNull STATE newState) { + if (! canChangeStateTo(newState)) { + throw new IllegalStateException(String.format("Cannot change NodeRegistryEntry state from %s to %s: client-id=%s, client-address=%s", + state, newState, clientId, ipAddress)); + } + } + + private NodeRegistryEntry _updateEntry(@NonNull STATE newState, @NonNull Map map, boolean clear, Map nodeInfo) { + _canUpdateEntry(newState); + if (clear) map.clear(); + map.putAll(StrUtil.deepFlattenMap(nodeInfo)); + setState(newState); return this; } - public NodeRegistryEntry nodeIgnore(Object nodeInfo) { - installation.clear(); - installation.put("ignore-node", nodeInfo!=null ? nodeInfo.toString() : null); + private NodeRegistryEntry _updateEntry(@NonNull STATE newState, @NonNull Map map, boolean clear, @NonNull String key, Object val, String defVal) { + _canUpdateEntry(newState); + if (clear) map.clear(); + map.put(key, val!=null ? val.toString() : defVal); setState(STATE.IGNORE_NODE); return this; } + public NodeRegistryEntry nodePreregistration(Map nodeInfo) { + return _updateEntry(STATE.PREREGISTERED, preregistration, true, nodeInfo); + } + + public NodeRegistryEntry nodeIgnore(Object nodeInfo) { + return _updateEntry(STATE.IGNORE_NODE, installation, true, "ignore-node", nodeInfo, null); + } + public NodeRegistryEntry nodeInstalling(Object nodeInfo) { - installation.clear(); - installation.put("installation-task", nodeInfo!=null ? nodeInfo.toString() : "INSTALLING"); - setState(STATE.INSTALLING); - return this; + return _updateEntry(STATE.INSTALLING, installation, true, "installation-task", nodeInfo, "INSTALLING"); } public NodeRegistryEntry nodeNotInstalled(Object nodeInfo) { - installation.clear(); - installation.put("installation-task-result", nodeInfo!=null ? nodeInfo.toString() : "NOT_INSTALLED"); - setState(STATE.NOT_INSTALLED); - return this; + return _updateEntry(STATE.NOT_INSTALLED, installation, true, "installation-task-result", nodeInfo, "NOT_INSTALLED"); } public NodeRegistryEntry nodeInstallationComplete(Object nodeInfo) { - installation.put("installation-task-result", nodeInfo!=null ? nodeInfo.toString() : "SUCCESS"); - setState(STATE.INSTALLED); - return this; + return _updateEntry(STATE.INSTALLED, installation, false, "installation-task-result", nodeInfo, "SUCCESS"); } public NodeRegistryEntry nodeInstallationError(Object nodeInfo) { - installation.put("installation-task-result", nodeInfo!=null ? nodeInfo.toString() : "ERROR"); - setState(STATE.INSTALL_ERROR); - return this; + return _updateEntry(STATE.INSTALL_ERROR, installation, false, "installation-task-result", nodeInfo, "ERROR"); } public NodeRegistryEntry nodeRegistering(Map nodeInfo) { - registration.clear(); - registration.putAll(StrUtil.deepFlattenMap(nodeInfo)); - setState(STATE.REGISTERING); - return this; + return _updateEntry(STATE.REGISTERING, registration, true, nodeInfo); } public NodeRegistryEntry nodeRegistered(Map nodeInfo) { - //registration.clear(); - registration.putAll(StrUtil.deepFlattenMap(nodeInfo)); - setState(STATE.REGISTERED); - return this; + return _updateEntry(STATE.REGISTERED, registration, false, nodeInfo); } public NodeRegistryEntry nodeRegistrationError(Map nodeInfo) { - registration.putAll(StrUtil.deepFlattenMap(nodeInfo)); - setState(STATE.REGISTRATION_ERROR); - return this; + return _updateEntry(STATE.REGISTRATION_ERROR, registration, false, nodeInfo); } public NodeRegistryEntry nodeRegistrationError(Throwable t) { - registration.putAll(StrUtil.deepFlattenMap(Collections.singletonMap("exception", t))); - setState(STATE.REGISTRATION_ERROR); - return this; + return _updateEntry(STATE.REGISTRATION_ERROR, registration, false, Collections.singletonMap("exception", t)); } public NodeRegistryEntry nodeDisconnected(Map nodeInfo) { - registration.putAll(StrUtil.deepFlattenMap(nodeInfo)); - setState(STATE.DISCONNECTED); - return this; + return _updateEntry(STATE.DISCONNECTED, registration, false, nodeInfo); } public NodeRegistryEntry nodeDisconnected(Throwable t) { - registration.putAll(StrUtil.deepFlattenMap(Collections.singletonMap("exception", t))); - setState(STATE.DISCONNECTED); - return this; + return _updateEntry(STATE.DISCONNECTED, registration, false, Collections.singletonMap("exception", t)); } public NodeRegistryEntry nodeExiting(Map nodeInfo) { - registration.putAll(StrUtil.deepFlattenMap(nodeInfo)); - setState(STATE.EXITING); - return this; + return _updateEntry(STATE.EXITING, registration, false, nodeInfo); } public NodeRegistryEntry nodeExited(Map nodeInfo) { - registration.putAll(StrUtil.deepFlattenMap(nodeInfo)); - setState(STATE.EXITED); - return this; + return _updateEntry(STATE.EXITED, registration, false, nodeInfo); } public NodeRegistryEntry nodeFailed(Map failInfo) { - if (failInfo!=null) - registration.putAll(StrUtil.deepFlattenMap(failInfo)); - setState(STATE.NODE_FAILED); - return this; + return _updateEntry(STATE.NODE_FAILED, registration, false, failInfo); + } + + public NodeRegistryEntry nodeRemoving(Map nodeInfo) { + return _updateEntry(STATE.REMOVING, removal, false, nodeInfo); + } + + public NodeRegistryEntry nodeRemoved(Map nodeInfo) { + return _updateEntry(STATE.REMOVED, removal, false, nodeInfo); + } + + public NodeRegistryEntry nodeRemoveError(Map failInfo) { + return _updateEntry(STATE.REMOVE_ERROR, removal, false, failInfo); + } + + public NodeRegistryEntry nodeArchived(Map nodeInfo) { + return _updateEntry(STATE.ARCHIVED, removal, false, nodeInfo); } } diff --git a/ems-core/control-service/src/main/java/gr/iccs/imu/ems/control/controller/NodeRegistrationController.java b/ems-core/control-service/src/main/java/gr/iccs/imu/ems/control/controller/NodeRegistrationController.java index 5be2e57..c98a7a3 100644 --- a/ems-core/control-service/src/main/java/gr/iccs/imu/ems/control/controller/NodeRegistrationController.java +++ b/ems-core/control-service/src/main/java/gr/iccs/imu/ems/control/controller/NodeRegistrationController.java @@ -53,7 +53,7 @@ public class NodeRegistrationController { return "OK"; } - @RequestMapping(value = "/baguette/registerNode", method = POST, + @RequestMapping(value = { "/baguette/registerNode", "/baguette/node/register" }, method = POST, consumes = MediaType.APPLICATION_JSON_VALUE) public String baguetteRegisterNode(@RequestBody String jsonNode, HttpServletRequest request) throws Exception { log.info("NodeRegistrationController.baguetteRegisterNode(): Invoked"); @@ -80,6 +80,26 @@ public class NodeRegistrationController { return response; } + @RequestMapping(value = "/baguette/node/unregister/{ipAddress:.+}", method = {GET, POST}, + produces = MediaType.TEXT_PLAIN_VALUE) + public String baguetteUnregisterNode(@PathVariable String ipAddress, HttpServletRequest request) throws Exception { + log.info("NodeRegistrationController.baguetteUnregisterNode(): Invoked"); + log.debug("NodeRegistrationController.baguetteUnregisterNode(): Node IP address:\n{}", ipAddress); + + String response; + try { + response = nodeRegistrationCoordinator.unregisterNode(ipAddress, + coordinator.getTranslationContextOfAppModel(coordinator.getCurrentAppModelId())); + } catch (Exception e) { + log.error("NodeRegistrationController.baguetteUnregisterNode(): EXCEPTION while unregistering node: address={}\n", ipAddress, e); + response = "ERROR "+e.getMessage(); + } + + log.info("NodeRegistrationController.baguetteUnregisterNode(): Node unregistered: node-address={}", ipAddress); + log.debug("NodeRegistrationController.baguetteUnregisterNode(): address={}, json={}", ipAddress, response); + return response; + } + @RequestMapping(value = "/baguette/node/list", method = GET) public Collection baguetteNodeList() throws Exception { log.info("NodeRegistrationController.baguetteNodeList(): Invoked"); diff --git a/ems-core/control-service/src/main/java/gr/iccs/imu/ems/control/controller/NodeRegistrationCoordinator.java b/ems-core/control-service/src/main/java/gr/iccs/imu/ems/control/controller/NodeRegistrationCoordinator.java index 90bb73f..bf5cdad 100644 --- a/ems-core/control-service/src/main/java/gr/iccs/imu/ems/control/controller/NodeRegistrationCoordinator.java +++ b/ems-core/control-service/src/main/java/gr/iccs/imu/ems/control/controller/NodeRegistrationCoordinator.java @@ -12,15 +12,19 @@ package gr.iccs.imu.ems.control.controller; import gr.iccs.imu.ems.baguette.client.install.ClientInstallationTask; import gr.iccs.imu.ems.baguette.client.install.ClientInstaller; import gr.iccs.imu.ems.baguette.client.install.helper.InstallationHelperFactory; +import gr.iccs.imu.ems.baguette.client.selfhealing.ClientRecoveryPlugin; import gr.iccs.imu.ems.baguette.server.BaguetteServer; +import gr.iccs.imu.ems.baguette.server.ClientShellCommand; import gr.iccs.imu.ems.baguette.server.NodeRegistryEntry; import gr.iccs.imu.ems.control.properties.ControlServiceProperties; import gr.iccs.imu.ems.control.properties.StaticResourceProperties; import gr.iccs.imu.ems.translate.TranslationContext; +import gr.iccs.imu.ems.util.EventBus; import gr.iccs.imu.ems.util.NetUtil; import gr.iccs.imu.ems.util.StrUtil; import jakarta.servlet.http.HttpServletRequest; import lombok.Getter; +import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -35,6 +39,7 @@ import org.springframework.web.servlet.support.ServletUriComponentsBuilder; import java.io.IOException; import java.util.Collections; import java.util.Map; +import java.util.concurrent.Callable; import java.util.concurrent.atomic.AtomicBoolean; @Slf4j @@ -51,6 +56,8 @@ public class NodeRegistrationCoordinator implements InitializingBean { private final ServerProperties serverProperties; private final ServletWebServerApplicationContext webServerAppCtxt; + private final EventBus eventBus; + @Override public void afterPropertiesSet() throws Exception { } @@ -130,6 +137,60 @@ public class NodeRegistrationCoordinator implements InitializingBean { return response; } + public String unregisterNode(@NonNull String ipAddress, TranslationContext translationContext) throws Exception { + log.debug("NodeRegistrationCoordinator.unregisterNode(): BEGIN: ip-address={}", ipAddress); + if (StringUtils.isBlank(ipAddress)) { + log.error("NodeRegistrationCoordinator.unregisterNode(): Blank IP address provided"); + throw new IllegalArgumentException("Blank IP address provided"); + } + + // Retrieve node entry from registry + NodeRegistryEntry entry = baguetteServer.getNodeRegistry().getNodeByAddress(ipAddress); + if (entry==null) { + log.error("NodeRegistrationCoordinator.unregisterNode(): Node not found in registry: address={}", ipAddress); + throw new IllegalArgumentException("Node not found in registry: address="+ipAddress); + } + + // Set node state to REMOVING in order to prevent self-healing + entry.nodeRemoving(null); + + // Signal self-healing plugin to cancel any pending recovery tasks + log.debug("NodeRegistrationCoordinator.unregisterNode(): Notifying Self-healing to remove any pending recovery tasks: address={}", ipAddress); + eventBus.send(ClientRecoveryPlugin.CLIENT_REMOVED_TOPIC, entry); + + // Close CSC connection + ClientShellCommand csc = ClientShellCommand.getActiveByIpAddress(entry.getIpAddress()); + if (csc!=null) { + log.info("NodeRegistrationCoordinator.unregisterNode(): Closing connection to EMS client: address={}", ipAddress); + csc.stop("REMOVING NODE"); + } else + log.warn("NodeRegistrationCoordinator.unregisterNode(): CSC is null. Cannot close connection to EMS client. Probably connection has already closed: address={}", ipAddress); + + // Continue processing according to ExecutionWare type + String response; + log.info("NodeRegistrationCoordinator.unregisterNode(): ExecutionWare: {}", properties.getExecutionware()); + if (properties.getExecutionware()==ControlServiceProperties.ExecutionWare.CLOUDIATOR) { + response = "NOT SUPPORTED"; + } else { + response = createClientUninstallTask(entry, translationContext, () -> { + // Unregister and Archive node + try { + entry.nodeRemoved(null); + baguetteServer.unregisterClient(entry); + return "OK"; + } catch (Exception e) { + log.error("NodeRegistrationCoordinator.unregisterNode(): EXCEPTION while unregistering node: address={}, entry={}\n", ipAddress, entry, e); + entry.nodeRemoveError(Map.of( + "error", e.getMessage() + )); + return "ERROR " + e.getMessage(); + } + }); + } + + return response; + } + void updateRegistrationInfo(Map nodeMap, String baseUrl) { // Set OS info String os = StringUtils.isNotBlank(nodeMap.get("operatingSystem.name")) @@ -211,14 +272,32 @@ public class NodeRegistrationCoordinator implements InitializingBean { } public String createClientInstallationTask(NodeRegistryEntry entry, TranslationContext translationContext) throws Exception { - //log.info("ControlServiceController.baguetteRegisterNodeForProactive(): INPUT: node-map: {}", nodeMap); + return createClientInstallationTask(entry, translationContext, null); + } + public String createClientInstallationTask(NodeRegistryEntry entry, TranslationContext translationContext, Callable callback) throws Exception { ClientInstallationTask installationTask = InstallationHelperFactory.getInstance() .createInstallationHelper(entry) .createClientInstallationTask(entry, translationContext); + installationTask.setCallback(callback); ClientInstaller.instance().addTask(installationTask); log.debug("NodeRegistrationCoordinator.createClientInstallationTask(): New installation-task: {}", installationTask); return "OK"; } + + public String createClientUninstallTask(NodeRegistryEntry entry, TranslationContext translationContext) throws Exception { + return createClientUninstallTask(entry, translationContext, null); + } + + public String createClientUninstallTask(NodeRegistryEntry entry, TranslationContext translationContext, Callable callback) throws Exception { + ClientInstallationTask uninstallTask = InstallationHelperFactory.getInstance() + .createInstallationHelper(entry) + .createClientUninstallTask(entry, translationContext); + uninstallTask.setCallback(callback); + ClientInstaller.instance().addTask(uninstallTask); + log.debug("NodeRegistrationCoordinator.createClientUninstallTask(): New uninstall-task: {}", uninstallTask); + + return "OK"; + } } diff --git a/ems-core/control-service/src/main/java/gr/iccs/imu/ems/control/util/NodeRegistrationHelper.java b/ems-core/control-service/src/main/java/gr/iccs/imu/ems/control/util/NodeRegistrationHelper.java index 7d73301..fe7d7fa 100644 --- a/ems-core/control-service/src/main/java/gr/iccs/imu/ems/control/util/NodeRegistrationHelper.java +++ b/ems-core/control-service/src/main/java/gr/iccs/imu/ems/control/util/NodeRegistrationHelper.java @@ -36,4 +36,11 @@ public class NodeRegistrationHelper implements InitializingBean, INodeRegistrati baseUrl, nodeInfo, translationContext); return nodeRegistrationCoordinator.registerNode(baseUrl, nodeInfo, translationContext); } + + @Override + public String unregisterNode(String nodeAddress, TranslationContext translationContext) throws Exception { + log.debug("NodeRegistrationHelper: Invoking unregisterNode: node-address={}, TC={}", + nodeAddress, translationContext); + return nodeRegistrationCoordinator.unregisterNode(nodeAddress, translationContext); + } }