EMS: Implementing node removal (off-boarding) [WIP]

This commit is contained in:
ipatini 2023-10-05 18:24:34 +03:00
parent 3083d7dad3
commit 271ae82855
15 changed files with 494 additions and 127 deletions

@ -130,6 +130,7 @@ public class ClientInstallationRequestListener implements InitializingBean {
switch (requestType) { switch (requestType) {
case "DIAGNOSTICS" -> processDiagnosticsRequest(request); case "DIAGNOSTICS" -> processDiagnosticsRequest(request);
case "VM" -> processOnboardingRequest(request); case "VM" -> processOnboardingRequest(request);
case "REMOVE" -> processRemoveRequest(request);
default -> throw new IllegalArgumentException("Unsupported request type: "+requestType); default -> throw new IllegalArgumentException("Unsupported request type: "+requestType);
}; };
@ -174,6 +175,7 @@ public class ClientInstallationRequestListener implements InitializingBean {
// Create client installation task // Create client installation task
ClientInstallationTask newTask = ClientInstallationTask.builder() ClientInstallationTask newTask = ClientInstallationTask.builder()
.id(request.get("requestId")) .id(request.get("requestId"))
.taskType(ClientInstallationTask.TASK_TYPE.DIAGNOSTIC)
.requestId(request.get("requestId")) .requestId(request.get("requestId"))
.type(request.get("requestType")) .type(request.get("requestType"))
.nodeId(request.get("deviceId")) .nodeId(request.get("deviceId"))
@ -265,4 +267,26 @@ public class ClientInstallationRequestListener implements InitializingBean {
log.trace("InstallationEventListener.convertToNodeInfoMap(): END: nodeMap: {}", nodeMap); log.trace("InstallationEventListener.convertToNodeInfoMap(): END: nodeMap: {}", nodeMap);
return nodeMap; return nodeMap;
} }
private void processRemoveRequest(Map<String,String> request) throws Exception {
String requestId = request.getOrDefault("requestId", "").trim();
String nodeAddress = request.getOrDefault("deviceIpAddress", "").trim();
log.info("InstallationEventListener: New node REMOVE request with Id: {}, address={}", requestId, nodeAddress);
if (StringUtils.isBlank(requestId)) {
clientInstaller.sendErrorClientInstallationReport("MISSING-REQUEST-ID", "INVALID REQUEST. MISSING REQUEST ID");
return;
}
if (StringUtils.isBlank(nodeAddress)) {
clientInstaller.sendErrorClientInstallationReport(requestId, "INVALID REQUEST. MISSING IP ADDRESS");
return;
}
try {
log.debug("InstallationEventListener: Off-boarding node due to REMOVE request with Id: {}", requestId);
nodeRegistration.unregisterNode(nodeAddress, new TranslationContext(requestId));
} catch (Exception e) {
log.warn("InstallationEventListener: EXCEPTION while executing REMOVE request with Id: {}\n", requestId, e);
clientInstaller.sendErrorClientInstallationReport(requestId, "ERROR: "+e.getMessage());
}
}
} }

@ -16,6 +16,7 @@ import lombok.Builder;
import lombok.Data; import lombok.Data;
import java.util.List; import java.util.List;
import java.util.concurrent.Callable;
/** /**
* Client installation task * Client installation task
@ -23,17 +24,22 @@ import java.util.List;
@Data @Data
@Builder @Builder
public class ClientInstallationTask { public class ClientInstallationTask {
public enum TASK_TYPE { INSTALL, UNINSTALL, DIAGNOSTIC, OTHER }
private final String id; private final String id;
private final TASK_TYPE taskType;
private final String nodeId; private final String nodeId;
private final String requestId; private final String requestId;
private final String name; private final String name;
private final String os; private final String os;
private final String address; private final String address;
private final String type; private final String type; // Node type (VM, baremetal etc)
private final String provider; private final String provider;
private final SshConfig ssh; private final SshConfig ssh;
private final NodeRegistryEntry nodeRegistryEntry; private final NodeRegistryEntry nodeRegistryEntry;
private final List<InstructionsSet> instructionSets; private final List<InstructionsSet> instructionSets;
private final TranslationContext translationContext; private final TranslationContext translationContext;
private boolean nodeMustBeInRegistry = true; private boolean nodeMustBeInRegistry = true;
private Callable<String> callback;
} }

@ -10,6 +10,7 @@
package gr.iccs.imu.ems.baguette.client.install; package gr.iccs.imu.ems.baguette.client.install;
import gr.iccs.imu.ems.baguette.server.BaguetteServer; import gr.iccs.imu.ems.baguette.server.BaguetteServer;
import gr.iccs.imu.ems.baguette.server.ClientShellCommand;
import gr.iccs.imu.ems.baguette.server.NodeRegistryEntry; import gr.iccs.imu.ems.baguette.server.NodeRegistryEntry;
import gr.iccs.imu.ems.brokercep.BrokerCepService; import gr.iccs.imu.ems.brokercep.BrokerCepService;
import gr.iccs.imu.ems.common.plugin.PluginManager; import gr.iccs.imu.ems.common.plugin.PluginManager;
@ -69,26 +70,49 @@ public class ClientInstaller implements InitializingBean {
executorService.submit(() -> { executorService.submit(() -> {
long taskCnt = taskCounter.getAndIncrement(); long taskCnt = taskCounter.getAndIncrement();
String resultStr = ""; String resultStr = "";
String callbackStr = "";
String errorStr = ""; String errorStr = "";
// Execute task // Execute task
boolean result = false;
try { try {
log.info("ClientInstaller: Executing Client installation Task #{}: task-id={}, node-id={}, name={}, type={}, address={}", log.info("ClientInstaller: Executing Client installation Task #{}: task-id={}, node-id={}, name={}, type={}, address={}",
taskCnt, task.getId(), task.getNodeId(), task.getName(), task.getType(), task.getAddress()); taskCnt, task.getId(), task.getNodeId(), task.getName(), task.getType(), task.getAddress());
long startTm = System.currentTimeMillis(); long startTm = System.currentTimeMillis();
boolean result = executeTask(task, taskCnt); result = executeTask(task, taskCnt);
long endTm = System.currentTimeMillis(); long endTm = System.currentTimeMillis();
resultStr = result ? "SUCCESS" : "FAILED"; resultStr = result ? "SUCCESS" : "FAILED";
log.info("ClientInstaller: Client installation Task #{}: result={}, duration={}ms", log.info("ClientInstaller: Client installation Task #{}: result={}, duration={}ms",
taskCnt, resultStr, endTm - startTm); taskCnt, resultStr, endTm - startTm);
} catch (Throwable t) { } catch (Throwable t) {
log.info("ClientInstaller: Exception caught in Client installation Task #{}: Exception: ", taskCnt, t); log.error("ClientInstaller: Exception caught in Client installation Task #{}: Exception: ", taskCnt, t);
errorStr = t.getMessage(); errorStr = "EXCEPTION " + t.getMessage();
}
// Run callback (if any)
if (result && task.getCallback()!=null) {
try {
log.debug("ClientInstaller: CALLBACK started: Task #{}: task-id={}", taskCnt, task.getId());
long startTm = System.currentTimeMillis();
callbackStr = task.getCallback().call();
long endTm = System.currentTimeMillis();
log.info("ClientInstaller: CALLBACK completed: Task #{}: callback-result={}, duration={}ms", taskCnt, callbackStr, endTm - startTm);
if (! "OK".equalsIgnoreCase(callbackStr)) resultStr = "FAILED";
} catch (Throwable t) {
log.error("ClientInstaller: CALLBACK: Exception caught while running callback of Client installation Task #{}: Exception: ", taskCnt, t);
callbackStr = "CALLBACK-EXCEPTION " + t.getMessage();
resultStr = "FAILED";
}
} else {
if (result)
log.debug("ClientInstaller: No CALLBACK found for Task #{}", taskCnt);
else
log.debug("ClientInstaller: Skipped CALLBACK because execution failed for Task #{}", taskCnt);
} }
// Send execution report to local broker // Send execution report to local broker
try { try {
resultStr = StringUtils.defaultIfBlank(resultStr, "ERROR: " + errorStr); resultStr = StringUtils.defaultIfBlank(resultStr, "ERROR: " + errorStr + " " + callbackStr);
sendSuccessClientInstallationReport(taskCnt, task, resultStr); sendSuccessClientInstallationReport(taskCnt, task, resultStr);
} catch (Throwable t) { } catch (Throwable t) {
log.info("ClientInstaller: EXCEPTION while sending Client installation report for Task #{}: Exception: ", taskCnt, t); log.info("ClientInstaller: EXCEPTION while sending Client installation report for Task #{}: Exception: ", taskCnt, t);
@ -103,7 +127,8 @@ public class ClientInstaller implements InitializingBean {
return executeVmOrBaremetalTask(task, taskCounter); return executeVmOrBaremetalTask(task, taskCounter);
} else } else
if ("DIAGNOSTICS".equalsIgnoreCase(task.getType())) { //if ("DIAGNOSTICS".equalsIgnoreCase(task.getType())) {
if (task.getTaskType()==ClientInstallationTask.TASK_TYPE.DIAGNOSTIC) {
return executeDiagnosticsTask(task, taskCounter); return executeDiagnosticsTask(task, taskCounter);
} else { } else {
log.error("ClientInstaller: UNSUPPORTED TASK TYPE: {}", task.getType()); log.error("ClientInstaller: UNSUPPORTED TASK TYPE: {}", task.getType());
@ -121,30 +146,55 @@ public class ClientInstaller implements InitializingBean {
if (! task.isNodeMustBeInRegistry()) if (! task.isNodeMustBeInRegistry())
entry = task.getNodeRegistryEntry(); entry = task.getNodeRegistryEntry();
entry.nodeInstalling(task); // Check if node is being removed or have been archived
if (task.getNodeRegistryEntry().isArchived()) {
// Call InstallationContextPlugin's before installation log.warn("ClientInstaller: Node is being removed or has been archived: {}", properties.getInstallationContextProcessorPlugins());
log.debug("ClientInstaller: PRE-INSTALLATION: Calling installation context processors: {}", properties.getInstallationContextProcessorPlugins()); throw new IllegalStateException("Node is being removed or has been archived: Node IP address: "+ task.getAddress());
pluginManager.getActivePlugins(InstallationContextProcessorPlugin.class)
.forEach(plugin->((InstallationContextProcessorPlugin)plugin).processBeforeInstallation(task, taskCounter));
log.debug("ClientInstaller: INSTALLATION: Executing installation task: task-counter={}, task={}", taskCounter, task);
boolean success = executeVmTask(task, taskCounter);
log.debug("ClientInstaller: NODE_REGISTRY_ENTRY after installation execution: \n{}", task.getNodeRegistryEntry());
if (entry.getState()==NodeRegistryEntry.STATE.INSTALLING) {
log.warn("ClientInstaller: NODE_REGISTRY_ENTRY status is still INSTALLING after executing client installation. Changing to INSTALL_ERROR");
entry.nodeInstallationError(null);
} }
// Call InstallationContextPlugin's after installation boolean success;
log.debug("ClientInstaller: POST-INSTALLATION: Calling installation context processors: {}", properties.getInstallationContextProcessorPlugins()); if (! task.getInstructionSets().isEmpty()) {
pluginManager.getActivePlugins(InstallationContextProcessorPlugin.class) // Starting installation
.forEach(plugin->((InstallationContextProcessorPlugin)plugin).processAfterInstallation(task, taskCounter, success)); entry.nodeInstalling(task);
// Call InstallationContextPlugin's before installation
log.debug("ClientInstaller: PRE-INSTALLATION: Calling installation context processors: {}", properties.getInstallationContextProcessorPlugins());
pluginManager.getActivePlugins(InstallationContextProcessorPlugin.class)
.forEach(plugin -> ((InstallationContextProcessorPlugin) plugin).processBeforeInstallation(task, taskCounter));
log.debug("ClientInstaller: INSTALLATION: Executing installation task: task-counter={}, task={}", taskCounter, task);
success = executeVmTask(task, taskCounter);
log.debug("ClientInstaller: NODE_REGISTRY_ENTRY after installation execution: \n{}", task.getNodeRegistryEntry());
if (entry.getState() == NodeRegistryEntry.STATE.INSTALLING) {
log.warn("ClientInstaller: NODE_REGISTRY_ENTRY status is still INSTALLING after executing client installation. Changing to INSTALL_ERROR");
entry.nodeInstallationError(null);
}
// Call InstallationContextPlugin's after installation
log.debug("ClientInstaller: POST-INSTALLATION: Calling installation context processors: {}", properties.getInstallationContextProcessorPlugins());
pluginManager.getActivePlugins(InstallationContextProcessorPlugin.class)
.forEach(plugin -> ((InstallationContextProcessorPlugin) plugin).processAfterInstallation(task, taskCounter, success));
} else {
log.debug("ClientInstaller: SKIP INSTALLATION: Task has no instructions sets. Skipping execution: Node IP address: "+ task.getAddress());
success = true;
}
// Pre-register Node to baguette Server Coordinator // Pre-register Node to baguette Server Coordinator
log.debug("ClientInstaller: POST-INSTALLATION: Node is being pre-registered: {}", entry); if (task.getTaskType()==ClientInstallationTask.TASK_TYPE.INSTALL) {
baguetteServer.getNodeRegistry().getCoordinator().preregister(entry); log.debug("ClientInstaller: POST-INSTALLATION: Node is being pre-registered: {}", entry);
baguetteServer.getNodeRegistry().getCoordinator().preregister(entry);
}
// Un-register Node from baguette Server Coordinator
if (task.getTaskType()==ClientInstallationTask.TASK_TYPE.UNINSTALL) {
ClientShellCommand csc = ClientShellCommand.getActiveByIpAddress(entry.getIpAddress());
log.debug("ClientInstaller: POST-INSTALLATION: CSC of node to be unregistered: {}", csc);
if (csc!=null) {
log.debug("ClientInstaller: POST-INSTALLATION: Node is going to be unregistered: {}", entry);
baguetteServer.getNodeRegistry().getCoordinator().unregister(csc);
}
}
log.debug("ClientInstaller: Installation outcome: {}", success ? "Success" : "Error"); log.debug("ClientInstaller: Installation outcome: {}", success ? "Success" : "Error");
return success; return success;

@ -15,4 +15,5 @@ import java.util.Map;
public interface INodeRegistration { public interface INodeRegistration {
String registerNode(String baseUrl, Map<String,Object> nodeInfo, TranslationContext translationContext) throws Exception; String registerNode(String baseUrl, Map<String,Object> nodeInfo, TranslationContext translationContext) throws Exception;
String unregisterNode(String nodeAddress, TranslationContext translationContext) throws Exception;
} }

@ -242,6 +242,21 @@ public abstract class AbstractInstallationHelper implements InitializingBean, Ap
return instructionsSetList; return instructionsSetList;
} }
public List<InstructionsSet> prepareUninstallInstructionsForOs(NodeRegistryEntry entry) throws IOException {
if (! entry.getBaguetteServer().isServerRunning()) throw new RuntimeException("Baguette Server is not running");
log.trace("AbstractInstallationHelper.prepareUninstallInstructionsForOs(): node-map={}", entry.getPreregistration());
String osFamily = entry.getPreregistration().get("operatingSystem");
List<InstructionsSet> instructionsSetList = null;
if (matchesOsFamily(osFamily, LINUX_OS_FAMILY))
instructionsSetList = prepareUninstallInstructionsForLinux(entry);
else if (matchesOsFamily(osFamily, WINDOWS_OS_FAMILY))
instructionsSetList = prepareUninstallInstructionsForWin(entry);
else
log.warn("AbstractInstallationHelper.prepareUninstallInstructionsForOs(): Unsupported OS family: {}", osFamily);
return instructionsSetList;
}
public boolean matchesOsFamily(@NonNull String lookup, @NonNull String osFamily) { public boolean matchesOsFamily(@NonNull String lookup, @NonNull String osFamily) {
lookup = lookup.trim().toUpperCase(); lookup = lookup.trim().toUpperCase();
List<String> familyList = properties.getOsFamilies().get(osFamily); List<String> familyList = properties.getOsFamilies().get(osFamily);

@ -25,8 +25,13 @@ public interface InstallationHelper {
List<InstructionsSet> prepareInstallationInstructionsForWin(NodeRegistryEntry entry); List<InstructionsSet> prepareInstallationInstructionsForWin(NodeRegistryEntry entry);
List<InstructionsSet> prepareInstallationInstructionsForLinux(NodeRegistryEntry entry) throws IOException; List<InstructionsSet> prepareInstallationInstructionsForLinux(NodeRegistryEntry entry) throws IOException;
List<InstructionsSet> prepareUninstallInstructionsForOs(NodeRegistryEntry entry) throws IOException;
List<InstructionsSet> prepareUninstallInstructionsForWin(NodeRegistryEntry entry);
List<InstructionsSet> prepareUninstallInstructionsForLinux(NodeRegistryEntry entry) throws IOException;
default ClientInstallationTask createClientInstallationTask(NodeRegistryEntry entry) throws Exception { default ClientInstallationTask createClientInstallationTask(NodeRegistryEntry entry) throws Exception {
return createClientInstallationTask(entry, null); return createClientInstallationTask(entry, null);
} }
ClientInstallationTask createClientInstallationTask(NodeRegistryEntry entry, TranslationContext translationContext) throws Exception; ClientInstallationTask createClientInstallationTask(NodeRegistryEntry entry, TranslationContext translationContext) throws Exception;
ClientInstallationTask createClientUninstallTask(NodeRegistryEntry entry, TranslationContext translationContext) throws Exception;
} }

@ -22,6 +22,7 @@ import gr.iccs.imu.ems.baguette.server.NodeRegistryEntry;
import gr.iccs.imu.ems.translate.TranslationContext; import gr.iccs.imu.ems.translate.TranslationContext;
import gr.iccs.imu.ems.util.CredentialsMap; import gr.iccs.imu.ems.util.CredentialsMap;
import gr.iccs.imu.ems.util.NetUtil; import gr.iccs.imu.ems.util.NetUtil;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.commons.text.StringSubstitutor; import org.apache.commons.text.StringSubstitutor;
@ -60,7 +61,31 @@ public class VmInstallationHelper extends AbstractInstallationHelper {
@Override @Override
public ClientInstallationTask createClientInstallationTask(NodeRegistryEntry entry, TranslationContext translationContext) throws IOException { public ClientInstallationTask createClientInstallationTask(NodeRegistryEntry entry, TranslationContext translationContext) throws IOException {
Map<String, String> nodeMap = entry.getPreregistration(); // Get EMS client installation instructions for VM node
List<InstructionsSet> instructionsSetList =
prepareInstallationInstructionsForOs(entry);
return createClientTask(ClientInstallationTask.TASK_TYPE.INSTALL, entry, translationContext, instructionsSetList);
}
@Override
public ClientInstallationTask createClientUninstallTask(NodeRegistryEntry entry, TranslationContext translationContext) throws Exception {
// Clear any cached 'instruction-files' override (from a previous run)
entry.getPreregistration().remove("instruction-files");
// Get EMS client uninstall instructions for VM node
List<InstructionsSet> instructionsSetList =
prepareUninstallInstructionsForOs(entry);
return createClientTask(ClientInstallationTask.TASK_TYPE.UNINSTALL, entry, translationContext, instructionsSetList);
}
private ClientInstallationTask createClientTask(@NonNull ClientInstallationTask.TASK_TYPE taskType,
NodeRegistryEntry entry,
TranslationContext translationContext,
List<InstructionsSet> instructionsSetList)
{
Map<String, String> nodeMap = initializeNodeMap(entry);
String baseUrl = nodeMap.get("BASE_URL"); String baseUrl = nodeMap.get("BASE_URL");
String clientId = nodeMap.get("CLIENT_ID"); String clientId = nodeMap.get("CLIENT_ID");
@ -95,13 +120,10 @@ public class VmInstallationHelper extends AbstractInstallationHelper {
if (StringUtils.isEmpty(password) && StringUtils.isBlank(privateKey)) if (StringUtils.isEmpty(password) && StringUtils.isBlank(privateKey))
throw new IllegalArgumentException("Missing SSH password or private key for Node"); throw new IllegalArgumentException("Missing SSH password or private key for Node");
// Get EMS client installation instructions for VM node
List<InstructionsSet> instructionsSetList =
prepareInstallationInstructionsForOs(entry);
// Create Installation Task for VM node // Create Installation Task for VM node
ClientInstallationTask installationTask = ClientInstallationTask.builder() ClientInstallationTask task = ClientInstallationTask.builder()
.id(clientId) .id(clientId)
.taskType(taskType)
.nodeId(nodeId) .nodeId(nodeId)
.requestId(requestId) .requestId(requestId)
.name(nodeName) .name(nodeName)
@ -121,29 +143,21 @@ public class VmInstallationHelper extends AbstractInstallationHelper {
.nodeRegistryEntry(entry) .nodeRegistryEntry(entry)
.translationContext(translationContext) .translationContext(translationContext)
.build(); .build();
log.debug("VmInstallationHelper.createClientInstallationTask(): Created client installation task: {}", installationTask); log.debug("VmInstallationHelper.createClientTask(): Created client task: {}", task);
return task;
return installationTask;
} }
@Override private Map<String, String> initializeNodeMap(NodeRegistryEntry entry) {
public List<InstructionsSet> prepareInstallationInstructionsForWin(NodeRegistryEntry entry) {
log.warn("VmInstallationHelper.prepareInstallationInstructionsForWin(): NOT YET IMPLEMENTED");
throw new IllegalArgumentException("VmInstallationHelper.prepareInstallationInstructionsForWin(): NOT YET IMPLEMENTED");
}
@Override
public List<InstructionsSet> prepareInstallationInstructionsForLinux(NodeRegistryEntry entry) throws IOException {
Map<String, String> nodeMap = entry.getPreregistration(); Map<String, String> nodeMap = entry.getPreregistration();
BaguetteServer baguette = entry.getBaguetteServer(); BaguetteServer baguette = entry.getBaguetteServer();
String baseUrl = StringUtils.removeEnd(nodeMap.get("BASE_URL"), "/"); String baseUrl = StringUtils.removeEnd(nodeMap.get("BASE_URL"), "/");
String clientId = nodeMap.get("CLIENT_ID"); String clientId = nodeMap.get("CLIENT_ID");
String ipSetting = nodeMap.get("IP_SETTING"); String ipSetting = nodeMap.get("IP_SETTING");
log.debug("VmInstallationHelper.prepareInstallationInstructionsForLinux(): Invoked: base-url={}", baseUrl); log.debug("VmInstallationHelper.initializeNodeMap(): Invoked: base-url={}", baseUrl);
// Get parameters // Get parameters
log.trace("VmInstallationHelper.prepareInstallationInstructionsForLinux(): properties: {}", properties); log.trace("VmInstallationHelper.initializeNodeMap(): properties: {}", properties);
String rootCmd = properties.getRootCmd(); String rootCmd = properties.getRootCmd();
String baseDir = properties.getBaseDir(); String baseDir = properties.getBaseDir();
String checkInstallationFile = properties.getCheckInstalledFile(); String checkInstallationFile = properties.getCheckInstalledFile();
@ -170,7 +184,7 @@ public class VmInstallationHelper extends AbstractInstallationHelper {
: "NODE_" + e.getKey().toUpperCase(), : "NODE_" + e.getKey().toUpperCase(),
Map.Entry::getValue, Map.Entry::getValue,
(v1, v2) -> { (v1, v2) -> {
log.warn("VmInstallationHelper.prepareInstallationInstructionsForLinux(): DUPLICATE KEY FOUND: key={}, old-value={}, new-value={}", log.warn("VmInstallationHelper.initializeNodeMap(): DUPLICATE KEY FOUND: key={}, old-value={}, new-value={}",
k, v1, v2); k, v1, v2);
return v2; return v2;
} }
@ -182,12 +196,12 @@ public class VmInstallationHelper extends AbstractInstallationHelper {
? "NODE_SSH_" + k.substring(4).toUpperCase() ? "NODE_SSH_" + k.substring(4).toUpperCase()
: "NODE_" + k.toUpperCase(); : "NODE_" + k.toUpperCase();
if (additionalKeysMap.containsKey(k)) { if (additionalKeysMap.containsKey(k)) {
log.warn("VmInstallationHelper.prepareInstallationInstructionsForLinux(): DUPLICATE KEY FOUND: key={}, old-value={}, new-value={}", log.warn("VmInstallationHelper.initializeNodeMap(): DUPLICATE KEY FOUND: key={}, old-value={}, new-value={}",
k, additionalKeysMap.get(k), v); k, additionalKeysMap.get(k), v);
} }
additionalKeysMap.put(k, v); additionalKeysMap.put(k, v);
} catch (Exception ex) { } catch (Exception ex) {
log.error("VmInstallationHelper.prepareInstallationInstructionsForLinux(): EXCEPTION in additional keys copy loop: key={}, value={}, additionalKeysMap={}, Exception:\n", log.error("VmInstallationHelper.initializeNodeMap(): EXCEPTION in additional keys copy loop: key={}, value={}, additionalKeysMap={}, Exception:\n",
k, v, additionalKeysMap, ex); k, v, additionalKeysMap, ex);
} }
}); });
@ -229,7 +243,7 @@ public class VmInstallationHelper extends AbstractInstallationHelper {
nodeMap.putAll(clientInstallationProperties.getParameters()); nodeMap.putAll(clientInstallationProperties.getParameters());
nodeMap.put("EMS_PUBLIC_DIR", System.getProperty("PUBLIC_DIR", System.getenv("PUBLIC_DIR"))); nodeMap.put("EMS_PUBLIC_DIR", System.getProperty("PUBLIC_DIR", System.getenv("PUBLIC_DIR")));
log.trace("VmInstallationHelper.prepareInstallationInstructionsForLinux: value-map: {}", nodeMap); log.trace("VmInstallationHelper.initializeNodeMap: value-map: {}", nodeMap);
/* // Clear EMS server certificate (PEM) file, if not secure /* // Clear EMS server certificate (PEM) file, if not secure
if (!isServerSecure) { if (!isServerSecure) {
@ -251,26 +265,65 @@ public class VmInstallationHelper extends AbstractInstallationHelper {
} }
}*/ }*/
return nodeMap;
}
@Override
public List<InstructionsSet> prepareInstallationInstructionsForWin(NodeRegistryEntry entry) {
log.warn("VmInstallationHelper.prepareInstallationInstructionsForWin(): NOT YET IMPLEMENTED");
throw new IllegalArgumentException("VmInstallationHelper.prepareInstallationInstructionsForWin(): NOT YET IMPLEMENTED");
}
@Override
public List<InstructionsSet> prepareUninstallInstructionsForWin(NodeRegistryEntry entry) {
log.warn("VmInstallationHelper.prepareUninstallInstructionsForWin(): NOT YET IMPLEMENTED");
throw new IllegalArgumentException("VmInstallationHelper.prepareUninstallInstructionsForWin(): NOT YET IMPLEMENTED");
}
@Override
public List<InstructionsSet> prepareInstallationInstructionsForLinux(NodeRegistryEntry entry) throws IOException {
return prepareInstructionsForLinux(entry, "LINUX");
}
@Override
public List<InstructionsSet> prepareUninstallInstructionsForLinux(NodeRegistryEntry entry) throws IOException {
return prepareInstructionsForLinux(entry, "REMOVE_LINUX");
}
private List<InstructionsSet> prepareInstructionsForLinux(@NonNull NodeRegistryEntry entry, String instructionsScenarioName) throws IOException {
Map<String, String> nodeMap = entry.getPreregistration();
List<InstructionsSet> instructionsSetList = new ArrayList<>(); List<InstructionsSet> instructionsSetList = new ArrayList<>();
try { try {
// Read installation instructions from JSON file // Read installation instructions from JSON file
List<String> instructionSetFileList = null; List<String> instructionSetFileList;
if (nodeMap.containsKey("instruction-files")) { if (nodeMap.containsKey("instruction-files")) {
log.trace("VmInstallationHelper.prepareInstructionsForLinux: FOUND instruction-files override: value={}", nodeMap.getOrDefault("instruction-files", null));
instructionSetFileList = Arrays.stream(nodeMap.getOrDefault("instruction-files", "").split(",")) instructionSetFileList = Arrays.stream(nodeMap.getOrDefault("instruction-files", "").split(","))
.filter(StringUtils::isNotBlank) .filter(StringUtils::isNotBlank)
.map(String::trim) .map(String::trim)
.collect(Collectors.toList()); .collect(Collectors.toList());
log.debug("VmInstallationHelper.prepareInstructionsForLinux: FOUND instruction-files override: list={}", instructionSetFileList);
if (instructionSetFileList.isEmpty()) if (instructionSetFileList.isEmpty())
log.warn("VmInstallationHelper.prepareInstallationInstructionsForLinux: Context map contains 'instruction-files' entry with no contents"); log.warn("VmInstallationHelper.prepareInstructionsForLinux: Context map contains 'instruction-files' entry with no contents");
} else { } else {
instructionSetFileList = properties.getInstructions().get("LINUX"); log.trace("VmInstallationHelper.prepareInstructionsForLinux: NOT FOUND instruction-files override. Using configured instructions sets: instructionsScenarioName={}", instructionsScenarioName);
instructionSetFileList = properties.getInstructions().get(instructionsScenarioName);
log.trace("VmInstallationHelper.prepareInstructionsForLinux: NOT FOUND instruction-files override. Using configured instructions sets: list={}", instructionSetFileList);
if (instructionSetFileList==null || instructionSetFileList.isEmpty()) {
log.warn("VmInstallationHelper.prepareInstructionsForLinux: No instructions set files provided in configuration with name: {}", instructionsScenarioName);
instructionSetFileList = Collections.emptyList();
}
} }
log.debug("VmInstallationHelper.prepareInstructionsForLinux: Instructions sets list: {}", instructionSetFileList);
// Load instructions set from file
for (String instructionSetFile : instructionSetFileList) { for (String instructionSetFile : instructionSetFileList) {
// Load instructions set from file // Load instructions set from file
log.debug("VmInstallationHelper.prepareInstallationInstructionsForLinux: Installation instructions file for LINUX: {}", instructionSetFile); log.debug("VmInstallationHelper.prepareInstructionsForLinux: Installation instructions file for LINUX: {}", instructionSetFile);
InstructionsSet instructionsSet = InstructionsService.getInstance().loadInstructionsFile(instructionSetFile); InstructionsSet instructionsSet = InstructionsService.getInstance().loadInstructionsFile(instructionSetFile);
log.debug("VmInstallationHelper.prepareInstallationInstructionsForLinux: Instructions set loaded from file: {}\n{}", instructionSetFile, instructionsSet); log.debug("VmInstallationHelper.prepareInstructionsForLinux: Instructions set loaded from file: {}\n{}", instructionSetFile, instructionsSet);
// Pretty print instructionsSet JSON // Pretty print instructionsSet JSON
if (log.isTraceEnabled()) { if (log.isTraceEnabled()) {
@ -279,7 +332,7 @@ public class VmInstallationHelper extends AbstractInstallationHelper {
try (PrintWriter writer = new PrintWriter(stringWriter)) { try (PrintWriter writer = new PrintWriter(stringWriter)) {
gson.toJson(instructionsSet, writer); gson.toJson(instructionsSet, writer);
} }
log.trace("VmInstallationHelper.prepareInstallationInstructionsForLinux: Installation instructions for LINUX: json:\n{}", stringWriter); log.trace("VmInstallationHelper.prepareInstructionsForLinux: Installation instructions for LINUX: json:\n{}", stringWriter);
} }
instructionsSetList.add(instructionsSet); instructionsSetList.add(instructionsSet);
@ -287,7 +340,7 @@ public class VmInstallationHelper extends AbstractInstallationHelper {
return instructionsSetList; return instructionsSetList;
} catch (Exception ex) { } catch (Exception ex) {
log.error("VmInstallationHelper.prepareInstallationInstructionsForLinux: Exception while reading Installation instructions for LINUX: ", ex); log.error("VmInstallationHelper.prepareInstructionsForLinux: Exception while reading Installation instructions for LINUX: ", ex);
throw ex; throw ex;
} }
} }

@ -48,8 +48,9 @@ public class ClientRecoveryPlugin implements InitializingBean, EventBus.EventCon
private long clientRecoveryDelay; private long clientRecoveryDelay;
private String recoveryInstructionsFile; private String recoveryInstructionsFile;
private final static String CLIENT_EXIT_TOPIC = "BAGUETTE_SERVER_CLIENT_EXITED"; public final static String CLIENT_EXIT_TOPIC = "BAGUETTE_SERVER_CLIENT_EXITED";
private final static String CLIENT_REGISTERED_TOPIC = "BAGUETTE_SERVER_CLIENT_REGISTERED"; public final static String CLIENT_REGISTERED_TOPIC = "BAGUETTE_SERVER_CLIENT_REGISTERED";
public final static String CLIENT_REMOVED_TOPIC = "BAGUETTE_SERVER_CLIENT_REMOVED";
@Override @Override
public void afterPropertiesSet() throws Exception { public void afterPropertiesSet() throws Exception {
@ -61,6 +62,8 @@ public class ClientRecoveryPlugin implements InitializingBean, EventBus.EventCon
log.debug("ClientRecoveryPlugin: Subscribed for BAGUETTE_SERVER_CLIENT_EXITED events"); log.debug("ClientRecoveryPlugin: Subscribed for BAGUETTE_SERVER_CLIENT_EXITED events");
eventBus.subscribe(CLIENT_REGISTERED_TOPIC, this); eventBus.subscribe(CLIENT_REGISTERED_TOPIC, this);
log.debug("ClientRecoveryPlugin: Subscribed for BAGUETTE_SERVER_CLIENT_REGISTERED events"); log.debug("ClientRecoveryPlugin: Subscribed for BAGUETTE_SERVER_CLIENT_REGISTERED events");
eventBus.subscribe(CLIENT_REMOVED_TOPIC, this);
log.debug("ClientRecoveryPlugin: Subscribed for CLIENT_REMOVED_TOPIC events");
log.trace("ClientRecoveryPlugin: clientInstallationProperties: {}", clientInstallationProperties); log.trace("ClientRecoveryPlugin: clientInstallationProperties: {}", clientInstallationProperties);
log.trace("ClientRecoveryPlugin: baguetteServer: {}", baguetteServer); log.trace("ClientRecoveryPlugin: baguetteServer: {}", baguetteServer);
@ -80,18 +83,28 @@ public class ClientRecoveryPlugin implements InitializingBean, EventBus.EventCon
} }
// Only process messages of ClientShellCommand type are accepted (sent by CSC instances) // Only process messages of ClientShellCommand type are accepted (sent by CSC instances)
if (! (message instanceof ClientShellCommand)) { if (! (message instanceof NodeRegistryEntry) && ! (message instanceof ClientShellCommand)) {
log.warn("ClientRecoveryPlugin: onMessage(): Message is not a {} object. Will ignore it.", ClientShellCommand.class.getSimpleName()); log.warn("ClientRecoveryPlugin: onMessage(): Message is neither a {} or a {} object. Will ignore it.",
NodeRegistryEntry.class.getSimpleName(), ClientShellCommand.class.getSimpleName());
return; return;
} }
// Get NodeRegistryEntry from ClientShellCommand passed with event NodeRegistryEntry nodeInfo;
ClientShellCommand csc = (ClientShellCommand)message; String clientId;
String clientId = csc.getId(); String address;
String address = csc.getClientIpAddress(); if (message instanceof NodeRegistryEntry entry) {
log.debug("ClientRecoveryPlugin: onMessage(): client-id={}, client-address={}", clientId, address); nodeInfo = entry;
clientId = entry.getClientId();
address = entry.getIpAddress();
} else {
// Get NodeRegistryEntry from ClientShellCommand passed with event
ClientShellCommand csc = (ClientShellCommand) message;
clientId = csc.getId();
address = csc.getClientIpAddress();
log.debug("ClientRecoveryPlugin: onMessage(): client-id={}, client-address={}", clientId, address);
NodeRegistryEntry nodeInfo = csc.getNodeRegistryEntry(); //or = nodeRegistry.getNodeByAddress(address); nodeInfo = csc.getNodeRegistryEntry(); //or = nodeRegistry.getNodeByAddress(address);
}
log.debug("ClientRecoveryPlugin: onMessage(): client-node-info={}", nodeInfo); log.debug("ClientRecoveryPlugin: onMessage(): client-node-info={}", nodeInfo);
log.trace("ClientRecoveryPlugin: onMessage(): node-registry.node-addresses={}", nodeRegistry.getNodeAddresses()); log.trace("ClientRecoveryPlugin: onMessage(): node-registry.node-addresses={}", nodeRegistry.getNodeAddresses());
log.trace("ClientRecoveryPlugin: onMessage(): node-registry.nodes={}", nodeRegistry.getNodes()); log.trace("ClientRecoveryPlugin: onMessage(): node-registry.nodes={}", nodeRegistry.getNodes());
@ -108,14 +121,25 @@ public class ClientRecoveryPlugin implements InitializingBean, EventBus.EventCon
processExitEvent(nodeInfo); processExitEvent(nodeInfo);
} }
if (CLIENT_REGISTERED_TOPIC.equals(topic)) { if (CLIENT_REGISTERED_TOPIC.equals(topic)) {
log.debug("ClientRecoveryPlugin: onMessage(): CLIENT REGISTERED_TOPIC: message={}", message); log.debug("ClientRecoveryPlugin: onMessage(): CLIENT REGISTERED: message={}", message);
processRegisteredEvent(nodeInfo); processRegisteredEvent(nodeInfo);
} }
if (CLIENT_REMOVED_TOPIC.equals(topic)) {
log.debug("ClientRecoveryPlugin: onMessage(): CLIENT REMOVED: message={}", message);
processRemovedEvent(nodeInfo);
}
} }
private void processExitEvent(NodeRegistryEntry nodeInfo) { private void processExitEvent(NodeRegistryEntry nodeInfo) {
log.debug("ClientRecoveryPlugin: processExitEvent(): BEGIN: client-id={}, client-address={}", nodeInfo.getClientId(), nodeInfo.getIpAddress()); log.debug("ClientRecoveryPlugin: processExitEvent(): BEGIN: client-id={}, client-address={}", nodeInfo.getClientId(), nodeInfo.getIpAddress());
// Check if node can be recovered (based on its Life-Cycle state)
if (! nodeInfo.canRecover()) {
log.warn("ClientRecoveryPlugin: processExitEvent(): Node will not be recovered because its state is {}: client-id={}, client-address={}",
nodeInfo.getState(), nodeInfo.getClientId(), nodeInfo.getIpAddress());
return;
}
// Set node state to DOWN // Set node state to DOWN
baguetteServer.getSelfHealingManager().setNodeSelfHealingState(nodeInfo, SelfHealingManager.NODE_STATE.DOWN); baguetteServer.getSelfHealingManager().setNodeSelfHealingState(nodeInfo, SelfHealingManager.NODE_STATE.DOWN);
@ -156,10 +180,29 @@ public class ClientRecoveryPlugin implements InitializingBean, EventBus.EventCon
baguetteServer.getSelfHealingManager().setNodeSelfHealingState(nodeInfo, SelfHealingManager.NODE_STATE.UP); baguetteServer.getSelfHealingManager().setNodeSelfHealingState(nodeInfo, SelfHealingManager.NODE_STATE.UP);
} }
private void processRemovedEvent(NodeRegistryEntry nodeInfo) {
log.debug("ClientRecoveryPlugin: processRemovedEvent(): BEGIN: client-id={}, client-address={}", nodeInfo.getClientId(), nodeInfo.getIpAddress());
// Cancel any pending recovery task (for the node)
ScheduledFuture<?> future = pendingTasks.remove(nodeInfo);
log.debug("ClientRecoveryPlugin: processRemovedEvent(): Recovery task: task={}, client-id={}, client-address={}", future, nodeInfo.getClientId(), nodeInfo.getIpAddress());
if (future!=null && ! future.isDone() && ! future.isCancelled()) {
log.warn("ClientRecoveryPlugin: processRemovedEvent(): Cancelled recovery task: client-id={}, client-address={}", nodeInfo.getClientId(), nodeInfo.getIpAddress());
future.cancel(false);
}
}
public void runClientRecovery(NodeRegistryEntry entry) throws Exception { public void runClientRecovery(NodeRegistryEntry entry) throws Exception {
log.debug("ClientRecoveryPlugin: runClientRecovery(): node-info={}", entry); log.debug("ClientRecoveryPlugin: runClientRecovery(): node-info={}", entry);
if (entry==null) return; if (entry==null) return;
if (! entry.canRecover()) {
log.info("ClientRecoveryPlugin: runClientRecovery(): Cannot recover node. Will not attempt recovery again: node-state={}, client-id={}, client-address={}",
entry.getState(), entry.getClientId(), entry.getIpAddress());
pendingTasks.remove(entry);
return;
}
log.trace("ClientRecoveryPlugin: runClientRecovery(): recoveryInstructionsFile={}", recoveryInstructionsFile); log.trace("ClientRecoveryPlugin: runClientRecovery(): recoveryInstructionsFile={}", recoveryInstructionsFile);
entry.getPreregistration().put("instruction-files", recoveryInstructionsFile); entry.getPreregistration().put("instruction-files", recoveryInstructionsFile);

@ -550,4 +550,13 @@ public class BaguetteServer implements InitializingBean, EventBus.EventConsumer<
} }
return clientId; return clientId;
} }
public NodeRegistryEntry unregisterClient(NodeRegistryEntry entry) throws UnknownHostException {
log.debug("BaguetteServer.unregisterClient(): entry={}", entry);
// Archive node entry in node registry
nodeRegistry.archiveNode(entry);
return entry;
}
} }

@ -265,8 +265,16 @@ public class ClientShellCommand implements Command, Runnable, ServerSessionAware
} }
} }
// Client connection closed // Client connection closed
eventBus.send("BAGUETTE_SERVER_CLIENT_EXITING", this); try {
getNodeRegistryEntry().nodeExiting(null); eventBus.send("BAGUETTE_SERVER_CLIENT_EXITING", this);
NodeRegistryEntry entry = getNodeRegistryEntry();
if (! entry.isArchived())
entry.nodeExiting(null);
else
log.warn("{}==> Node is archived", id);
} catch (Exception e) {
log.warn("{}==> EXCEPTION: ", id, e);
}
log.info("{}==> Signaling client to exit", id); log.info("{}==> Signaling client to exit", id);
out.println("EXIT"); out.println("EXIT");

@ -10,6 +10,7 @@
package gr.iccs.imu.ems.baguette.server; package gr.iccs.imu.ems.baguette.server;
import lombok.Getter; import lombok.Getter;
import lombok.NonNull;
import lombok.Setter; import lombok.Setter;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
@ -17,9 +18,7 @@ import org.springframework.stereotype.Service;
import java.net.InetAddress; import java.net.InetAddress;
import java.net.UnknownHostException; import java.net.UnknownHostException;
import java.util.Collection; import java.util.*;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.stream.Collectors; import java.util.stream.Collectors;
/** /**
@ -29,6 +28,7 @@ import java.util.stream.Collectors;
@Service @Service
public class NodeRegistry { public class NodeRegistry {
private final Map<String,NodeRegistryEntry> registry = new LinkedHashMap<>(); private final Map<String,NodeRegistryEntry> registry = new LinkedHashMap<>();
private final List<NodeRegistryEntry> archived = new LinkedList<>();
@Getter @Setter @Getter @Setter
private ServerCoordinator coordinator; private ServerCoordinator coordinator;
@ -134,4 +134,27 @@ public class NodeRegistry {
public Collection<String> getNodeReferences() { public Collection<String> getNodeReferences() {
return registry.values().stream().map(NodeRegistryEntry::getReference).collect(Collectors.toList()); return registry.values().stream().map(NodeRegistryEntry::getReference).collect(Collectors.toList());
} }
public synchronized void archiveNode(@NonNull NodeRegistryEntry entry) {
log.debug("NodeRegistry.archiveNode(): Archiving node: entry={}", entry);
// Run checks
if (StringUtils.isBlank(entry.getIpAddress())) {
log.warn("NodeRegistry.archiveNode(): Node entry does not have an IP address. Ignoring archiving operation: entry={}", entry);
return;
}
if (! registry.containsKey(entry.getIpAddress())) {
log.warn("NodeRegistry.archiveNode(): Node with IP address {} not found in registry. Ignoring archiving operation: entry={}", entry.getIpAddress(), entry);
return;
}
if (registry.get(entry.getIpAddress())!=entry) {
log.warn("NodeRegistry.archiveNode(): Node passed does not match with node registered to IP address {}. Ignoring archiving operation: entry={}", entry.getIpAddress(), entry);
return;
}
// Archive node
archived.add(entry);
registry.remove(entry.getIpAddress());
entry.nodeArchived(null);
}
} }

@ -21,8 +21,12 @@ import java.util.*;
@AllArgsConstructor @AllArgsConstructor
@RequiredArgsConstructor @RequiredArgsConstructor
public class NodeRegistryEntry { public class NodeRegistryEntry {
public enum STATE { PREREGISTERED, IGNORE_NODE, INSTALLING, NOT_INSTALLED, INSTALLED, INSTALL_ERROR, public enum STATE {
WAITING_REGISTRATION, REGISTERING, REGISTERED, REGISTRATION_ERROR, DISCONNECTED, EXITING, EXITED, NODE_FAILED PREREGISTERED, IGNORE_NODE,
INSTALLING, NOT_INSTALLED, INSTALLED, INSTALL_ERROR,
WAITING_REGISTRATION, REGISTERING, REGISTERED, REGISTRATION_ERROR,
DISCONNECTED, EXITING, EXITED, NODE_FAILED,
REMOVING, REMOVED, REMOVE_ERROR, ARCHIVED
}; };
@Getter private final String ipAddress; @Getter private final String ipAddress;
@Getter private final String clientId; @Getter private final String clientId;
@ -40,6 +44,8 @@ public class NodeRegistryEntry {
@JsonIgnore @JsonIgnore
@Getter private transient Map<String, String> registration = new LinkedHashMap<>(); @Getter private transient Map<String, String> registration = new LinkedHashMap<>();
@JsonIgnore @JsonIgnore
@Getter private transient Map<String, String> removal = new LinkedHashMap<>();
@JsonIgnore
@Getter @Setter private transient IClusterZone clusterZone; @Getter @Setter private transient IClusterZone clusterZone;
@JsonIgnore @JsonIgnore
@ -70,100 +76,118 @@ public class NodeRegistryEntry {
public void refreshReference() { reference = UUID.randomUUID().toString(); } public void refreshReference() { reference = UUID.randomUUID().toString(); }
public NodeRegistryEntry nodePreregistration(Map<String,Object> nodeInfo) { public boolean canRecover() {
preregistration.clear(); return state != null && switch (state) {
preregistration.putAll(StrUtil.deepFlattenMap(nodeInfo)); case PREREGISTERED, IGNORE_NODE -> false;
setState(STATE.PREREGISTERED); case REMOVING, REMOVED, REMOVE_ERROR, ARCHIVED -> false;
default -> true;
};
}
public boolean isArchived() {
return state!=null && state==STATE.ARCHIVED;
}
public boolean canChangeStateTo(@NonNull STATE newState) {
return ! isArchived();
}
private void _canUpdateEntry(@NonNull STATE newState) {
if (! canChangeStateTo(newState)) {
throw new IllegalStateException(String.format("Cannot change NodeRegistryEntry state from %s to %s: client-id=%s, client-address=%s",
state, newState, clientId, ipAddress));
}
}
private NodeRegistryEntry _updateEntry(@NonNull STATE newState, @NonNull Map<String, String> map, boolean clear, Map<String,Object> nodeInfo) {
_canUpdateEntry(newState);
if (clear) map.clear();
map.putAll(StrUtil.deepFlattenMap(nodeInfo));
setState(newState);
return this; return this;
} }
public NodeRegistryEntry nodeIgnore(Object nodeInfo) { private NodeRegistryEntry _updateEntry(@NonNull STATE newState, @NonNull Map<String, String> map, boolean clear, @NonNull String key, Object val, String defVal) {
installation.clear(); _canUpdateEntry(newState);
installation.put("ignore-node", nodeInfo!=null ? nodeInfo.toString() : null); if (clear) map.clear();
map.put(key, val!=null ? val.toString() : defVal);
setState(STATE.IGNORE_NODE); setState(STATE.IGNORE_NODE);
return this; return this;
} }
public NodeRegistryEntry nodePreregistration(Map<String,Object> nodeInfo) {
return _updateEntry(STATE.PREREGISTERED, preregistration, true, nodeInfo);
}
public NodeRegistryEntry nodeIgnore(Object nodeInfo) {
return _updateEntry(STATE.IGNORE_NODE, installation, true, "ignore-node", nodeInfo, null);
}
public NodeRegistryEntry nodeInstalling(Object nodeInfo) { public NodeRegistryEntry nodeInstalling(Object nodeInfo) {
installation.clear(); return _updateEntry(STATE.INSTALLING, installation, true, "installation-task", nodeInfo, "INSTALLING");
installation.put("installation-task", nodeInfo!=null ? nodeInfo.toString() : "INSTALLING");
setState(STATE.INSTALLING);
return this;
} }
public NodeRegistryEntry nodeNotInstalled(Object nodeInfo) { public NodeRegistryEntry nodeNotInstalled(Object nodeInfo) {
installation.clear(); return _updateEntry(STATE.NOT_INSTALLED, installation, true, "installation-task-result", nodeInfo, "NOT_INSTALLED");
installation.put("installation-task-result", nodeInfo!=null ? nodeInfo.toString() : "NOT_INSTALLED");
setState(STATE.NOT_INSTALLED);
return this;
} }
public NodeRegistryEntry nodeInstallationComplete(Object nodeInfo) { public NodeRegistryEntry nodeInstallationComplete(Object nodeInfo) {
installation.put("installation-task-result", nodeInfo!=null ? nodeInfo.toString() : "SUCCESS"); return _updateEntry(STATE.INSTALLED, installation, false, "installation-task-result", nodeInfo, "SUCCESS");
setState(STATE.INSTALLED);
return this;
} }
public NodeRegistryEntry nodeInstallationError(Object nodeInfo) { public NodeRegistryEntry nodeInstallationError(Object nodeInfo) {
installation.put("installation-task-result", nodeInfo!=null ? nodeInfo.toString() : "ERROR"); return _updateEntry(STATE.INSTALL_ERROR, installation, false, "installation-task-result", nodeInfo, "ERROR");
setState(STATE.INSTALL_ERROR);
return this;
} }
public NodeRegistryEntry nodeRegistering(Map<String,Object> nodeInfo) { public NodeRegistryEntry nodeRegistering(Map<String,Object> nodeInfo) {
registration.clear(); return _updateEntry(STATE.REGISTERING, registration, true, nodeInfo);
registration.putAll(StrUtil.deepFlattenMap(nodeInfo));
setState(STATE.REGISTERING);
return this;
} }
public NodeRegistryEntry nodeRegistered(Map<String,Object> nodeInfo) { public NodeRegistryEntry nodeRegistered(Map<String,Object> nodeInfo) {
//registration.clear(); return _updateEntry(STATE.REGISTERED, registration, false, nodeInfo);
registration.putAll(StrUtil.deepFlattenMap(nodeInfo));
setState(STATE.REGISTERED);
return this;
} }
public NodeRegistryEntry nodeRegistrationError(Map<String,Object> nodeInfo) { public NodeRegistryEntry nodeRegistrationError(Map<String,Object> nodeInfo) {
registration.putAll(StrUtil.deepFlattenMap(nodeInfo)); return _updateEntry(STATE.REGISTRATION_ERROR, registration, false, nodeInfo);
setState(STATE.REGISTRATION_ERROR);
return this;
} }
public NodeRegistryEntry nodeRegistrationError(Throwable t) { public NodeRegistryEntry nodeRegistrationError(Throwable t) {
registration.putAll(StrUtil.deepFlattenMap(Collections.singletonMap("exception", t))); return _updateEntry(STATE.REGISTRATION_ERROR, registration, false, Collections.singletonMap("exception", t));
setState(STATE.REGISTRATION_ERROR);
return this;
} }
public NodeRegistryEntry nodeDisconnected(Map<String,Object> nodeInfo) { public NodeRegistryEntry nodeDisconnected(Map<String,Object> nodeInfo) {
registration.putAll(StrUtil.deepFlattenMap(nodeInfo)); return _updateEntry(STATE.DISCONNECTED, registration, false, nodeInfo);
setState(STATE.DISCONNECTED);
return this;
} }
public NodeRegistryEntry nodeDisconnected(Throwable t) { public NodeRegistryEntry nodeDisconnected(Throwable t) {
registration.putAll(StrUtil.deepFlattenMap(Collections.singletonMap("exception", t))); return _updateEntry(STATE.DISCONNECTED, registration, false, Collections.singletonMap("exception", t));
setState(STATE.DISCONNECTED);
return this;
} }
public NodeRegistryEntry nodeExiting(Map<String,Object> nodeInfo) { public NodeRegistryEntry nodeExiting(Map<String,Object> nodeInfo) {
registration.putAll(StrUtil.deepFlattenMap(nodeInfo)); return _updateEntry(STATE.EXITING, registration, false, nodeInfo);
setState(STATE.EXITING);
return this;
} }
public NodeRegistryEntry nodeExited(Map<String,Object> nodeInfo) { public NodeRegistryEntry nodeExited(Map<String,Object> nodeInfo) {
registration.putAll(StrUtil.deepFlattenMap(nodeInfo)); return _updateEntry(STATE.EXITED, registration, false, nodeInfo);
setState(STATE.EXITED);
return this;
} }
public NodeRegistryEntry nodeFailed(Map<String,Object> failInfo) { public NodeRegistryEntry nodeFailed(Map<String,Object> failInfo) {
if (failInfo!=null) return _updateEntry(STATE.NODE_FAILED, registration, false, failInfo);
registration.putAll(StrUtil.deepFlattenMap(failInfo)); }
setState(STATE.NODE_FAILED);
return this; public NodeRegistryEntry nodeRemoving(Map<String,Object> nodeInfo) {
return _updateEntry(STATE.REMOVING, removal, false, nodeInfo);
}
public NodeRegistryEntry nodeRemoved(Map<String,Object> nodeInfo) {
return _updateEntry(STATE.REMOVED, removal, false, nodeInfo);
}
public NodeRegistryEntry nodeRemoveError(Map<String,Object> failInfo) {
return _updateEntry(STATE.REMOVE_ERROR, removal, false, failInfo);
}
public NodeRegistryEntry nodeArchived(Map<String,Object> nodeInfo) {
return _updateEntry(STATE.ARCHIVED, removal, false, nodeInfo);
} }
} }

@ -53,7 +53,7 @@ public class NodeRegistrationController {
return "OK"; return "OK";
} }
@RequestMapping(value = "/baguette/registerNode", method = POST, @RequestMapping(value = { "/baguette/registerNode", "/baguette/node/register" }, method = POST,
consumes = MediaType.APPLICATION_JSON_VALUE) consumes = MediaType.APPLICATION_JSON_VALUE)
public String baguetteRegisterNode(@RequestBody String jsonNode, HttpServletRequest request) throws Exception { public String baguetteRegisterNode(@RequestBody String jsonNode, HttpServletRequest request) throws Exception {
log.info("NodeRegistrationController.baguetteRegisterNode(): Invoked"); log.info("NodeRegistrationController.baguetteRegisterNode(): Invoked");
@ -80,6 +80,26 @@ public class NodeRegistrationController {
return response; return response;
} }
@RequestMapping(value = "/baguette/node/unregister/{ipAddress:.+}", method = {GET, POST},
produces = MediaType.TEXT_PLAIN_VALUE)
public String baguetteUnregisterNode(@PathVariable String ipAddress, HttpServletRequest request) throws Exception {
log.info("NodeRegistrationController.baguetteUnregisterNode(): Invoked");
log.debug("NodeRegistrationController.baguetteUnregisterNode(): Node IP address:\n{}", ipAddress);
String response;
try {
response = nodeRegistrationCoordinator.unregisterNode(ipAddress,
coordinator.getTranslationContextOfAppModel(coordinator.getCurrentAppModelId()));
} catch (Exception e) {
log.error("NodeRegistrationController.baguetteUnregisterNode(): EXCEPTION while unregistering node: address={}\n", ipAddress, e);
response = "ERROR "+e.getMessage();
}
log.info("NodeRegistrationController.baguetteUnregisterNode(): Node unregistered: node-address={}", ipAddress);
log.debug("NodeRegistrationController.baguetteUnregisterNode(): address={}, json={}", ipAddress, response);
return response;
}
@RequestMapping(value = "/baguette/node/list", method = GET) @RequestMapping(value = "/baguette/node/list", method = GET)
public Collection<String> baguetteNodeList() throws Exception { public Collection<String> baguetteNodeList() throws Exception {
log.info("NodeRegistrationController.baguetteNodeList(): Invoked"); log.info("NodeRegistrationController.baguetteNodeList(): Invoked");

@ -12,15 +12,19 @@ package gr.iccs.imu.ems.control.controller;
import gr.iccs.imu.ems.baguette.client.install.ClientInstallationTask; import gr.iccs.imu.ems.baguette.client.install.ClientInstallationTask;
import gr.iccs.imu.ems.baguette.client.install.ClientInstaller; import gr.iccs.imu.ems.baguette.client.install.ClientInstaller;
import gr.iccs.imu.ems.baguette.client.install.helper.InstallationHelperFactory; import gr.iccs.imu.ems.baguette.client.install.helper.InstallationHelperFactory;
import gr.iccs.imu.ems.baguette.client.selfhealing.ClientRecoveryPlugin;
import gr.iccs.imu.ems.baguette.server.BaguetteServer; import gr.iccs.imu.ems.baguette.server.BaguetteServer;
import gr.iccs.imu.ems.baguette.server.ClientShellCommand;
import gr.iccs.imu.ems.baguette.server.NodeRegistryEntry; import gr.iccs.imu.ems.baguette.server.NodeRegistryEntry;
import gr.iccs.imu.ems.control.properties.ControlServiceProperties; import gr.iccs.imu.ems.control.properties.ControlServiceProperties;
import gr.iccs.imu.ems.control.properties.StaticResourceProperties; import gr.iccs.imu.ems.control.properties.StaticResourceProperties;
import gr.iccs.imu.ems.translate.TranslationContext; import gr.iccs.imu.ems.translate.TranslationContext;
import gr.iccs.imu.ems.util.EventBus;
import gr.iccs.imu.ems.util.NetUtil; import gr.iccs.imu.ems.util.NetUtil;
import gr.iccs.imu.ems.util.StrUtil; import gr.iccs.imu.ems.util.StrUtil;
import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletRequest;
import lombok.Getter; import lombok.Getter;
import lombok.NonNull;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows; import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@ -35,6 +39,7 @@ import org.springframework.web.servlet.support.ServletUriComponentsBuilder;
import java.io.IOException; import java.io.IOException;
import java.util.Collections; import java.util.Collections;
import java.util.Map; import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
@Slf4j @Slf4j
@ -51,6 +56,8 @@ public class NodeRegistrationCoordinator implements InitializingBean {
private final ServerProperties serverProperties; private final ServerProperties serverProperties;
private final ServletWebServerApplicationContext webServerAppCtxt; private final ServletWebServerApplicationContext webServerAppCtxt;
private final EventBus<String,Object,Object> eventBus;
@Override @Override
public void afterPropertiesSet() throws Exception { public void afterPropertiesSet() throws Exception {
} }
@ -130,6 +137,60 @@ public class NodeRegistrationCoordinator implements InitializingBean {
return response; return response;
} }
public String unregisterNode(@NonNull String ipAddress, TranslationContext translationContext) throws Exception {
log.debug("NodeRegistrationCoordinator.unregisterNode(): BEGIN: ip-address={}", ipAddress);
if (StringUtils.isBlank(ipAddress)) {
log.error("NodeRegistrationCoordinator.unregisterNode(): Blank IP address provided");
throw new IllegalArgumentException("Blank IP address provided");
}
// Retrieve node entry from registry
NodeRegistryEntry entry = baguetteServer.getNodeRegistry().getNodeByAddress(ipAddress);
if (entry==null) {
log.error("NodeRegistrationCoordinator.unregisterNode(): Node not found in registry: address={}", ipAddress);
throw new IllegalArgumentException("Node not found in registry: address="+ipAddress);
}
// Set node state to REMOVING in order to prevent self-healing
entry.nodeRemoving(null);
// Signal self-healing plugin to cancel any pending recovery tasks
log.debug("NodeRegistrationCoordinator.unregisterNode(): Notifying Self-healing to remove any pending recovery tasks: address={}", ipAddress);
eventBus.send(ClientRecoveryPlugin.CLIENT_REMOVED_TOPIC, entry);
// Close CSC connection
ClientShellCommand csc = ClientShellCommand.getActiveByIpAddress(entry.getIpAddress());
if (csc!=null) {
log.info("NodeRegistrationCoordinator.unregisterNode(): Closing connection to EMS client: address={}", ipAddress);
csc.stop("REMOVING NODE");
} else
log.warn("NodeRegistrationCoordinator.unregisterNode(): CSC is null. Cannot close connection to EMS client. Probably connection has already closed: address={}", ipAddress);
// Continue processing according to ExecutionWare type
String response;
log.info("NodeRegistrationCoordinator.unregisterNode(): ExecutionWare: {}", properties.getExecutionware());
if (properties.getExecutionware()==ControlServiceProperties.ExecutionWare.CLOUDIATOR) {
response = "NOT SUPPORTED";
} else {
response = createClientUninstallTask(entry, translationContext, () -> {
// Unregister and Archive node
try {
entry.nodeRemoved(null);
baguetteServer.unregisterClient(entry);
return "OK";
} catch (Exception e) {
log.error("NodeRegistrationCoordinator.unregisterNode(): EXCEPTION while unregistering node: address={}, entry={}\n", ipAddress, entry, e);
entry.nodeRemoveError(Map.of(
"error", e.getMessage()
));
return "ERROR " + e.getMessage();
}
});
}
return response;
}
void updateRegistrationInfo(Map<String, String> nodeMap, String baseUrl) { void updateRegistrationInfo(Map<String, String> nodeMap, String baseUrl) {
// Set OS info // Set OS info
String os = StringUtils.isNotBlank(nodeMap.get("operatingSystem.name")) String os = StringUtils.isNotBlank(nodeMap.get("operatingSystem.name"))
@ -211,14 +272,32 @@ public class NodeRegistrationCoordinator implements InitializingBean {
} }
public String createClientInstallationTask(NodeRegistryEntry entry, TranslationContext translationContext) throws Exception { public String createClientInstallationTask(NodeRegistryEntry entry, TranslationContext translationContext) throws Exception {
//log.info("ControlServiceController.baguetteRegisterNodeForProactive(): INPUT: node-map: {}", nodeMap); return createClientInstallationTask(entry, translationContext, null);
}
public String createClientInstallationTask(NodeRegistryEntry entry, TranslationContext translationContext, Callable<String> callback) throws Exception {
ClientInstallationTask installationTask = InstallationHelperFactory.getInstance() ClientInstallationTask installationTask = InstallationHelperFactory.getInstance()
.createInstallationHelper(entry) .createInstallationHelper(entry)
.createClientInstallationTask(entry, translationContext); .createClientInstallationTask(entry, translationContext);
installationTask.setCallback(callback);
ClientInstaller.instance().addTask(installationTask); ClientInstaller.instance().addTask(installationTask);
log.debug("NodeRegistrationCoordinator.createClientInstallationTask(): New installation-task: {}", installationTask); log.debug("NodeRegistrationCoordinator.createClientInstallationTask(): New installation-task: {}", installationTask);
return "OK"; return "OK";
} }
public String createClientUninstallTask(NodeRegistryEntry entry, TranslationContext translationContext) throws Exception {
return createClientUninstallTask(entry, translationContext, null);
}
public String createClientUninstallTask(NodeRegistryEntry entry, TranslationContext translationContext, Callable<String> callback) throws Exception {
ClientInstallationTask uninstallTask = InstallationHelperFactory.getInstance()
.createInstallationHelper(entry)
.createClientUninstallTask(entry, translationContext);
uninstallTask.setCallback(callback);
ClientInstaller.instance().addTask(uninstallTask);
log.debug("NodeRegistrationCoordinator.createClientUninstallTask(): New uninstall-task: {}", uninstallTask);
return "OK";
}
} }

@ -36,4 +36,11 @@ public class NodeRegistrationHelper implements InitializingBean, INodeRegistrati
baseUrl, nodeInfo, translationContext); baseUrl, nodeInfo, translationContext);
return nodeRegistrationCoordinator.registerNode(baseUrl, nodeInfo, translationContext); return nodeRegistrationCoordinator.registerNode(baseUrl, nodeInfo, translationContext);
} }
@Override
public String unregisterNode(String nodeAddress, TranslationContext translationContext) throws Exception {
log.debug("NodeRegistrationHelper: Invoking unregisterNode: node-address={}, TC={}",
nodeAddress, translationContext);
return nodeRegistrationCoordinator.unregisterNode(nodeAddress, translationContext);
}
} }