nebulous/monitoring
Code Issues Proposed changes

EMS: Implementing node removal (off-boarding) [WIP]

Browse Source
This commit is contained in:
ipatini 2023-10-05 18:24:34 +03:00
parent 3083d7dad3
commit 271ae82855
15 changed files with 494 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/ClientInstallationRequestListener.java
View File

@ -130,6 +130,7 @@ public class ClientInstallationRequestListener implements InitializingBean {
switch (requestType) {
case "DIAGNOSTICS" -> processDiagnosticsRequest(request);
case "VM" -> processOnboardingRequest(request);
case "REMOVE" -> processRemoveRequest(request);
default -> throw new IllegalArgumentException("Unsupported request type: "+requestType);
};
@ -174,6 +175,7 @@ public class ClientInstallationRequestListener implements InitializingBean {
// Create client installation task
ClientInstallationTask newTask = ClientInstallationTask.builder()
.id(request.get("requestId"))
.taskType(ClientInstallationTask.TASK_TYPE.DIAGNOSTIC)
.requestId(request.get("requestId"))
.type(request.get("requestType"))
.nodeId(request.get("deviceId"))
@ -265,4 +267,26 @@ public class ClientInstallationRequestListener implements InitializingBean {
log.trace("InstallationEventListener.convertToNodeInfoMap(): END: nodeMap: {}", nodeMap);
return nodeMap;
}
private void processRemoveRequest(Map<String,String> request) throws Exception {
String requestId = request.getOrDefault("requestId", "").trim();
String nodeAddress = request.getOrDefault("deviceIpAddress", "").trim();
log.info("InstallationEventListener: New node REMOVE request with Id: {}, address={}", requestId, nodeAddress);
if (StringUtils.isBlank(requestId)) {
clientInstaller.sendErrorClientInstallationReport("MISSING-REQUEST-ID", "INVALID REQUEST. MISSING REQUEST ID");
return;
}
if (StringUtils.isBlank(nodeAddress)) {
clientInstaller.sendErrorClientInstallationReport(requestId, "INVALID REQUEST. MISSING IP ADDRESS");
return;
}
try {
log.debug("InstallationEventListener: Off-boarding node due to REMOVE request with Id: {}", requestId);
nodeRegistration.unregisterNode(nodeAddress, new TranslationContext(requestId));
} catch (Exception e) {
log.warn("InstallationEventListener: EXCEPTION while executing REMOVE request with Id: {}\n", requestId, e);
clientInstaller.sendErrorClientInstallationReport(requestId, "ERROR: "+e.getMessage());
}
}
}

8
ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/ClientInstallationTask.java
View File

@ -16,6 +16,7 @@ import lombok.Builder;
import lombok.Data;
import java.util.List;
import java.util.concurrent.Callable;
/**
* Client installation task
@ -23,17 +24,22 @@ import java.util.List;
@Data
@Builder
public class ClientInstallationTask {
public enum TASK_TYPE { INSTALL, UNINSTALL, DIAGNOSTIC, OTHER }
private final String id;
private final TASK_TYPE taskType;
private final String nodeId;
private final String requestId;
private final String name;
private final String os;
private final String address;
private final String type;
private final String type; // Node type (VM, baremetal etc)
private final String provider;
private final SshConfig ssh;
private final NodeRegistryEntry nodeRegistryEntry;
private final List<InstructionsSet> instructionSets;
private final TranslationContext translationContext;
private boolean nodeMustBeInRegistry = true;
private Callable<String> callback;
}

100
ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/ClientInstaller.java
View File

@ -10,6 +10,7 @@
package gr.iccs.imu.ems.baguette.client.install;
import gr.iccs.imu.ems.baguette.server.BaguetteServer;
import gr.iccs.imu.ems.baguette.server.ClientShellCommand;
import gr.iccs.imu.ems.baguette.server.NodeRegistryEntry;
import gr.iccs.imu.ems.brokercep.BrokerCepService;
import gr.iccs.imu.ems.common.plugin.PluginManager;
@ -69,26 +70,49 @@ public class ClientInstaller implements InitializingBean {
executorService.submit(() -> {
long taskCnt = taskCounter.getAndIncrement();
String resultStr = "";
String callbackStr = "";
String errorStr = "";
// Execute task
boolean result = false;
try {
log.info("ClientInstaller: Executing Client installation Task #{}: task-id={}, node-id={}, name={}, type={}, address={}",
taskCnt, task.getId(), task.getNodeId(), task.getName(), task.getType(), task.getAddress());
long startTm = System.currentTimeMillis();
boolean result = executeTask(task, taskCnt);
result = executeTask(task, taskCnt);
long endTm = System.currentTimeMillis();
resultStr = result ? "SUCCESS" : "FAILED";
log.info("ClientInstaller: Client installation Task #{}: result={}, duration={}ms",
taskCnt, resultStr, endTm - startTm);
} catch (Throwable t) {
log.info("ClientInstaller: Exception caught in Client installation Task #{}: Exception: ", taskCnt, t);
errorStr = t.getMessage();
log.error("ClientInstaller: Exception caught in Client installation Task #{}: Exception: ", taskCnt, t);
errorStr = "EXCEPTION " + t.getMessage();
}
// Run callback (if any)
if (result && task.getCallback()!=null) {
try {
log.debug("ClientInstaller: CALLBACK started: Task #{}: task-id={}", taskCnt, task.getId());
long startTm = System.currentTimeMillis();
callbackStr = task.getCallback().call();
long endTm = System.currentTimeMillis();
log.info("ClientInstaller: CALLBACK completed: Task #{}: callback-result={}, duration={}ms", taskCnt, callbackStr, endTm - startTm);
if (! "OK".equalsIgnoreCase(callbackStr)) resultStr = "FAILED";
} catch (Throwable t) {
log.error("ClientInstaller: CALLBACK: Exception caught while running callback of Client installation Task #{}: Exception: ", taskCnt, t);
callbackStr = "CALLBACK-EXCEPTION " + t.getMessage();
resultStr = "FAILED";
}
} else {
if (result)
log.debug("ClientInstaller: No CALLBACK found for Task #{}", taskCnt);
else
log.debug("ClientInstaller: Skipped CALLBACK because execution failed for Task #{}", taskCnt);
}
// Send execution report to local broker
try {
resultStr = StringUtils.defaultIfBlank(resultStr, "ERROR: " + errorStr);
resultStr = StringUtils.defaultIfBlank(resultStr, "ERROR: " + errorStr + " " + callbackStr);
sendSuccessClientInstallationReport(taskCnt, task, resultStr);
} catch (Throwable t) {
log.info("ClientInstaller: EXCEPTION while sending Client installation report for Task #{}: Exception: ", taskCnt, t);
@ -103,7 +127,8 @@ public class ClientInstaller implements InitializingBean {
return executeVmOrBaremetalTask(task, taskCounter);
} else
if ("DIAGNOSTICS".equalsIgnoreCase(task.getType())) {
//if ("DIAGNOSTICS".equalsIgnoreCase(task.getType())) {
if (task.getTaskType()==ClientInstallationTask.TASK_TYPE.DIAGNOSTIC) {
return executeDiagnosticsTask(task, taskCounter);
} else {
log.error("ClientInstaller: UNSUPPORTED TASK TYPE: {}", task.getType());
@ -121,30 +146,55 @@ public class ClientInstaller implements InitializingBean {
if (! task.isNodeMustBeInRegistry())
entry = task.getNodeRegistryEntry();
entry.nodeInstalling(task);
// Call InstallationContextPlugin's before installation
log.debug("ClientInstaller: PRE-INSTALLATION: Calling installation context processors: {}", properties.getInstallationContextProcessorPlugins());
pluginManager.getActivePlugins(InstallationContextProcessorPlugin.class)
.forEach(plugin->((InstallationContextProcessorPlugin)plugin).processBeforeInstallation(task, taskCounter));
log.debug("ClientInstaller: INSTALLATION: Executing installation task: task-counter={}, task={}", taskCounter, task);
boolean success = executeVmTask(task, taskCounter);
log.debug("ClientInstaller: NODE_REGISTRY_ENTRY after installation execution: \n{}", task.getNodeRegistryEntry());
if (entry.getState()==NodeRegistryEntry.STATE.INSTALLING) {
log.warn("ClientInstaller: NODE_REGISTRY_ENTRY status is still INSTALLING after executing client installation. Changing to INSTALL_ERROR");
entry.nodeInstallationError(null);
// Check if node is being removed or have been archived
if (task.getNodeRegistryEntry().isArchived()) {
log.warn("ClientInstaller: Node is being removed or has been archived: {}", properties.getInstallationContextProcessorPlugins());
throw new IllegalStateException("Node is being removed or has been archived: Node IP address: "+ task.getAddress());
}
// Call InstallationContextPlugin's after installation
log.debug("ClientInstaller: POST-INSTALLATION: Calling installation context processors: {}", properties.getInstallationContextProcessorPlugins());
pluginManager.getActivePlugins(InstallationContextProcessorPlugin.class)
.forEach(plugin->((InstallationContextProcessorPlugin)plugin).processAfterInstallation(task, taskCounter, success));
boolean success;
if (! task.getInstructionSets().isEmpty()) {
// Starting installation
entry.nodeInstalling(task);
// Call InstallationContextPlugin's before installation
log.debug("ClientInstaller: PRE-INSTALLATION: Calling installation context processors: {}", properties.getInstallationContextProcessorPlugins());
pluginManager.getActivePlugins(InstallationContextProcessorPlugin.class)
.forEach(plugin -> ((InstallationContextProcessorPlugin) plugin).processBeforeInstallation(task, taskCounter));
log.debug("ClientInstaller: INSTALLATION: Executing installation task: task-counter={}, task={}", taskCounter, task);
success = executeVmTask(task, taskCounter);
log.debug("ClientInstaller: NODE_REGISTRY_ENTRY after installation execution: \n{}", task.getNodeRegistryEntry());
if (entry.getState() == NodeRegistryEntry.STATE.INSTALLING) {
log.warn("ClientInstaller: NODE_REGISTRY_ENTRY status is still INSTALLING after executing client installation. Changing to INSTALL_ERROR");
entry.nodeInstallationError(null);
}
// Call InstallationContextPlugin's after installation
log.debug("ClientInstaller: POST-INSTALLATION: Calling installation context processors: {}", properties.getInstallationContextProcessorPlugins());
pluginManager.getActivePlugins(InstallationContextProcessorPlugin.class)
.forEach(plugin -> ((InstallationContextProcessorPlugin) plugin).processAfterInstallation(task, taskCounter, success));
} else {
log.debug("ClientInstaller: SKIP INSTALLATION: Task has no instructions sets. Skipping execution: Node IP address: "+ task.getAddress());
success = true;
}
// Pre-register Node to baguette Server Coordinator
log.debug("ClientInstaller: POST-INSTALLATION: Node is being pre-registered: {}", entry);
baguetteServer.getNodeRegistry().getCoordinator().preregister(entry);
if (task.getTaskType()==ClientInstallationTask.TASK_TYPE.INSTALL) {
log.debug("ClientInstaller: POST-INSTALLATION: Node is being pre-registered: {}", entry);
baguetteServer.getNodeRegistry().getCoordinator().preregister(entry);
}
// Un-register Node from baguette Server Coordinator
if (task.getTaskType()==ClientInstallationTask.TASK_TYPE.UNINSTALL) {
ClientShellCommand csc = ClientShellCommand.getActiveByIpAddress(entry.getIpAddress());
log.debug("ClientInstaller: POST-INSTALLATION: CSC of node to be unregistered: {}", csc);
if (csc!=null) {
log.debug("ClientInstaller: POST-INSTALLATION: Node is going to be unregistered: {}", entry);
baguetteServer.getNodeRegistry().getCoordinator().unregister(csc);
}
}
log.debug("ClientInstaller: Installation outcome: {}", success ? "Success" : "Error");
return success;

1
ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/api/INodeRegistration.java
View File

@ -15,4 +15,5 @@ import java.util.Map;
public interface INodeRegistration {
String registerNode(String baseUrl, Map<String,Object> nodeInfo, TranslationContext translationContext) throws Exception;
String unregisterNode(String nodeAddress, TranslationContext translationContext) throws Exception;
}

15
ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/helper/AbstractInstallationHelper.java
View File

@ -242,6 +242,21 @@ public abstract class AbstractInstallationHelper implements InitializingBean, Ap
return instructionsSetList;
}
public List<InstructionsSet> prepareUninstallInstructionsForOs(NodeRegistryEntry entry) throws IOException {
if (! entry.getBaguetteServer().isServerRunning()) throw new RuntimeException("Baguette Server is not running");
log.trace("AbstractInstallationHelper.prepareUninstallInstructionsForOs(): node-map={}", entry.getPreregistration());
String osFamily = entry.getPreregistration().get("operatingSystem");
List<InstructionsSet> instructionsSetList = null;
if (matchesOsFamily(osFamily, LINUX_OS_FAMILY))
instructionsSetList = prepareUninstallInstructionsForLinux(entry);
else if (matchesOsFamily(osFamily, WINDOWS_OS_FAMILY))
instructionsSetList = prepareUninstallInstructionsForWin(entry);
else
log.warn("AbstractInstallationHelper.prepareUninstallInstructionsForOs(): Unsupported OS family: {}", osFamily);
return instructionsSetList;
}
public boolean matchesOsFamily(@NonNull String lookup, @NonNull String osFamily) {
lookup = lookup.trim().toUpperCase();
List<String> familyList = properties.getOsFamilies().get(osFamily);

5
ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/helper/InstallationHelper.java
View File

@ -25,8 +25,13 @@ public interface InstallationHelper {
List<InstructionsSet> prepareInstallationInstructionsForWin(NodeRegistryEntry entry);
List<InstructionsSet> prepareInstallationInstructionsForLinux(NodeRegistryEntry entry) throws IOException;
List<InstructionsSet> prepareUninstallInstructionsForOs(NodeRegistryEntry entry) throws IOException;
List<InstructionsSet> prepareUninstallInstructionsForWin(NodeRegistryEntry entry);
List<InstructionsSet> prepareUninstallInstructionsForLinux(NodeRegistryEntry entry) throws IOException;
default ClientInstallationTask createClientInstallationTask(NodeRegistryEntry entry) throws Exception {
return createClientInstallationTask(entry, null);
}
ClientInstallationTask createClientInstallationTask(NodeRegistryEntry entry, TranslationContext translationContext) throws Exception;
ClientInstallationTask createClientUninstallTask(NodeRegistryEntry entry, TranslationContext translationContext) throws Exception;
}

113
ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/helper/VmInstallationHelper.java
View File

@ -22,6 +22,7 @@ import gr.iccs.imu.ems.baguette.server.NodeRegistryEntry;
import gr.iccs.imu.ems.translate.TranslationContext;
import gr.iccs.imu.ems.util.CredentialsMap;
import gr.iccs.imu.ems.util.NetUtil;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.text.StringSubstitutor;
@ -60,7 +61,31 @@ public class VmInstallationHelper extends AbstractInstallationHelper {
@Override
public ClientInstallationTask createClientInstallationTask(NodeRegistryEntry entry, TranslationContext translationContext) throws IOException {
Map<String, String> nodeMap = entry.getPreregistration();
// Get EMS client installation instructions for VM node
List<InstructionsSet> instructionsSetList =
prepareInstallationInstructionsForOs(entry);
return createClientTask(ClientInstallationTask.TASK_TYPE.INSTALL, entry, translationContext, instructionsSetList);
}
@Override
public ClientInstallationTask createClientUninstallTask(NodeRegistryEntry entry, TranslationContext translationContext) throws Exception {
// Clear any cached 'instruction-files' override (from a previous run)
entry.getPreregistration().remove("instruction-files");
// Get EMS client uninstall instructions for VM node
List<InstructionsSet> instructionsSetList =
prepareUninstallInstructionsForOs(entry);
return createClientTask(ClientInstallationTask.TASK_TYPE.UNINSTALL, entry, translationContext, instructionsSetList);
}
private ClientInstallationTask createClientTask(@NonNull ClientInstallationTask.TASK_TYPE taskType,
NodeRegistryEntry entry,
TranslationContext translationContext,
List<InstructionsSet> instructionsSetList)
{
Map<String, String> nodeMap = initializeNodeMap(entry);
String baseUrl = nodeMap.get("BASE_URL");
String clientId = nodeMap.get("CLIENT_ID");
@ -95,13 +120,10 @@ public class VmInstallationHelper extends AbstractInstallationHelper {
if (StringUtils.isEmpty(password) && StringUtils.isBlank(privateKey))
throw new IllegalArgumentException("Missing SSH password or private key for Node");
// Get EMS client installation instructions for VM node
List<InstructionsSet> instructionsSetList =
prepareInstallationInstructionsForOs(entry);
// Create Installation Task for VM node
ClientInstallationTask installationTask = ClientInstallationTask.builder()
ClientInstallationTask task = ClientInstallationTask.builder()
.id(clientId)
.taskType(taskType)
.nodeId(nodeId)
.requestId(requestId)
.name(nodeName)
@ -121,29 +143,21 @@ public class VmInstallationHelper extends AbstractInstallationHelper {
.nodeRegistryEntry(entry)
.translationContext(translationContext)
.build();
log.debug("VmInstallationHelper.createClientInstallationTask(): Created client installation task: {}", installationTask);
return installationTask;
log.debug("VmInstallationHelper.createClientTask(): Created client task: {}", task);
return task;
}
@Override
public List<InstructionsSet> prepareInstallationInstructionsForWin(NodeRegistryEntry entry) {
log.warn("VmInstallationHelper.prepareInstallationInstructionsForWin(): NOT YET IMPLEMENTED");
throw new IllegalArgumentException("VmInstallationHelper.prepareInstallationInstructionsForWin(): NOT YET IMPLEMENTED");
}
@Override
public List<InstructionsSet> prepareInstallationInstructionsForLinux(NodeRegistryEntry entry) throws IOException {
private Map<String, String> initializeNodeMap(NodeRegistryEntry entry) {
Map<String, String> nodeMap = entry.getPreregistration();
BaguetteServer baguette = entry.getBaguetteServer();
String baseUrl = StringUtils.removeEnd(nodeMap.get("BASE_URL"), "/");
String clientId = nodeMap.get("CLIENT_ID");
String ipSetting = nodeMap.get("IP_SETTING");
log.debug("VmInstallationHelper.prepareInstallationInstructionsForLinux(): Invoked: base-url={}", baseUrl);
log.debug("VmInstallationHelper.initializeNodeMap(): Invoked: base-url={}", baseUrl);
// Get parameters
log.trace("VmInstallationHelper.prepareInstallationInstructionsForLinux(): properties: {}", properties);
log.trace("VmInstallationHelper.initializeNodeMap(): properties: {}", properties);
String rootCmd = properties.getRootCmd();
String baseDir = properties.getBaseDir();
String checkInstallationFile = properties.getCheckInstalledFile();
@ -170,7 +184,7 @@ public class VmInstallationHelper extends AbstractInstallationHelper {
: "NODE_" + e.getKey().toUpperCase(),
Map.Entry::getValue,
(v1, v2) -> {
log.warn("VmInstallationHelper.prepareInstallationInstructionsForLinux(): DUPLICATE KEY FOUND: key={}, old-value={}, new-value={}",
log.warn("VmInstallationHelper.initializeNodeMap(): DUPLICATE KEY FOUND: key={}, old-value={}, new-value={}",
k, v1, v2);
return v2;
}
@ -182,12 +196,12 @@ public class VmInstallationHelper extends AbstractInstallationHelper {
? "NODE_SSH_" + k.substring(4).toUpperCase()
: "NODE_" + k.toUpperCase();
if (additionalKeysMap.containsKey(k)) {
log.warn("VmInstallationHelper.prepareInstallationInstructionsForLinux(): DUPLICATE KEY FOUND: key={}, old-value={}, new-value={}",
log.warn("VmInstallationHelper.initializeNodeMap(): DUPLICATE KEY FOUND: key={}, old-value={}, new-value={}",
k, additionalKeysMap.get(k), v);
}
additionalKeysMap.put(k, v);
} catch (Exception ex) {
log.error("VmInstallationHelper.prepareInstallationInstructionsForLinux(): EXCEPTION in additional keys copy loop: key={}, value={}, additionalKeysMap={}, Exception:\n",
log.error("VmInstallationHelper.initializeNodeMap(): EXCEPTION in additional keys copy loop: key={}, value={}, additionalKeysMap={}, Exception:\n",
k, v, additionalKeysMap, ex);
}
});
@ -229,7 +243,7 @@ public class VmInstallationHelper extends AbstractInstallationHelper {
nodeMap.putAll(clientInstallationProperties.getParameters());
nodeMap.put("EMS_PUBLIC_DIR", System.getProperty("PUBLIC_DIR", System.getenv("PUBLIC_DIR")));
log.trace("VmInstallationHelper.prepareInstallationInstructionsForLinux: value-map: {}", nodeMap);
log.trace("VmInstallationHelper.initializeNodeMap: value-map: {}", nodeMap);
/* // Clear EMS server certificate (PEM) file, if not secure
if (!isServerSecure) {
@ -251,26 +265,65 @@ public class VmInstallationHelper extends AbstractInstallationHelper {
}
}*/
return nodeMap;
}
@Override
public List<InstructionsSet> prepareInstallationInstructionsForWin(NodeRegistryEntry entry) {
log.warn("VmInstallationHelper.prepareInstallationInstructionsForWin(): NOT YET IMPLEMENTED");
throw new IllegalArgumentException("VmInstallationHelper.prepareInstallationInstructionsForWin(): NOT YET IMPLEMENTED");
}
@Override
public List<InstructionsSet> prepareUninstallInstructionsForWin(NodeRegistryEntry entry) {
log.warn("VmInstallationHelper.prepareUninstallInstructionsForWin(): NOT YET IMPLEMENTED");
throw new IllegalArgumentException("VmInstallationHelper.prepareUninstallInstructionsForWin(): NOT YET IMPLEMENTED");
}
@Override
public List<InstructionsSet> prepareInstallationInstructionsForLinux(NodeRegistryEntry entry) throws IOException {
return prepareInstructionsForLinux(entry, "LINUX");
}
@Override
public List<InstructionsSet> prepareUninstallInstructionsForLinux(NodeRegistryEntry entry) throws IOException {
return prepareInstructionsForLinux(entry, "REMOVE_LINUX");
}
private List<InstructionsSet> prepareInstructionsForLinux(@NonNull NodeRegistryEntry entry, String instructionsScenarioName) throws IOException {
Map<String, String> nodeMap = entry.getPreregistration();
List<InstructionsSet> instructionsSetList = new ArrayList<>();
try {
// Read installation instructions from JSON file
List<String> instructionSetFileList = null;
List<String> instructionSetFileList;
if (nodeMap.containsKey("instruction-files")) {
log.trace("VmInstallationHelper.prepareInstructionsForLinux: FOUND instruction-files override: value={}", nodeMap.getOrDefault("instruction-files", null));
instructionSetFileList = Arrays.stream(nodeMap.getOrDefault("instruction-files", "").split(","))
.filter(StringUtils::isNotBlank)
.map(String::trim)
.collect(Collectors.toList());
log.debug("VmInstallationHelper.prepareInstructionsForLinux: FOUND instruction-files override: list={}", instructionSetFileList);
if (instructionSetFileList.isEmpty())
log.warn("VmInstallationHelper.prepareInstallationInstructionsForLinux: Context map contains 'instruction-files' entry with no contents");
log.warn("VmInstallationHelper.prepareInstructionsForLinux: Context map contains 'instruction-files' entry with no contents");
} else {
instructionSetFileList = properties.getInstructions().get("LINUX");
log.trace("VmInstallationHelper.prepareInstructionsForLinux: NOT FOUND instruction-files override. Using configured instructions sets: instructionsScenarioName={}", instructionsScenarioName);
instructionSetFileList = properties.getInstructions().get(instructionsScenarioName);
log.trace("VmInstallationHelper.prepareInstructionsForLinux: NOT FOUND instruction-files override. Using configured instructions sets: list={}", instructionSetFileList);
if (instructionSetFileList==null || instructionSetFileList.isEmpty()) {
log.warn("VmInstallationHelper.prepareInstructionsForLinux: No instructions set files provided in configuration with name: {}", instructionsScenarioName);
instructionSetFileList = Collections.emptyList();
}
}
log.debug("VmInstallationHelper.prepareInstructionsForLinux: Instructions sets list: {}", instructionSetFileList);
// Load instructions set from file
for (String instructionSetFile : instructionSetFileList) {
// Load instructions set from file
log.debug("VmInstallationHelper.prepareInstallationInstructionsForLinux: Installation instructions file for LINUX: {}", instructionSetFile);
log.debug("VmInstallationHelper.prepareInstructionsForLinux: Installation instructions file for LINUX: {}", instructionSetFile);
InstructionsSet instructionsSet = InstructionsService.getInstance().loadInstructionsFile(instructionSetFile);
log.debug("VmInstallationHelper.prepareInstallationInstructionsForLinux: Instructions set loaded from file: {}\n{}", instructionSetFile, instructionsSet);
log.debug("VmInstallationHelper.prepareInstructionsForLinux: Instructions set loaded from file: {}\n{}", instructionSetFile, instructionsSet);
// Pretty print instructionsSet JSON
if (log.isTraceEnabled()) {
@ -279,7 +332,7 @@ public class VmInstallationHelper extends AbstractInstallationHelper {
try (PrintWriter writer = new PrintWriter(stringWriter)) {
gson.toJson(instructionsSet, writer);
}
log.trace("VmInstallationHelper.prepareInstallationInstructionsForLinux: Installation instructions for LINUX: json:\n{}", stringWriter);
log.trace("VmInstallationHelper.prepareInstructionsForLinux: Installation instructions for LINUX: json:\n{}", stringWriter);
}
instructionsSetList.add(instructionsSet);
@ -287,7 +340,7 @@ public class VmInstallationHelper extends AbstractInstallationHelper {
return instructionsSetList;
} catch (Exception ex) {
log.error("VmInstallationHelper.prepareInstallationInstructionsForLinux: Exception while reading Installation instructions for LINUX: ", ex);
log.error("VmInstallationHelper.prepareInstructionsForLinux: Exception while reading Installation instructions for LINUX: ", ex);
throw ex;
}
}

65
ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/selfhealing/ClientRecoveryPlugin.java
View File

@ -48,8 +48,9 @@ public class ClientRecoveryPlugin implements InitializingBean, EventBus.EventCon
private long clientRecoveryDelay;
private String recoveryInstructionsFile;
private final static String CLIENT_EXIT_TOPIC = "BAGUETTE_SERVER_CLIENT_EXITED";
private final static String CLIENT_REGISTERED_TOPIC = "BAGUETTE_SERVER_CLIENT_REGISTERED";
public final static String CLIENT_EXIT_TOPIC = "BAGUETTE_SERVER_CLIENT_EXITED";
public final static String CLIENT_REGISTERED_TOPIC = "BAGUETTE_SERVER_CLIENT_REGISTERED";
public final static String CLIENT_REMOVED_TOPIC = "BAGUETTE_SERVER_CLIENT_REMOVED";
@Override
public void afterPropertiesSet() throws Exception {
@ -61,6 +62,8 @@ public class ClientRecoveryPlugin implements InitializingBean, EventBus.EventCon
log.debug("ClientRecoveryPlugin: Subscribed for BAGUETTE_SERVER_CLIENT_EXITED events");
eventBus.subscribe(CLIENT_REGISTERED_TOPIC, this);
log.debug("ClientRecoveryPlugin: Subscribed for BAGUETTE_SERVER_CLIENT_REGISTERED events");
eventBus.subscribe(CLIENT_REMOVED_TOPIC, this);
log.debug("ClientRecoveryPlugin: Subscribed for CLIENT_REMOVED_TOPIC events");
log.trace("ClientRecoveryPlugin: clientInstallationProperties: {}", clientInstallationProperties);
log.trace("ClientRecoveryPlugin: baguetteServer: {}", baguetteServer);
@ -80,18 +83,28 @@ public class ClientRecoveryPlugin implements InitializingBean, EventBus.EventCon
}
// Only process messages of ClientShellCommand type are accepted (sent by CSC instances)
if (! (message instanceof ClientShellCommand)) {
log.warn("ClientRecoveryPlugin: onMessage(): Message is not a {} object. Will ignore it.", ClientShellCommand.class.getSimpleName());
if (! (message instanceof NodeRegistryEntry) && ! (message instanceof ClientShellCommand)) {
log.warn("ClientRecoveryPlugin: onMessage(): Message is neither a {} or a {} object. Will ignore it.",
NodeRegistryEntry.class.getSimpleName(), ClientShellCommand.class.getSimpleName());
return;
}
// Get NodeRegistryEntry from ClientShellCommand passed with event
ClientShellCommand csc = (ClientShellCommand)message;
String clientId = csc.getId();
String address = csc.getClientIpAddress();
log.debug("ClientRecoveryPlugin: onMessage(): client-id={}, client-address={}", clientId, address);
NodeRegistryEntry nodeInfo;
String clientId;
String address;
if (message instanceof NodeRegistryEntry entry) {
nodeInfo = entry;
clientId = entry.getClientId();
address = entry.getIpAddress();
} else {
// Get NodeRegistryEntry from ClientShellCommand passed with event
ClientShellCommand csc = (ClientShellCommand) message;
clientId = csc.getId();
address = csc.getClientIpAddress();
log.debug("ClientRecoveryPlugin: onMessage(): client-id={}, client-address={}", clientId, address);
NodeRegistryEntry nodeInfo = csc.getNodeRegistryEntry(); //or = nodeRegistry.getNodeByAddress(address);
nodeInfo = csc.getNodeRegistryEntry(); //or = nodeRegistry.getNodeByAddress(address);
}
log.debug("ClientRecoveryPlugin: onMessage(): client-node-info={}", nodeInfo);
log.trace("ClientRecoveryPlugin: onMessage(): node-registry.node-addresses={}", nodeRegistry.getNodeAddresses());
log.trace("ClientRecoveryPlugin: onMessage(): node-registry.nodes={}", nodeRegistry.getNodes());
@ -108,14 +121,25 @@ public class ClientRecoveryPlugin implements InitializingBean, EventBus.EventCon
processExitEvent(nodeInfo);
}
if (CLIENT_REGISTERED_TOPIC.equals(topic)) {
log.debug("ClientRecoveryPlugin: onMessage(): CLIENT REGISTERED_TOPIC: message={}", message);
log.debug("ClientRecoveryPlugin: onMessage(): CLIENT REGISTERED: message={}", message);
processRegisteredEvent(nodeInfo);
}
if (CLIENT_REMOVED_TOPIC.equals(topic)) {
log.debug("ClientRecoveryPlugin: onMessage(): CLIENT REMOVED: message={}", message);
processRemovedEvent(nodeInfo);
}
}
private void processExitEvent(NodeRegistryEntry nodeInfo) {
log.debug("ClientRecoveryPlugin: processExitEvent(): BEGIN: client-id={}, client-address={}", nodeInfo.getClientId(), nodeInfo.getIpAddress());
// Check if node can be recovered (based on its Life-Cycle state)
if (! nodeInfo.canRecover()) {
log.warn("ClientRecoveryPlugin: processExitEvent(): Node will not be recovered because its state is {}: client-id={}, client-address={}",
nodeInfo.getState(), nodeInfo.getClientId(), nodeInfo.getIpAddress());
return;
}
// Set node state to DOWN
baguetteServer.getSelfHealingManager().setNodeSelfHealingState(nodeInfo, SelfHealingManager.NODE_STATE.DOWN);
@ -156,10 +180,29 @@ public class ClientRecoveryPlugin implements InitializingBean, EventBus.EventCon
baguetteServer.getSelfHealingManager().setNodeSelfHealingState(nodeInfo, SelfHealingManager.NODE_STATE.UP);
}
private void processRemovedEvent(NodeRegistryEntry nodeInfo) {
log.debug("ClientRecoveryPlugin: processRemovedEvent(): BEGIN: client-id={}, client-address={}", nodeInfo.getClientId(), nodeInfo.getIpAddress());
// Cancel any pending recovery task (for the node)
ScheduledFuture<?> future = pendingTasks.remove(nodeInfo);
log.debug("ClientRecoveryPlugin: processRemovedEvent(): Recovery task: task={}, client-id={}, client-address={}", future, nodeInfo.getClientId(), nodeInfo.getIpAddress());
if (future!=null && ! future.isDone() && ! future.isCancelled()) {
log.warn("ClientRecoveryPlugin: processRemovedEvent(): Cancelled recovery task: client-id={}, client-address={}", nodeInfo.getClientId(), nodeInfo.getIpAddress());
future.cancel(false);
}
}
public void runClientRecovery(NodeRegistryEntry entry) throws Exception {
log.debug("ClientRecoveryPlugin: runClientRecovery(): node-info={}", entry);
if (entry==null) return;
if (! entry.canRecover()) {
log.info("ClientRecoveryPlugin: runClientRecovery(): Cannot recover node. Will not attempt recovery again: node-state={}, client-id={}, client-address={}",
entry.getState(), entry.getClientId(), entry.getIpAddress());
pendingTasks.remove(entry);
return;
}
log.trace("ClientRecoveryPlugin: runClientRecovery(): recoveryInstructionsFile={}", recoveryInstructionsFile);
entry.getPreregistration().put("instruction-files", recoveryInstructionsFile);

9
ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/BaguetteServer.java
View File

@ -550,4 +550,13 @@ public class BaguetteServer implements InitializingBean, EventBus.EventConsumer<
}
return clientId;
}
public NodeRegistryEntry unregisterClient(NodeRegistryEntry entry) throws UnknownHostException {
log.debug("BaguetteServer.unregisterClient(): entry={}", entry);
// Archive node entry in node registry
nodeRegistry.archiveNode(entry);
return entry;
}
}

12
ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/ClientShellCommand.java
View File

@ -265,8 +265,16 @@ public class ClientShellCommand implements Command, Runnable, ServerSessionAware
}
}
// Client connection closed
eventBus.send("BAGUETTE_SERVER_CLIENT_EXITING", this);
getNodeRegistryEntry().nodeExiting(null);
try {
eventBus.send("BAGUETTE_SERVER_CLIENT_EXITING", this);
NodeRegistryEntry entry = getNodeRegistryEntry();
if (! entry.isArchived())
entry.nodeExiting(null);
else
log.warn("{}==> Node is archived", id);
} catch (Exception e) {
log.warn("{}==> EXCEPTION: ", id, e);
}
log.info("{}==> Signaling client to exit", id);
out.println("EXIT");

29
ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/NodeRegistry.java
View File

@ -10,6 +10,7 @@
package gr.iccs.imu.ems.baguette.server;
import lombok.Getter;
import lombok.NonNull;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
@ -17,9 +18,7 @@ import org.springframework.stereotype.Service;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.*;
import java.util.stream.Collectors;
/**
@ -29,6 +28,7 @@ import java.util.stream.Collectors;
@Service
public class NodeRegistry {
private final Map<String,NodeRegistryEntry> registry = new LinkedHashMap<>();
private final List<NodeRegistryEntry> archived = new LinkedList<>();
@Getter @Setter
private ServerCoordinator coordinator;
@ -134,4 +134,27 @@ public class NodeRegistry {
public Collection<String> getNodeReferences() {
return registry.values().stream().map(NodeRegistryEntry::getReference).collect(Collectors.toList());
}
public synchronized void archiveNode(@NonNull NodeRegistryEntry entry) {
log.debug("NodeRegistry.archiveNode(): Archiving node: entry={}", entry);
// Run checks
if (StringUtils.isBlank(entry.getIpAddress())) {
log.warn("NodeRegistry.archiveNode(): Node entry does not have an IP address. Ignoring archiving operation: entry={}", entry);
return;
}
if (! registry.containsKey(entry.getIpAddress())) {
log.warn("NodeRegistry.archiveNode(): Node with IP address {} not found in registry. Ignoring archiving operation: entry={}", entry.getIpAddress(), entry);
return;
}
if (registry.get(entry.getIpAddress())!=entry) {
log.warn("NodeRegistry.archiveNode(): Node passed does not match with node registered to IP address {}. Ignoring archiving operation: entry={}", entry.getIpAddress(), entry);
return;
}
// Archive node
archived.add(entry);
registry.remove(entry.getIpAddress());
entry.nodeArchived(null);
}
}

130
ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/NodeRegistryEntry.java
View File

@ -21,8 +21,12 @@ import java.util.*;
@AllArgsConstructor
@RequiredArgsConstructor
public class NodeRegistryEntry {
public enum STATE { PREREGISTERED, IGNORE_NODE, INSTALLING, NOT_INSTALLED, INSTALLED, INSTALL_ERROR,
WAITING_REGISTRATION, REGISTERING, REGISTERED, REGISTRATION_ERROR, DISCONNECTED, EXITING, EXITED, NODE_FAILED
public enum STATE {
PREREGISTERED, IGNORE_NODE,
INSTALLING, NOT_INSTALLED, INSTALLED, INSTALL_ERROR,
WAITING_REGISTRATION, REGISTERING, REGISTERED, REGISTRATION_ERROR,
DISCONNECTED, EXITING, EXITED, NODE_FAILED,
REMOVING, REMOVED, REMOVE_ERROR, ARCHIVED
};
@Getter private final String ipAddress;
@Getter private final String clientId;
@ -40,6 +44,8 @@ public class NodeRegistryEntry {
@JsonIgnore
@Getter private transient Map<String, String> registration = new LinkedHashMap<>();
@JsonIgnore
@Getter private transient Map<String, String> removal = new LinkedHashMap<>();
@JsonIgnore
@Getter @Setter private transient IClusterZone clusterZone;
@JsonIgnore
@ -70,100 +76,118 @@ public class NodeRegistryEntry {
public void refreshReference() { reference = UUID.randomUUID().toString(); }
public NodeRegistryEntry nodePreregistration(Map<String,Object> nodeInfo) {
preregistration.clear();
preregistration.putAll(StrUtil.deepFlattenMap(nodeInfo));
setState(STATE.PREREGISTERED);
public boolean canRecover() {
return state != null && switch (state) {
case PREREGISTERED, IGNORE_NODE -> false;
case REMOVING, REMOVED, REMOVE_ERROR, ARCHIVED -> false;
default -> true;
};
}
public boolean isArchived() {
return state!=null && state==STATE.ARCHIVED;
}
public boolean canChangeStateTo(@NonNull STATE newState) {
return ! isArchived();
}
private void _canUpdateEntry(@NonNull STATE newState) {
if (! canChangeStateTo(newState)) {
throw new IllegalStateException(String.format("Cannot change NodeRegistryEntry state from %s to %s: client-id=%s, client-address=%s",
state, newState, clientId, ipAddress));
}
}
private NodeRegistryEntry _updateEntry(@NonNull STATE newState, @NonNull Map<String, String> map, boolean clear, Map<String,Object> nodeInfo) {
_canUpdateEntry(newState);
if (clear) map.clear();
map.putAll(StrUtil.deepFlattenMap(nodeInfo));
setState(newState);
return this;
}
public NodeRegistryEntry nodeIgnore(Object nodeInfo) {
installation.clear();
installation.put("ignore-node", nodeInfo!=null ? nodeInfo.toString() : null);
private NodeRegistryEntry _updateEntry(@NonNull STATE newState, @NonNull Map<String, String> map, boolean clear, @NonNull String key, Object val, String defVal) {
_canUpdateEntry(newState);
if (clear) map.clear();
map.put(key, val!=null ? val.toString() : defVal);
setState(STATE.IGNORE_NODE);
return this;
}
public NodeRegistryEntry nodePreregistration(Map<String,Object> nodeInfo) {
return _updateEntry(STATE.PREREGISTERED, preregistration, true, nodeInfo);
}
public NodeRegistryEntry nodeIgnore(Object nodeInfo) {
return _updateEntry(STATE.IGNORE_NODE, installation, true, "ignore-node", nodeInfo, null);
}
public NodeRegistryEntry nodeInstalling(Object nodeInfo) {
installation.clear();
installation.put("installation-task", nodeInfo!=null ? nodeInfo.toString() : "INSTALLING");
setState(STATE.INSTALLING);
return this;
return _updateEntry(STATE.INSTALLING, installation, true, "installation-task", nodeInfo, "INSTALLING");
}
public NodeRegistryEntry nodeNotInstalled(Object nodeInfo) {
installation.clear();
installation.put("installation-task-result", nodeInfo!=null ? nodeInfo.toString() : "NOT_INSTALLED");
setState(STATE.NOT_INSTALLED);
return this;
return _updateEntry(STATE.NOT_INSTALLED, installation, true, "installation-task-result", nodeInfo, "NOT_INSTALLED");
}
public NodeRegistryEntry nodeInstallationComplete(Object nodeInfo) {
installation.put("installation-task-result", nodeInfo!=null ? nodeInfo.toString() : "SUCCESS");
setState(STATE.INSTALLED);
return this;
return _updateEntry(STATE.INSTALLED, installation, false, "installation-task-result", nodeInfo, "SUCCESS");
}
public NodeRegistryEntry nodeInstallationError(Object nodeInfo) {
installation.put("installation-task-result", nodeInfo!=null ? nodeInfo.toString() : "ERROR");
setState(STATE.INSTALL_ERROR);
return this;
return _updateEntry(STATE.INSTALL_ERROR, installation, false, "installation-task-result", nodeInfo, "ERROR");
}
public NodeRegistryEntry nodeRegistering(Map<String,Object> nodeInfo) {
registration.clear();
registration.putAll(StrUtil.deepFlattenMap(nodeInfo));
setState(STATE.REGISTERING);
return this;
return _updateEntry(STATE.REGISTERING, registration, true, nodeInfo);
}
public NodeRegistryEntry nodeRegistered(Map<String,Object> nodeInfo) {
//registration.clear();
registration.putAll(StrUtil.deepFlattenMap(nodeInfo));
setState(STATE.REGISTERED);
return this;
return _updateEntry(STATE.REGISTERED, registration, false, nodeInfo);
}
public NodeRegistryEntry nodeRegistrationError(Map<String,Object> nodeInfo) {
registration.putAll(StrUtil.deepFlattenMap(nodeInfo));
setState(STATE.REGISTRATION_ERROR);
return this;
return _updateEntry(STATE.REGISTRATION_ERROR, registration, false, nodeInfo);
}
public NodeRegistryEntry nodeRegistrationError(Throwable t) {
registration.putAll(StrUtil.deepFlattenMap(Collections.singletonMap("exception", t)));
setState(STATE.REGISTRATION_ERROR);
return this;
return _updateEntry(STATE.REGISTRATION_ERROR, registration, false, Collections.singletonMap("exception", t));
}
public NodeRegistryEntry nodeDisconnected(Map<String,Object> nodeInfo) {
registration.putAll(StrUtil.deepFlattenMap(nodeInfo));
setState(STATE.DISCONNECTED);
return this;
return _updateEntry(STATE.DISCONNECTED, registration, false, nodeInfo);
}
public NodeRegistryEntry nodeDisconnected(Throwable t) {
registration.putAll(StrUtil.deepFlattenMap(Collections.singletonMap("exception", t)));
setState(STATE.DISCONNECTED);
return this;
return _updateEntry(STATE.DISCONNECTED, registration, false, Collections.singletonMap("exception", t));
}
public NodeRegistryEntry nodeExiting(Map<String,Object> nodeInfo) {
registration.putAll(StrUtil.deepFlattenMap(nodeInfo));
setState(STATE.EXITING);
return this;
return _updateEntry(STATE.EXITING, registration, false, nodeInfo);
}
public NodeRegistryEntry nodeExited(Map<String,Object> nodeInfo) {
registration.putAll(StrUtil.deepFlattenMap(nodeInfo));
setState(STATE.EXITED);
return this;
return _updateEntry(STATE.EXITED, registration, false, nodeInfo);
}
public NodeRegistryEntry nodeFailed(Map<String,Object> failInfo) {
if (failInfo!=null)
registration.putAll(StrUtil.deepFlattenMap(failInfo));
setState(STATE.NODE_FAILED);
return this;
return _updateEntry(STATE.NODE_FAILED, registration, false, failInfo);
}
public NodeRegistryEntry nodeRemoving(Map<String,Object> nodeInfo) {
return _updateEntry(STATE.REMOVING, removal, false, nodeInfo);
}
public NodeRegistryEntry nodeRemoved(Map<String,Object> nodeInfo) {
return _updateEntry(STATE.REMOVED, removal, false, nodeInfo);
}
public NodeRegistryEntry nodeRemoveError(Map<String,Object> failInfo) {
return _updateEntry(STATE.REMOVE_ERROR, removal, false, failInfo);
}
public NodeRegistryEntry nodeArchived(Map<String,Object> nodeInfo) {
return _updateEntry(STATE.ARCHIVED, removal, false, nodeInfo);
}
}

22
ems-core/control-service/src/main/java/gr/iccs/imu/ems/control/controller/NodeRegistrationController.java
View File

@ -53,7 +53,7 @@ public class NodeRegistrationController {
return "OK";
}
@RequestMapping(value = "/baguette/registerNode", method = POST,
@RequestMapping(value = { "/baguette/registerNode", "/baguette/node/register" }, method = POST,
consumes = MediaType.APPLICATION_JSON_VALUE)
public String baguetteRegisterNode(@RequestBody String jsonNode, HttpServletRequest request) throws Exception {
log.info("NodeRegistrationController.baguetteRegisterNode(): Invoked");
@ -80,6 +80,26 @@ public class NodeRegistrationController {
return response;
}
@RequestMapping(value = "/baguette/node/unregister/{ipAddress:.+}", method = {GET, POST},
produces = MediaType.TEXT_PLAIN_VALUE)
public String baguetteUnregisterNode(@PathVariable String ipAddress, HttpServletRequest request) throws Exception {
log.info("NodeRegistrationController.baguetteUnregisterNode(): Invoked");
log.debug("NodeRegistrationController.baguetteUnregisterNode(): Node IP address:\n{}", ipAddress);
String response;
try {
response = nodeRegistrationCoordinator.unregisterNode(ipAddress,
coordinator.getTranslationContextOfAppModel(coordinator.getCurrentAppModelId()));
} catch (Exception e) {
log.error("NodeRegistrationController.baguetteUnregisterNode(): EXCEPTION while unregistering node: address={}\n", ipAddress, e);
response = "ERROR "+e.getMessage();
}
log.info("NodeRegistrationController.baguetteUnregisterNode(): Node unregistered: node-address={}", ipAddress);
log.debug("NodeRegistrationController.baguetteUnregisterNode(): address={}, json={}", ipAddress, response);
return response;
}
@RequestMapping(value = "/baguette/node/list", method = GET)
public Collection<String> baguetteNodeList() throws Exception {
log.info("NodeRegistrationController.baguetteNodeList(): Invoked");

81
ems-core/control-service/src/main/java/gr/iccs/imu/ems/control/controller/NodeRegistrationCoordinator.java
View File

@ -12,15 +12,19 @@ package gr.iccs.imu.ems.control.controller;
import gr.iccs.imu.ems.baguette.client.install.ClientInstallationTask;
import gr.iccs.imu.ems.baguette.client.install.ClientInstaller;
import gr.iccs.imu.ems.baguette.client.install.helper.InstallationHelperFactory;
import gr.iccs.imu.ems.baguette.client.selfhealing.ClientRecoveryPlugin;
import gr.iccs.imu.ems.baguette.server.BaguetteServer;
import gr.iccs.imu.ems.baguette.server.ClientShellCommand;
import gr.iccs.imu.ems.baguette.server.NodeRegistryEntry;
import gr.iccs.imu.ems.control.properties.ControlServiceProperties;
import gr.iccs.imu.ems.control.properties.StaticResourceProperties;
import gr.iccs.imu.ems.translate.TranslationContext;
import gr.iccs.imu.ems.util.EventBus;
import gr.iccs.imu.ems.util.NetUtil;
import gr.iccs.imu.ems.util.StrUtil;
import jakarta.servlet.http.HttpServletRequest;
import lombok.Getter;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@ -35,6 +39,7 @@ import org.springframework.web.servlet.support.ServletUriComponentsBuilder;
import java.io.IOException;
import java.util.Collections;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicBoolean;
@Slf4j
@ -51,6 +56,8 @@ public class NodeRegistrationCoordinator implements InitializingBean {
private final ServerProperties serverProperties;
private final ServletWebServerApplicationContext webServerAppCtxt;
private final EventBus<String,Object,Object> eventBus;
@Override
public void afterPropertiesSet() throws Exception {
}
@ -130,6 +137,60 @@ public class NodeRegistrationCoordinator implements InitializingBean {
return response;
}
public String unregisterNode(@NonNull String ipAddress, TranslationContext translationContext) throws Exception {
log.debug("NodeRegistrationCoordinator.unregisterNode(): BEGIN: ip-address={}", ipAddress);
if (StringUtils.isBlank(ipAddress)) {
log.error("NodeRegistrationCoordinator.unregisterNode(): Blank IP address provided");
throw new IllegalArgumentException("Blank IP address provided");
}
// Retrieve node entry from registry
NodeRegistryEntry entry = baguetteServer.getNodeRegistry().getNodeByAddress(ipAddress);
if (entry==null) {
log.error("NodeRegistrationCoordinator.unregisterNode(): Node not found in registry: address={}", ipAddress);
throw new IllegalArgumentException("Node not found in registry: address="+ipAddress);
}
// Set node state to REMOVING in order to prevent self-healing
entry.nodeRemoving(null);
// Signal self-healing plugin to cancel any pending recovery tasks
log.debug("NodeRegistrationCoordinator.unregisterNode(): Notifying Self-healing to remove any pending recovery tasks: address={}", ipAddress);
eventBus.send(ClientRecoveryPlugin.CLIENT_REMOVED_TOPIC, entry);
// Close CSC connection
ClientShellCommand csc = ClientShellCommand.getActiveByIpAddress(entry.getIpAddress());
if (csc!=null) {
log.info("NodeRegistrationCoordinator.unregisterNode(): Closing connection to EMS client: address={}", ipAddress);
csc.stop("REMOVING NODE");
} else
log.warn("NodeRegistrationCoordinator.unregisterNode(): CSC is null. Cannot close connection to EMS client. Probably connection has already closed: address={}", ipAddress);
// Continue processing according to ExecutionWare type
String response;
log.info("NodeRegistrationCoordinator.unregisterNode(): ExecutionWare: {}", properties.getExecutionware());
if (properties.getExecutionware()==ControlServiceProperties.ExecutionWare.CLOUDIATOR) {
response = "NOT SUPPORTED";
} else {
response = createClientUninstallTask(entry, translationContext, () -> {
// Unregister and Archive node
try {
entry.nodeRemoved(null);
baguetteServer.unregisterClient(entry);
return "OK";
} catch (Exception e) {
log.error("NodeRegistrationCoordinator.unregisterNode(): EXCEPTION while unregistering node: address={}, entry={}\n", ipAddress, entry, e);
entry.nodeRemoveError(Map.of(
"error", e.getMessage()
));
return "ERROR " + e.getMessage();
}
});
}
return response;
}
void updateRegistrationInfo(Map<String, String> nodeMap, String baseUrl) {
// Set OS info
String os = StringUtils.isNotBlank(nodeMap.get("operatingSystem.name"))
@ -211,14 +272,32 @@ public class NodeRegistrationCoordinator implements InitializingBean {
}
public String createClientInstallationTask(NodeRegistryEntry entry, TranslationContext translationContext) throws Exception {
//log.info("ControlServiceController.baguetteRegisterNodeForProactive(): INPUT: node-map: {}", nodeMap);
return createClientInstallationTask(entry, translationContext, null);
}
public String createClientInstallationTask(NodeRegistryEntry entry, TranslationContext translationContext, Callable<String> callback) throws Exception {
ClientInstallationTask installationTask = InstallationHelperFactory.getInstance()
.createInstallationHelper(entry)
.createClientInstallationTask(entry, translationContext);
installationTask.setCallback(callback);
ClientInstaller.instance().addTask(installationTask);
log.debug("NodeRegistrationCoordinator.createClientInstallationTask(): New installation-task: {}", installationTask);
return "OK";
}
public String createClientUninstallTask(NodeRegistryEntry entry, TranslationContext translationContext) throws Exception {
return createClientUninstallTask(entry, translationContext, null);
}
public String createClientUninstallTask(NodeRegistryEntry entry, TranslationContext translationContext, Callable<String> callback) throws Exception {
ClientInstallationTask uninstallTask = InstallationHelperFactory.getInstance()
.createInstallationHelper(entry)
.createClientUninstallTask(entry, translationContext);
uninstallTask.setCallback(callback);
ClientInstaller.instance().addTask(uninstallTask);
log.debug("NodeRegistrationCoordinator.createClientUninstallTask(): New uninstall-task: {}", uninstallTask);
return "OK";
}
}

7
ems-core/control-service/src/main/java/gr/iccs/imu/ems/control/util/NodeRegistrationHelper.java
View File

@ -36,4 +36,11 @@ public class NodeRegistrationHelper implements InitializingBean, INodeRegistrati
baseUrl, nodeInfo, translationContext);
return nodeRegistrationCoordinator.registerNode(baseUrl, nodeInfo, translationContext);
}
@Override
public String unregisterNode(String nodeAddress, TranslationContext translationContext) throws Exception {
log.debug("NodeRegistrationHelper: Invoking unregisterNode: node-address={}, TC={}",
nodeAddress, translationContext);
return nodeRegistrationCoordinator.unregisterNode(nodeAddress, translationContext);
}
}