EMS: Implementing node removal (off-boarding) [WIP]
This commit is contained in:
parent
3083d7dad3
commit
271ae82855
ems-core
baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client
install
selfhealing
baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server
control-service/src/main/java/gr/iccs/imu/ems/control
@ -130,6 +130,7 @@ public class ClientInstallationRequestListener implements InitializingBean {
|
|||||||
switch (requestType) {
|
switch (requestType) {
|
||||||
case "DIAGNOSTICS" -> processDiagnosticsRequest(request);
|
case "DIAGNOSTICS" -> processDiagnosticsRequest(request);
|
||||||
case "VM" -> processOnboardingRequest(request);
|
case "VM" -> processOnboardingRequest(request);
|
||||||
|
case "REMOVE" -> processRemoveRequest(request);
|
||||||
default -> throw new IllegalArgumentException("Unsupported request type: "+requestType);
|
default -> throw new IllegalArgumentException("Unsupported request type: "+requestType);
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -174,6 +175,7 @@ public class ClientInstallationRequestListener implements InitializingBean {
|
|||||||
// Create client installation task
|
// Create client installation task
|
||||||
ClientInstallationTask newTask = ClientInstallationTask.builder()
|
ClientInstallationTask newTask = ClientInstallationTask.builder()
|
||||||
.id(request.get("requestId"))
|
.id(request.get("requestId"))
|
||||||
|
.taskType(ClientInstallationTask.TASK_TYPE.DIAGNOSTIC)
|
||||||
.requestId(request.get("requestId"))
|
.requestId(request.get("requestId"))
|
||||||
.type(request.get("requestType"))
|
.type(request.get("requestType"))
|
||||||
.nodeId(request.get("deviceId"))
|
.nodeId(request.get("deviceId"))
|
||||||
@ -265,4 +267,26 @@ public class ClientInstallationRequestListener implements InitializingBean {
|
|||||||
log.trace("InstallationEventListener.convertToNodeInfoMap(): END: nodeMap: {}", nodeMap);
|
log.trace("InstallationEventListener.convertToNodeInfoMap(): END: nodeMap: {}", nodeMap);
|
||||||
return nodeMap;
|
return nodeMap;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void processRemoveRequest(Map<String,String> request) throws Exception {
|
||||||
|
String requestId = request.getOrDefault("requestId", "").trim();
|
||||||
|
String nodeAddress = request.getOrDefault("deviceIpAddress", "").trim();
|
||||||
|
log.info("InstallationEventListener: New node REMOVE request with Id: {}, address={}", requestId, nodeAddress);
|
||||||
|
if (StringUtils.isBlank(requestId)) {
|
||||||
|
clientInstaller.sendErrorClientInstallationReport("MISSING-REQUEST-ID", "INVALID REQUEST. MISSING REQUEST ID");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (StringUtils.isBlank(nodeAddress)) {
|
||||||
|
clientInstaller.sendErrorClientInstallationReport(requestId, "INVALID REQUEST. MISSING IP ADDRESS");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
log.debug("InstallationEventListener: Off-boarding node due to REMOVE request with Id: {}", requestId);
|
||||||
|
nodeRegistration.unregisterNode(nodeAddress, new TranslationContext(requestId));
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("InstallationEventListener: EXCEPTION while executing REMOVE request with Id: {}\n", requestId, e);
|
||||||
|
clientInstaller.sendErrorClientInstallationReport(requestId, "ERROR: "+e.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -16,6 +16,7 @@ import lombok.Builder;
|
|||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.concurrent.Callable;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Client installation task
|
* Client installation task
|
||||||
@ -23,17 +24,22 @@ import java.util.List;
|
|||||||
@Data
|
@Data
|
||||||
@Builder
|
@Builder
|
||||||
public class ClientInstallationTask {
|
public class ClientInstallationTask {
|
||||||
|
public enum TASK_TYPE { INSTALL, UNINSTALL, DIAGNOSTIC, OTHER }
|
||||||
|
|
||||||
private final String id;
|
private final String id;
|
||||||
|
private final TASK_TYPE taskType;
|
||||||
private final String nodeId;
|
private final String nodeId;
|
||||||
private final String requestId;
|
private final String requestId;
|
||||||
private final String name;
|
private final String name;
|
||||||
private final String os;
|
private final String os;
|
||||||
private final String address;
|
private final String address;
|
||||||
private final String type;
|
private final String type; // Node type (VM, baremetal etc)
|
||||||
private final String provider;
|
private final String provider;
|
||||||
private final SshConfig ssh;
|
private final SshConfig ssh;
|
||||||
private final NodeRegistryEntry nodeRegistryEntry;
|
private final NodeRegistryEntry nodeRegistryEntry;
|
||||||
private final List<InstructionsSet> instructionSets;
|
private final List<InstructionsSet> instructionSets;
|
||||||
private final TranslationContext translationContext;
|
private final TranslationContext translationContext;
|
||||||
private boolean nodeMustBeInRegistry = true;
|
private boolean nodeMustBeInRegistry = true;
|
||||||
|
|
||||||
|
private Callable<String> callback;
|
||||||
}
|
}
|
||||||
|
@ -10,6 +10,7 @@
|
|||||||
package gr.iccs.imu.ems.baguette.client.install;
|
package gr.iccs.imu.ems.baguette.client.install;
|
||||||
|
|
||||||
import gr.iccs.imu.ems.baguette.server.BaguetteServer;
|
import gr.iccs.imu.ems.baguette.server.BaguetteServer;
|
||||||
|
import gr.iccs.imu.ems.baguette.server.ClientShellCommand;
|
||||||
import gr.iccs.imu.ems.baguette.server.NodeRegistryEntry;
|
import gr.iccs.imu.ems.baguette.server.NodeRegistryEntry;
|
||||||
import gr.iccs.imu.ems.brokercep.BrokerCepService;
|
import gr.iccs.imu.ems.brokercep.BrokerCepService;
|
||||||
import gr.iccs.imu.ems.common.plugin.PluginManager;
|
import gr.iccs.imu.ems.common.plugin.PluginManager;
|
||||||
@ -69,26 +70,49 @@ public class ClientInstaller implements InitializingBean {
|
|||||||
executorService.submit(() -> {
|
executorService.submit(() -> {
|
||||||
long taskCnt = taskCounter.getAndIncrement();
|
long taskCnt = taskCounter.getAndIncrement();
|
||||||
String resultStr = "";
|
String resultStr = "";
|
||||||
|
String callbackStr = "";
|
||||||
String errorStr = "";
|
String errorStr = "";
|
||||||
|
|
||||||
// Execute task
|
// Execute task
|
||||||
|
boolean result = false;
|
||||||
try {
|
try {
|
||||||
log.info("ClientInstaller: Executing Client installation Task #{}: task-id={}, node-id={}, name={}, type={}, address={}",
|
log.info("ClientInstaller: Executing Client installation Task #{}: task-id={}, node-id={}, name={}, type={}, address={}",
|
||||||
taskCnt, task.getId(), task.getNodeId(), task.getName(), task.getType(), task.getAddress());
|
taskCnt, task.getId(), task.getNodeId(), task.getName(), task.getType(), task.getAddress());
|
||||||
long startTm = System.currentTimeMillis();
|
long startTm = System.currentTimeMillis();
|
||||||
boolean result = executeTask(task, taskCnt);
|
result = executeTask(task, taskCnt);
|
||||||
long endTm = System.currentTimeMillis();
|
long endTm = System.currentTimeMillis();
|
||||||
resultStr = result ? "SUCCESS" : "FAILED";
|
resultStr = result ? "SUCCESS" : "FAILED";
|
||||||
log.info("ClientInstaller: Client installation Task #{}: result={}, duration={}ms",
|
log.info("ClientInstaller: Client installation Task #{}: result={}, duration={}ms",
|
||||||
taskCnt, resultStr, endTm - startTm);
|
taskCnt, resultStr, endTm - startTm);
|
||||||
} catch (Throwable t) {
|
} catch (Throwable t) {
|
||||||
log.info("ClientInstaller: Exception caught in Client installation Task #{}: Exception: ", taskCnt, t);
|
log.error("ClientInstaller: Exception caught in Client installation Task #{}: Exception: ", taskCnt, t);
|
||||||
errorStr = t.getMessage();
|
errorStr = "EXCEPTION " + t.getMessage();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run callback (if any)
|
||||||
|
if (result && task.getCallback()!=null) {
|
||||||
|
try {
|
||||||
|
log.debug("ClientInstaller: CALLBACK started: Task #{}: task-id={}", taskCnt, task.getId());
|
||||||
|
long startTm = System.currentTimeMillis();
|
||||||
|
callbackStr = task.getCallback().call();
|
||||||
|
long endTm = System.currentTimeMillis();
|
||||||
|
log.info("ClientInstaller: CALLBACK completed: Task #{}: callback-result={}, duration={}ms", taskCnt, callbackStr, endTm - startTm);
|
||||||
|
if (! "OK".equalsIgnoreCase(callbackStr)) resultStr = "FAILED";
|
||||||
|
} catch (Throwable t) {
|
||||||
|
log.error("ClientInstaller: CALLBACK: Exception caught while running callback of Client installation Task #{}: Exception: ", taskCnt, t);
|
||||||
|
callbackStr = "CALLBACK-EXCEPTION " + t.getMessage();
|
||||||
|
resultStr = "FAILED";
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if (result)
|
||||||
|
log.debug("ClientInstaller: No CALLBACK found for Task #{}", taskCnt);
|
||||||
|
else
|
||||||
|
log.debug("ClientInstaller: Skipped CALLBACK because execution failed for Task #{}", taskCnt);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Send execution report to local broker
|
// Send execution report to local broker
|
||||||
try {
|
try {
|
||||||
resultStr = StringUtils.defaultIfBlank(resultStr, "ERROR: " + errorStr);
|
resultStr = StringUtils.defaultIfBlank(resultStr, "ERROR: " + errorStr + " " + callbackStr);
|
||||||
sendSuccessClientInstallationReport(taskCnt, task, resultStr);
|
sendSuccessClientInstallationReport(taskCnt, task, resultStr);
|
||||||
} catch (Throwable t) {
|
} catch (Throwable t) {
|
||||||
log.info("ClientInstaller: EXCEPTION while sending Client installation report for Task #{}: Exception: ", taskCnt, t);
|
log.info("ClientInstaller: EXCEPTION while sending Client installation report for Task #{}: Exception: ", taskCnt, t);
|
||||||
@ -103,7 +127,8 @@ public class ClientInstaller implements InitializingBean {
|
|||||||
|
|
||||||
return executeVmOrBaremetalTask(task, taskCounter);
|
return executeVmOrBaremetalTask(task, taskCounter);
|
||||||
} else
|
} else
|
||||||
if ("DIAGNOSTICS".equalsIgnoreCase(task.getType())) {
|
//if ("DIAGNOSTICS".equalsIgnoreCase(task.getType())) {
|
||||||
|
if (task.getTaskType()==ClientInstallationTask.TASK_TYPE.DIAGNOSTIC) {
|
||||||
return executeDiagnosticsTask(task, taskCounter);
|
return executeDiagnosticsTask(task, taskCounter);
|
||||||
} else {
|
} else {
|
||||||
log.error("ClientInstaller: UNSUPPORTED TASK TYPE: {}", task.getType());
|
log.error("ClientInstaller: UNSUPPORTED TASK TYPE: {}", task.getType());
|
||||||
@ -121,30 +146,55 @@ public class ClientInstaller implements InitializingBean {
|
|||||||
if (! task.isNodeMustBeInRegistry())
|
if (! task.isNodeMustBeInRegistry())
|
||||||
entry = task.getNodeRegistryEntry();
|
entry = task.getNodeRegistryEntry();
|
||||||
|
|
||||||
entry.nodeInstalling(task);
|
// Check if node is being removed or have been archived
|
||||||
|
if (task.getNodeRegistryEntry().isArchived()) {
|
||||||
// Call InstallationContextPlugin's before installation
|
log.warn("ClientInstaller: Node is being removed or has been archived: {}", properties.getInstallationContextProcessorPlugins());
|
||||||
log.debug("ClientInstaller: PRE-INSTALLATION: Calling installation context processors: {}", properties.getInstallationContextProcessorPlugins());
|
throw new IllegalStateException("Node is being removed or has been archived: Node IP address: "+ task.getAddress());
|
||||||
pluginManager.getActivePlugins(InstallationContextProcessorPlugin.class)
|
|
||||||
.forEach(plugin->((InstallationContextProcessorPlugin)plugin).processBeforeInstallation(task, taskCounter));
|
|
||||||
|
|
||||||
log.debug("ClientInstaller: INSTALLATION: Executing installation task: task-counter={}, task={}", taskCounter, task);
|
|
||||||
boolean success = executeVmTask(task, taskCounter);
|
|
||||||
log.debug("ClientInstaller: NODE_REGISTRY_ENTRY after installation execution: \n{}", task.getNodeRegistryEntry());
|
|
||||||
|
|
||||||
if (entry.getState()==NodeRegistryEntry.STATE.INSTALLING) {
|
|
||||||
log.warn("ClientInstaller: NODE_REGISTRY_ENTRY status is still INSTALLING after executing client installation. Changing to INSTALL_ERROR");
|
|
||||||
entry.nodeInstallationError(null);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Call InstallationContextPlugin's after installation
|
boolean success;
|
||||||
log.debug("ClientInstaller: POST-INSTALLATION: Calling installation context processors: {}", properties.getInstallationContextProcessorPlugins());
|
if (! task.getInstructionSets().isEmpty()) {
|
||||||
pluginManager.getActivePlugins(InstallationContextProcessorPlugin.class)
|
// Starting installation
|
||||||
.forEach(plugin->((InstallationContextProcessorPlugin)plugin).processAfterInstallation(task, taskCounter, success));
|
entry.nodeInstalling(task);
|
||||||
|
|
||||||
|
// Call InstallationContextPlugin's before installation
|
||||||
|
log.debug("ClientInstaller: PRE-INSTALLATION: Calling installation context processors: {}", properties.getInstallationContextProcessorPlugins());
|
||||||
|
pluginManager.getActivePlugins(InstallationContextProcessorPlugin.class)
|
||||||
|
.forEach(plugin -> ((InstallationContextProcessorPlugin) plugin).processBeforeInstallation(task, taskCounter));
|
||||||
|
|
||||||
|
log.debug("ClientInstaller: INSTALLATION: Executing installation task: task-counter={}, task={}", taskCounter, task);
|
||||||
|
success = executeVmTask(task, taskCounter);
|
||||||
|
log.debug("ClientInstaller: NODE_REGISTRY_ENTRY after installation execution: \n{}", task.getNodeRegistryEntry());
|
||||||
|
|
||||||
|
if (entry.getState() == NodeRegistryEntry.STATE.INSTALLING) {
|
||||||
|
log.warn("ClientInstaller: NODE_REGISTRY_ENTRY status is still INSTALLING after executing client installation. Changing to INSTALL_ERROR");
|
||||||
|
entry.nodeInstallationError(null);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Call InstallationContextPlugin's after installation
|
||||||
|
log.debug("ClientInstaller: POST-INSTALLATION: Calling installation context processors: {}", properties.getInstallationContextProcessorPlugins());
|
||||||
|
pluginManager.getActivePlugins(InstallationContextProcessorPlugin.class)
|
||||||
|
.forEach(plugin -> ((InstallationContextProcessorPlugin) plugin).processAfterInstallation(task, taskCounter, success));
|
||||||
|
} else {
|
||||||
|
log.debug("ClientInstaller: SKIP INSTALLATION: Task has no instructions sets. Skipping execution: Node IP address: "+ task.getAddress());
|
||||||
|
success = true;
|
||||||
|
}
|
||||||
|
|
||||||
// Pre-register Node to baguette Server Coordinator
|
// Pre-register Node to baguette Server Coordinator
|
||||||
log.debug("ClientInstaller: POST-INSTALLATION: Node is being pre-registered: {}", entry);
|
if (task.getTaskType()==ClientInstallationTask.TASK_TYPE.INSTALL) {
|
||||||
baguetteServer.getNodeRegistry().getCoordinator().preregister(entry);
|
log.debug("ClientInstaller: POST-INSTALLATION: Node is being pre-registered: {}", entry);
|
||||||
|
baguetteServer.getNodeRegistry().getCoordinator().preregister(entry);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Un-register Node from baguette Server Coordinator
|
||||||
|
if (task.getTaskType()==ClientInstallationTask.TASK_TYPE.UNINSTALL) {
|
||||||
|
ClientShellCommand csc = ClientShellCommand.getActiveByIpAddress(entry.getIpAddress());
|
||||||
|
log.debug("ClientInstaller: POST-INSTALLATION: CSC of node to be unregistered: {}", csc);
|
||||||
|
if (csc!=null) {
|
||||||
|
log.debug("ClientInstaller: POST-INSTALLATION: Node is going to be unregistered: {}", entry);
|
||||||
|
baguetteServer.getNodeRegistry().getCoordinator().unregister(csc);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
log.debug("ClientInstaller: Installation outcome: {}", success ? "Success" : "Error");
|
log.debug("ClientInstaller: Installation outcome: {}", success ? "Success" : "Error");
|
||||||
return success;
|
return success;
|
||||||
|
@ -15,4 +15,5 @@ import java.util.Map;
|
|||||||
|
|
||||||
public interface INodeRegistration {
|
public interface INodeRegistration {
|
||||||
String registerNode(String baseUrl, Map<String,Object> nodeInfo, TranslationContext translationContext) throws Exception;
|
String registerNode(String baseUrl, Map<String,Object> nodeInfo, TranslationContext translationContext) throws Exception;
|
||||||
|
String unregisterNode(String nodeAddress, TranslationContext translationContext) throws Exception;
|
||||||
}
|
}
|
||||||
|
@ -242,6 +242,21 @@ public abstract class AbstractInstallationHelper implements InitializingBean, Ap
|
|||||||
return instructionsSetList;
|
return instructionsSetList;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public List<InstructionsSet> prepareUninstallInstructionsForOs(NodeRegistryEntry entry) throws IOException {
|
||||||
|
if (! entry.getBaguetteServer().isServerRunning()) throw new RuntimeException("Baguette Server is not running");
|
||||||
|
log.trace("AbstractInstallationHelper.prepareUninstallInstructionsForOs(): node-map={}", entry.getPreregistration());
|
||||||
|
|
||||||
|
String osFamily = entry.getPreregistration().get("operatingSystem");
|
||||||
|
List<InstructionsSet> instructionsSetList = null;
|
||||||
|
if (matchesOsFamily(osFamily, LINUX_OS_FAMILY))
|
||||||
|
instructionsSetList = prepareUninstallInstructionsForLinux(entry);
|
||||||
|
else if (matchesOsFamily(osFamily, WINDOWS_OS_FAMILY))
|
||||||
|
instructionsSetList = prepareUninstallInstructionsForWin(entry);
|
||||||
|
else
|
||||||
|
log.warn("AbstractInstallationHelper.prepareUninstallInstructionsForOs(): Unsupported OS family: {}", osFamily);
|
||||||
|
return instructionsSetList;
|
||||||
|
}
|
||||||
|
|
||||||
public boolean matchesOsFamily(@NonNull String lookup, @NonNull String osFamily) {
|
public boolean matchesOsFamily(@NonNull String lookup, @NonNull String osFamily) {
|
||||||
lookup = lookup.trim().toUpperCase();
|
lookup = lookup.trim().toUpperCase();
|
||||||
List<String> familyList = properties.getOsFamilies().get(osFamily);
|
List<String> familyList = properties.getOsFamilies().get(osFamily);
|
||||||
|
@ -25,8 +25,13 @@ public interface InstallationHelper {
|
|||||||
List<InstructionsSet> prepareInstallationInstructionsForWin(NodeRegistryEntry entry);
|
List<InstructionsSet> prepareInstallationInstructionsForWin(NodeRegistryEntry entry);
|
||||||
List<InstructionsSet> prepareInstallationInstructionsForLinux(NodeRegistryEntry entry) throws IOException;
|
List<InstructionsSet> prepareInstallationInstructionsForLinux(NodeRegistryEntry entry) throws IOException;
|
||||||
|
|
||||||
|
List<InstructionsSet> prepareUninstallInstructionsForOs(NodeRegistryEntry entry) throws IOException;
|
||||||
|
List<InstructionsSet> prepareUninstallInstructionsForWin(NodeRegistryEntry entry);
|
||||||
|
List<InstructionsSet> prepareUninstallInstructionsForLinux(NodeRegistryEntry entry) throws IOException;
|
||||||
|
|
||||||
default ClientInstallationTask createClientInstallationTask(NodeRegistryEntry entry) throws Exception {
|
default ClientInstallationTask createClientInstallationTask(NodeRegistryEntry entry) throws Exception {
|
||||||
return createClientInstallationTask(entry, null);
|
return createClientInstallationTask(entry, null);
|
||||||
}
|
}
|
||||||
ClientInstallationTask createClientInstallationTask(NodeRegistryEntry entry, TranslationContext translationContext) throws Exception;
|
ClientInstallationTask createClientInstallationTask(NodeRegistryEntry entry, TranslationContext translationContext) throws Exception;
|
||||||
|
ClientInstallationTask createClientUninstallTask(NodeRegistryEntry entry, TranslationContext translationContext) throws Exception;
|
||||||
}
|
}
|
||||||
|
@ -22,6 +22,7 @@ import gr.iccs.imu.ems.baguette.server.NodeRegistryEntry;
|
|||||||
import gr.iccs.imu.ems.translate.TranslationContext;
|
import gr.iccs.imu.ems.translate.TranslationContext;
|
||||||
import gr.iccs.imu.ems.util.CredentialsMap;
|
import gr.iccs.imu.ems.util.CredentialsMap;
|
||||||
import gr.iccs.imu.ems.util.NetUtil;
|
import gr.iccs.imu.ems.util.NetUtil;
|
||||||
|
import lombok.NonNull;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.apache.commons.text.StringSubstitutor;
|
import org.apache.commons.text.StringSubstitutor;
|
||||||
@ -60,7 +61,31 @@ public class VmInstallationHelper extends AbstractInstallationHelper {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public ClientInstallationTask createClientInstallationTask(NodeRegistryEntry entry, TranslationContext translationContext) throws IOException {
|
public ClientInstallationTask createClientInstallationTask(NodeRegistryEntry entry, TranslationContext translationContext) throws IOException {
|
||||||
Map<String, String> nodeMap = entry.getPreregistration();
|
// Get EMS client installation instructions for VM node
|
||||||
|
List<InstructionsSet> instructionsSetList =
|
||||||
|
prepareInstallationInstructionsForOs(entry);
|
||||||
|
|
||||||
|
return createClientTask(ClientInstallationTask.TASK_TYPE.INSTALL, entry, translationContext, instructionsSetList);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ClientInstallationTask createClientUninstallTask(NodeRegistryEntry entry, TranslationContext translationContext) throws Exception {
|
||||||
|
// Clear any cached 'instruction-files' override (from a previous run)
|
||||||
|
entry.getPreregistration().remove("instruction-files");
|
||||||
|
|
||||||
|
// Get EMS client uninstall instructions for VM node
|
||||||
|
List<InstructionsSet> instructionsSetList =
|
||||||
|
prepareUninstallInstructionsForOs(entry);
|
||||||
|
|
||||||
|
return createClientTask(ClientInstallationTask.TASK_TYPE.UNINSTALL, entry, translationContext, instructionsSetList);
|
||||||
|
}
|
||||||
|
|
||||||
|
private ClientInstallationTask createClientTask(@NonNull ClientInstallationTask.TASK_TYPE taskType,
|
||||||
|
NodeRegistryEntry entry,
|
||||||
|
TranslationContext translationContext,
|
||||||
|
List<InstructionsSet> instructionsSetList)
|
||||||
|
{
|
||||||
|
Map<String, String> nodeMap = initializeNodeMap(entry);
|
||||||
|
|
||||||
String baseUrl = nodeMap.get("BASE_URL");
|
String baseUrl = nodeMap.get("BASE_URL");
|
||||||
String clientId = nodeMap.get("CLIENT_ID");
|
String clientId = nodeMap.get("CLIENT_ID");
|
||||||
@ -95,13 +120,10 @@ public class VmInstallationHelper extends AbstractInstallationHelper {
|
|||||||
if (StringUtils.isEmpty(password) && StringUtils.isBlank(privateKey))
|
if (StringUtils.isEmpty(password) && StringUtils.isBlank(privateKey))
|
||||||
throw new IllegalArgumentException("Missing SSH password or private key for Node");
|
throw new IllegalArgumentException("Missing SSH password or private key for Node");
|
||||||
|
|
||||||
// Get EMS client installation instructions for VM node
|
|
||||||
List<InstructionsSet> instructionsSetList =
|
|
||||||
prepareInstallationInstructionsForOs(entry);
|
|
||||||
|
|
||||||
// Create Installation Task for VM node
|
// Create Installation Task for VM node
|
||||||
ClientInstallationTask installationTask = ClientInstallationTask.builder()
|
ClientInstallationTask task = ClientInstallationTask.builder()
|
||||||
.id(clientId)
|
.id(clientId)
|
||||||
|
.taskType(taskType)
|
||||||
.nodeId(nodeId)
|
.nodeId(nodeId)
|
||||||
.requestId(requestId)
|
.requestId(requestId)
|
||||||
.name(nodeName)
|
.name(nodeName)
|
||||||
@ -121,29 +143,21 @@ public class VmInstallationHelper extends AbstractInstallationHelper {
|
|||||||
.nodeRegistryEntry(entry)
|
.nodeRegistryEntry(entry)
|
||||||
.translationContext(translationContext)
|
.translationContext(translationContext)
|
||||||
.build();
|
.build();
|
||||||
log.debug("VmInstallationHelper.createClientInstallationTask(): Created client installation task: {}", installationTask);
|
log.debug("VmInstallationHelper.createClientTask(): Created client task: {}", task);
|
||||||
|
return task;
|
||||||
return installationTask;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
private Map<String, String> initializeNodeMap(NodeRegistryEntry entry) {
|
||||||
public List<InstructionsSet> prepareInstallationInstructionsForWin(NodeRegistryEntry entry) {
|
|
||||||
log.warn("VmInstallationHelper.prepareInstallationInstructionsForWin(): NOT YET IMPLEMENTED");
|
|
||||||
throw new IllegalArgumentException("VmInstallationHelper.prepareInstallationInstructionsForWin(): NOT YET IMPLEMENTED");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public List<InstructionsSet> prepareInstallationInstructionsForLinux(NodeRegistryEntry entry) throws IOException {
|
|
||||||
Map<String, String> nodeMap = entry.getPreregistration();
|
Map<String, String> nodeMap = entry.getPreregistration();
|
||||||
BaguetteServer baguette = entry.getBaguetteServer();
|
BaguetteServer baguette = entry.getBaguetteServer();
|
||||||
|
|
||||||
String baseUrl = StringUtils.removeEnd(nodeMap.get("BASE_URL"), "/");
|
String baseUrl = StringUtils.removeEnd(nodeMap.get("BASE_URL"), "/");
|
||||||
String clientId = nodeMap.get("CLIENT_ID");
|
String clientId = nodeMap.get("CLIENT_ID");
|
||||||
String ipSetting = nodeMap.get("IP_SETTING");
|
String ipSetting = nodeMap.get("IP_SETTING");
|
||||||
log.debug("VmInstallationHelper.prepareInstallationInstructionsForLinux(): Invoked: base-url={}", baseUrl);
|
log.debug("VmInstallationHelper.initializeNodeMap(): Invoked: base-url={}", baseUrl);
|
||||||
|
|
||||||
// Get parameters
|
// Get parameters
|
||||||
log.trace("VmInstallationHelper.prepareInstallationInstructionsForLinux(): properties: {}", properties);
|
log.trace("VmInstallationHelper.initializeNodeMap(): properties: {}", properties);
|
||||||
String rootCmd = properties.getRootCmd();
|
String rootCmd = properties.getRootCmd();
|
||||||
String baseDir = properties.getBaseDir();
|
String baseDir = properties.getBaseDir();
|
||||||
String checkInstallationFile = properties.getCheckInstalledFile();
|
String checkInstallationFile = properties.getCheckInstalledFile();
|
||||||
@ -170,7 +184,7 @@ public class VmInstallationHelper extends AbstractInstallationHelper {
|
|||||||
: "NODE_" + e.getKey().toUpperCase(),
|
: "NODE_" + e.getKey().toUpperCase(),
|
||||||
Map.Entry::getValue,
|
Map.Entry::getValue,
|
||||||
(v1, v2) -> {
|
(v1, v2) -> {
|
||||||
log.warn("VmInstallationHelper.prepareInstallationInstructionsForLinux(): DUPLICATE KEY FOUND: key={}, old-value={}, new-value={}",
|
log.warn("VmInstallationHelper.initializeNodeMap(): DUPLICATE KEY FOUND: key={}, old-value={}, new-value={}",
|
||||||
k, v1, v2);
|
k, v1, v2);
|
||||||
return v2;
|
return v2;
|
||||||
}
|
}
|
||||||
@ -182,12 +196,12 @@ public class VmInstallationHelper extends AbstractInstallationHelper {
|
|||||||
? "NODE_SSH_" + k.substring(4).toUpperCase()
|
? "NODE_SSH_" + k.substring(4).toUpperCase()
|
||||||
: "NODE_" + k.toUpperCase();
|
: "NODE_" + k.toUpperCase();
|
||||||
if (additionalKeysMap.containsKey(k)) {
|
if (additionalKeysMap.containsKey(k)) {
|
||||||
log.warn("VmInstallationHelper.prepareInstallationInstructionsForLinux(): DUPLICATE KEY FOUND: key={}, old-value={}, new-value={}",
|
log.warn("VmInstallationHelper.initializeNodeMap(): DUPLICATE KEY FOUND: key={}, old-value={}, new-value={}",
|
||||||
k, additionalKeysMap.get(k), v);
|
k, additionalKeysMap.get(k), v);
|
||||||
}
|
}
|
||||||
additionalKeysMap.put(k, v);
|
additionalKeysMap.put(k, v);
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
log.error("VmInstallationHelper.prepareInstallationInstructionsForLinux(): EXCEPTION in additional keys copy loop: key={}, value={}, additionalKeysMap={}, Exception:\n",
|
log.error("VmInstallationHelper.initializeNodeMap(): EXCEPTION in additional keys copy loop: key={}, value={}, additionalKeysMap={}, Exception:\n",
|
||||||
k, v, additionalKeysMap, ex);
|
k, v, additionalKeysMap, ex);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
@ -229,7 +243,7 @@ public class VmInstallationHelper extends AbstractInstallationHelper {
|
|||||||
|
|
||||||
nodeMap.putAll(clientInstallationProperties.getParameters());
|
nodeMap.putAll(clientInstallationProperties.getParameters());
|
||||||
nodeMap.put("EMS_PUBLIC_DIR", System.getProperty("PUBLIC_DIR", System.getenv("PUBLIC_DIR")));
|
nodeMap.put("EMS_PUBLIC_DIR", System.getProperty("PUBLIC_DIR", System.getenv("PUBLIC_DIR")));
|
||||||
log.trace("VmInstallationHelper.prepareInstallationInstructionsForLinux: value-map: {}", nodeMap);
|
log.trace("VmInstallationHelper.initializeNodeMap: value-map: {}", nodeMap);
|
||||||
|
|
||||||
/* // Clear EMS server certificate (PEM) file, if not secure
|
/* // Clear EMS server certificate (PEM) file, if not secure
|
||||||
if (!isServerSecure) {
|
if (!isServerSecure) {
|
||||||
@ -251,26 +265,65 @@ public class VmInstallationHelper extends AbstractInstallationHelper {
|
|||||||
}
|
}
|
||||||
}*/
|
}*/
|
||||||
|
|
||||||
|
return nodeMap;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<InstructionsSet> prepareInstallationInstructionsForWin(NodeRegistryEntry entry) {
|
||||||
|
log.warn("VmInstallationHelper.prepareInstallationInstructionsForWin(): NOT YET IMPLEMENTED");
|
||||||
|
throw new IllegalArgumentException("VmInstallationHelper.prepareInstallationInstructionsForWin(): NOT YET IMPLEMENTED");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<InstructionsSet> prepareUninstallInstructionsForWin(NodeRegistryEntry entry) {
|
||||||
|
log.warn("VmInstallationHelper.prepareUninstallInstructionsForWin(): NOT YET IMPLEMENTED");
|
||||||
|
throw new IllegalArgumentException("VmInstallationHelper.prepareUninstallInstructionsForWin(): NOT YET IMPLEMENTED");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<InstructionsSet> prepareInstallationInstructionsForLinux(NodeRegistryEntry entry) throws IOException {
|
||||||
|
return prepareInstructionsForLinux(entry, "LINUX");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<InstructionsSet> prepareUninstallInstructionsForLinux(NodeRegistryEntry entry) throws IOException {
|
||||||
|
return prepareInstructionsForLinux(entry, "REMOVE_LINUX");
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<InstructionsSet> prepareInstructionsForLinux(@NonNull NodeRegistryEntry entry, String instructionsScenarioName) throws IOException {
|
||||||
|
Map<String, String> nodeMap = entry.getPreregistration();
|
||||||
|
|
||||||
List<InstructionsSet> instructionsSetList = new ArrayList<>();
|
List<InstructionsSet> instructionsSetList = new ArrayList<>();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Read installation instructions from JSON file
|
// Read installation instructions from JSON file
|
||||||
List<String> instructionSetFileList = null;
|
List<String> instructionSetFileList;
|
||||||
if (nodeMap.containsKey("instruction-files")) {
|
if (nodeMap.containsKey("instruction-files")) {
|
||||||
|
log.trace("VmInstallationHelper.prepareInstructionsForLinux: FOUND instruction-files override: value={}", nodeMap.getOrDefault("instruction-files", null));
|
||||||
instructionSetFileList = Arrays.stream(nodeMap.getOrDefault("instruction-files", "").split(","))
|
instructionSetFileList = Arrays.stream(nodeMap.getOrDefault("instruction-files", "").split(","))
|
||||||
.filter(StringUtils::isNotBlank)
|
.filter(StringUtils::isNotBlank)
|
||||||
.map(String::trim)
|
.map(String::trim)
|
||||||
.collect(Collectors.toList());
|
.collect(Collectors.toList());
|
||||||
|
log.debug("VmInstallationHelper.prepareInstructionsForLinux: FOUND instruction-files override: list={}", instructionSetFileList);
|
||||||
if (instructionSetFileList.isEmpty())
|
if (instructionSetFileList.isEmpty())
|
||||||
log.warn("VmInstallationHelper.prepareInstallationInstructionsForLinux: Context map contains 'instruction-files' entry with no contents");
|
log.warn("VmInstallationHelper.prepareInstructionsForLinux: Context map contains 'instruction-files' entry with no contents");
|
||||||
} else {
|
} else {
|
||||||
instructionSetFileList = properties.getInstructions().get("LINUX");
|
log.trace("VmInstallationHelper.prepareInstructionsForLinux: NOT FOUND instruction-files override. Using configured instructions sets: instructionsScenarioName={}", instructionsScenarioName);
|
||||||
|
instructionSetFileList = properties.getInstructions().get(instructionsScenarioName);
|
||||||
|
log.trace("VmInstallationHelper.prepareInstructionsForLinux: NOT FOUND instruction-files override. Using configured instructions sets: list={}", instructionSetFileList);
|
||||||
|
if (instructionSetFileList==null || instructionSetFileList.isEmpty()) {
|
||||||
|
log.warn("VmInstallationHelper.prepareInstructionsForLinux: No instructions set files provided in configuration with name: {}", instructionsScenarioName);
|
||||||
|
instructionSetFileList = Collections.emptyList();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
log.debug("VmInstallationHelper.prepareInstructionsForLinux: Instructions sets list: {}", instructionSetFileList);
|
||||||
|
|
||||||
|
// Load instructions set from file
|
||||||
for (String instructionSetFile : instructionSetFileList) {
|
for (String instructionSetFile : instructionSetFileList) {
|
||||||
// Load instructions set from file
|
// Load instructions set from file
|
||||||
log.debug("VmInstallationHelper.prepareInstallationInstructionsForLinux: Installation instructions file for LINUX: {}", instructionSetFile);
|
log.debug("VmInstallationHelper.prepareInstructionsForLinux: Installation instructions file for LINUX: {}", instructionSetFile);
|
||||||
InstructionsSet instructionsSet = InstructionsService.getInstance().loadInstructionsFile(instructionSetFile);
|
InstructionsSet instructionsSet = InstructionsService.getInstance().loadInstructionsFile(instructionSetFile);
|
||||||
log.debug("VmInstallationHelper.prepareInstallationInstructionsForLinux: Instructions set loaded from file: {}\n{}", instructionSetFile, instructionsSet);
|
log.debug("VmInstallationHelper.prepareInstructionsForLinux: Instructions set loaded from file: {}\n{}", instructionSetFile, instructionsSet);
|
||||||
|
|
||||||
// Pretty print instructionsSet JSON
|
// Pretty print instructionsSet JSON
|
||||||
if (log.isTraceEnabled()) {
|
if (log.isTraceEnabled()) {
|
||||||
@ -279,7 +332,7 @@ public class VmInstallationHelper extends AbstractInstallationHelper {
|
|||||||
try (PrintWriter writer = new PrintWriter(stringWriter)) {
|
try (PrintWriter writer = new PrintWriter(stringWriter)) {
|
||||||
gson.toJson(instructionsSet, writer);
|
gson.toJson(instructionsSet, writer);
|
||||||
}
|
}
|
||||||
log.trace("VmInstallationHelper.prepareInstallationInstructionsForLinux: Installation instructions for LINUX: json:\n{}", stringWriter);
|
log.trace("VmInstallationHelper.prepareInstructionsForLinux: Installation instructions for LINUX: json:\n{}", stringWriter);
|
||||||
}
|
}
|
||||||
|
|
||||||
instructionsSetList.add(instructionsSet);
|
instructionsSetList.add(instructionsSet);
|
||||||
@ -287,7 +340,7 @@ public class VmInstallationHelper extends AbstractInstallationHelper {
|
|||||||
|
|
||||||
return instructionsSetList;
|
return instructionsSetList;
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
log.error("VmInstallationHelper.prepareInstallationInstructionsForLinux: Exception while reading Installation instructions for LINUX: ", ex);
|
log.error("VmInstallationHelper.prepareInstructionsForLinux: Exception while reading Installation instructions for LINUX: ", ex);
|
||||||
throw ex;
|
throw ex;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -48,8 +48,9 @@ public class ClientRecoveryPlugin implements InitializingBean, EventBus.EventCon
|
|||||||
private long clientRecoveryDelay;
|
private long clientRecoveryDelay;
|
||||||
private String recoveryInstructionsFile;
|
private String recoveryInstructionsFile;
|
||||||
|
|
||||||
private final static String CLIENT_EXIT_TOPIC = "BAGUETTE_SERVER_CLIENT_EXITED";
|
public final static String CLIENT_EXIT_TOPIC = "BAGUETTE_SERVER_CLIENT_EXITED";
|
||||||
private final static String CLIENT_REGISTERED_TOPIC = "BAGUETTE_SERVER_CLIENT_REGISTERED";
|
public final static String CLIENT_REGISTERED_TOPIC = "BAGUETTE_SERVER_CLIENT_REGISTERED";
|
||||||
|
public final static String CLIENT_REMOVED_TOPIC = "BAGUETTE_SERVER_CLIENT_REMOVED";
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void afterPropertiesSet() throws Exception {
|
public void afterPropertiesSet() throws Exception {
|
||||||
@ -61,6 +62,8 @@ public class ClientRecoveryPlugin implements InitializingBean, EventBus.EventCon
|
|||||||
log.debug("ClientRecoveryPlugin: Subscribed for BAGUETTE_SERVER_CLIENT_EXITED events");
|
log.debug("ClientRecoveryPlugin: Subscribed for BAGUETTE_SERVER_CLIENT_EXITED events");
|
||||||
eventBus.subscribe(CLIENT_REGISTERED_TOPIC, this);
|
eventBus.subscribe(CLIENT_REGISTERED_TOPIC, this);
|
||||||
log.debug("ClientRecoveryPlugin: Subscribed for BAGUETTE_SERVER_CLIENT_REGISTERED events");
|
log.debug("ClientRecoveryPlugin: Subscribed for BAGUETTE_SERVER_CLIENT_REGISTERED events");
|
||||||
|
eventBus.subscribe(CLIENT_REMOVED_TOPIC, this);
|
||||||
|
log.debug("ClientRecoveryPlugin: Subscribed for CLIENT_REMOVED_TOPIC events");
|
||||||
|
|
||||||
log.trace("ClientRecoveryPlugin: clientInstallationProperties: {}", clientInstallationProperties);
|
log.trace("ClientRecoveryPlugin: clientInstallationProperties: {}", clientInstallationProperties);
|
||||||
log.trace("ClientRecoveryPlugin: baguetteServer: {}", baguetteServer);
|
log.trace("ClientRecoveryPlugin: baguetteServer: {}", baguetteServer);
|
||||||
@ -80,18 +83,28 @@ public class ClientRecoveryPlugin implements InitializingBean, EventBus.EventCon
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Only process messages of ClientShellCommand type are accepted (sent by CSC instances)
|
// Only process messages of ClientShellCommand type are accepted (sent by CSC instances)
|
||||||
if (! (message instanceof ClientShellCommand)) {
|
if (! (message instanceof NodeRegistryEntry) && ! (message instanceof ClientShellCommand)) {
|
||||||
log.warn("ClientRecoveryPlugin: onMessage(): Message is not a {} object. Will ignore it.", ClientShellCommand.class.getSimpleName());
|
log.warn("ClientRecoveryPlugin: onMessage(): Message is neither a {} or a {} object. Will ignore it.",
|
||||||
|
NodeRegistryEntry.class.getSimpleName(), ClientShellCommand.class.getSimpleName());
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get NodeRegistryEntry from ClientShellCommand passed with event
|
NodeRegistryEntry nodeInfo;
|
||||||
ClientShellCommand csc = (ClientShellCommand)message;
|
String clientId;
|
||||||
String clientId = csc.getId();
|
String address;
|
||||||
String address = csc.getClientIpAddress();
|
if (message instanceof NodeRegistryEntry entry) {
|
||||||
log.debug("ClientRecoveryPlugin: onMessage(): client-id={}, client-address={}", clientId, address);
|
nodeInfo = entry;
|
||||||
|
clientId = entry.getClientId();
|
||||||
|
address = entry.getIpAddress();
|
||||||
|
} else {
|
||||||
|
// Get NodeRegistryEntry from ClientShellCommand passed with event
|
||||||
|
ClientShellCommand csc = (ClientShellCommand) message;
|
||||||
|
clientId = csc.getId();
|
||||||
|
address = csc.getClientIpAddress();
|
||||||
|
log.debug("ClientRecoveryPlugin: onMessage(): client-id={}, client-address={}", clientId, address);
|
||||||
|
|
||||||
NodeRegistryEntry nodeInfo = csc.getNodeRegistryEntry(); //or = nodeRegistry.getNodeByAddress(address);
|
nodeInfo = csc.getNodeRegistryEntry(); //or = nodeRegistry.getNodeByAddress(address);
|
||||||
|
}
|
||||||
log.debug("ClientRecoveryPlugin: onMessage(): client-node-info={}", nodeInfo);
|
log.debug("ClientRecoveryPlugin: onMessage(): client-node-info={}", nodeInfo);
|
||||||
log.trace("ClientRecoveryPlugin: onMessage(): node-registry.node-addresses={}", nodeRegistry.getNodeAddresses());
|
log.trace("ClientRecoveryPlugin: onMessage(): node-registry.node-addresses={}", nodeRegistry.getNodeAddresses());
|
||||||
log.trace("ClientRecoveryPlugin: onMessage(): node-registry.nodes={}", nodeRegistry.getNodes());
|
log.trace("ClientRecoveryPlugin: onMessage(): node-registry.nodes={}", nodeRegistry.getNodes());
|
||||||
@ -108,14 +121,25 @@ public class ClientRecoveryPlugin implements InitializingBean, EventBus.EventCon
|
|||||||
processExitEvent(nodeInfo);
|
processExitEvent(nodeInfo);
|
||||||
}
|
}
|
||||||
if (CLIENT_REGISTERED_TOPIC.equals(topic)) {
|
if (CLIENT_REGISTERED_TOPIC.equals(topic)) {
|
||||||
log.debug("ClientRecoveryPlugin: onMessage(): CLIENT REGISTERED_TOPIC: message={}", message);
|
log.debug("ClientRecoveryPlugin: onMessage(): CLIENT REGISTERED: message={}", message);
|
||||||
processRegisteredEvent(nodeInfo);
|
processRegisteredEvent(nodeInfo);
|
||||||
}
|
}
|
||||||
|
if (CLIENT_REMOVED_TOPIC.equals(topic)) {
|
||||||
|
log.debug("ClientRecoveryPlugin: onMessage(): CLIENT REMOVED: message={}", message);
|
||||||
|
processRemovedEvent(nodeInfo);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void processExitEvent(NodeRegistryEntry nodeInfo) {
|
private void processExitEvent(NodeRegistryEntry nodeInfo) {
|
||||||
log.debug("ClientRecoveryPlugin: processExitEvent(): BEGIN: client-id={}, client-address={}", nodeInfo.getClientId(), nodeInfo.getIpAddress());
|
log.debug("ClientRecoveryPlugin: processExitEvent(): BEGIN: client-id={}, client-address={}", nodeInfo.getClientId(), nodeInfo.getIpAddress());
|
||||||
|
|
||||||
|
// Check if node can be recovered (based on its Life-Cycle state)
|
||||||
|
if (! nodeInfo.canRecover()) {
|
||||||
|
log.warn("ClientRecoveryPlugin: processExitEvent(): Node will not be recovered because its state is {}: client-id={}, client-address={}",
|
||||||
|
nodeInfo.getState(), nodeInfo.getClientId(), nodeInfo.getIpAddress());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
// Set node state to DOWN
|
// Set node state to DOWN
|
||||||
baguetteServer.getSelfHealingManager().setNodeSelfHealingState(nodeInfo, SelfHealingManager.NODE_STATE.DOWN);
|
baguetteServer.getSelfHealingManager().setNodeSelfHealingState(nodeInfo, SelfHealingManager.NODE_STATE.DOWN);
|
||||||
|
|
||||||
@ -156,10 +180,29 @@ public class ClientRecoveryPlugin implements InitializingBean, EventBus.EventCon
|
|||||||
baguetteServer.getSelfHealingManager().setNodeSelfHealingState(nodeInfo, SelfHealingManager.NODE_STATE.UP);
|
baguetteServer.getSelfHealingManager().setNodeSelfHealingState(nodeInfo, SelfHealingManager.NODE_STATE.UP);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void processRemovedEvent(NodeRegistryEntry nodeInfo) {
|
||||||
|
log.debug("ClientRecoveryPlugin: processRemovedEvent(): BEGIN: client-id={}, client-address={}", nodeInfo.getClientId(), nodeInfo.getIpAddress());
|
||||||
|
|
||||||
|
// Cancel any pending recovery task (for the node)
|
||||||
|
ScheduledFuture<?> future = pendingTasks.remove(nodeInfo);
|
||||||
|
log.debug("ClientRecoveryPlugin: processRemovedEvent(): Recovery task: task={}, client-id={}, client-address={}", future, nodeInfo.getClientId(), nodeInfo.getIpAddress());
|
||||||
|
if (future!=null && ! future.isDone() && ! future.isCancelled()) {
|
||||||
|
log.warn("ClientRecoveryPlugin: processRemovedEvent(): Cancelled recovery task: client-id={}, client-address={}", nodeInfo.getClientId(), nodeInfo.getIpAddress());
|
||||||
|
future.cancel(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public void runClientRecovery(NodeRegistryEntry entry) throws Exception {
|
public void runClientRecovery(NodeRegistryEntry entry) throws Exception {
|
||||||
log.debug("ClientRecoveryPlugin: runClientRecovery(): node-info={}", entry);
|
log.debug("ClientRecoveryPlugin: runClientRecovery(): node-info={}", entry);
|
||||||
if (entry==null) return;
|
if (entry==null) return;
|
||||||
|
|
||||||
|
if (! entry.canRecover()) {
|
||||||
|
log.info("ClientRecoveryPlugin: runClientRecovery(): Cannot recover node. Will not attempt recovery again: node-state={}, client-id={}, client-address={}",
|
||||||
|
entry.getState(), entry.getClientId(), entry.getIpAddress());
|
||||||
|
pendingTasks.remove(entry);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
log.trace("ClientRecoveryPlugin: runClientRecovery(): recoveryInstructionsFile={}", recoveryInstructionsFile);
|
log.trace("ClientRecoveryPlugin: runClientRecovery(): recoveryInstructionsFile={}", recoveryInstructionsFile);
|
||||||
entry.getPreregistration().put("instruction-files", recoveryInstructionsFile);
|
entry.getPreregistration().put("instruction-files", recoveryInstructionsFile);
|
||||||
|
|
||||||
|
@ -550,4 +550,13 @@ public class BaguetteServer implements InitializingBean, EventBus.EventConsumer<
|
|||||||
}
|
}
|
||||||
return clientId;
|
return clientId;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public NodeRegistryEntry unregisterClient(NodeRegistryEntry entry) throws UnknownHostException {
|
||||||
|
log.debug("BaguetteServer.unregisterClient(): entry={}", entry);
|
||||||
|
|
||||||
|
// Archive node entry in node registry
|
||||||
|
nodeRegistry.archiveNode(entry);
|
||||||
|
|
||||||
|
return entry;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -265,8 +265,16 @@ public class ClientShellCommand implements Command, Runnable, ServerSessionAware
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Client connection closed
|
// Client connection closed
|
||||||
eventBus.send("BAGUETTE_SERVER_CLIENT_EXITING", this);
|
try {
|
||||||
getNodeRegistryEntry().nodeExiting(null);
|
eventBus.send("BAGUETTE_SERVER_CLIENT_EXITING", this);
|
||||||
|
NodeRegistryEntry entry = getNodeRegistryEntry();
|
||||||
|
if (! entry.isArchived())
|
||||||
|
entry.nodeExiting(null);
|
||||||
|
else
|
||||||
|
log.warn("{}==> Node is archived", id);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("{}==> EXCEPTION: ", id, e);
|
||||||
|
}
|
||||||
|
|
||||||
log.info("{}==> Signaling client to exit", id);
|
log.info("{}==> Signaling client to exit", id);
|
||||||
out.println("EXIT");
|
out.println("EXIT");
|
||||||
|
@ -10,6 +10,7 @@
|
|||||||
package gr.iccs.imu.ems.baguette.server;
|
package gr.iccs.imu.ems.baguette.server;
|
||||||
|
|
||||||
import lombok.Getter;
|
import lombok.Getter;
|
||||||
|
import lombok.NonNull;
|
||||||
import lombok.Setter;
|
import lombok.Setter;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
@ -17,9 +18,7 @@ import org.springframework.stereotype.Service;
|
|||||||
|
|
||||||
import java.net.InetAddress;
|
import java.net.InetAddress;
|
||||||
import java.net.UnknownHostException;
|
import java.net.UnknownHostException;
|
||||||
import java.util.Collection;
|
import java.util.*;
|
||||||
import java.util.LinkedHashMap;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -29,6 +28,7 @@ import java.util.stream.Collectors;
|
|||||||
@Service
|
@Service
|
||||||
public class NodeRegistry {
|
public class NodeRegistry {
|
||||||
private final Map<String,NodeRegistryEntry> registry = new LinkedHashMap<>();
|
private final Map<String,NodeRegistryEntry> registry = new LinkedHashMap<>();
|
||||||
|
private final List<NodeRegistryEntry> archived = new LinkedList<>();
|
||||||
@Getter @Setter
|
@Getter @Setter
|
||||||
private ServerCoordinator coordinator;
|
private ServerCoordinator coordinator;
|
||||||
|
|
||||||
@ -134,4 +134,27 @@ public class NodeRegistry {
|
|||||||
public Collection<String> getNodeReferences() {
|
public Collection<String> getNodeReferences() {
|
||||||
return registry.values().stream().map(NodeRegistryEntry::getReference).collect(Collectors.toList());
|
return registry.values().stream().map(NodeRegistryEntry::getReference).collect(Collectors.toList());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public synchronized void archiveNode(@NonNull NodeRegistryEntry entry) {
|
||||||
|
log.debug("NodeRegistry.archiveNode(): Archiving node: entry={}", entry);
|
||||||
|
|
||||||
|
// Run checks
|
||||||
|
if (StringUtils.isBlank(entry.getIpAddress())) {
|
||||||
|
log.warn("NodeRegistry.archiveNode(): Node entry does not have an IP address. Ignoring archiving operation: entry={}", entry);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (! registry.containsKey(entry.getIpAddress())) {
|
||||||
|
log.warn("NodeRegistry.archiveNode(): Node with IP address {} not found in registry. Ignoring archiving operation: entry={}", entry.getIpAddress(), entry);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (registry.get(entry.getIpAddress())!=entry) {
|
||||||
|
log.warn("NodeRegistry.archiveNode(): Node passed does not match with node registered to IP address {}. Ignoring archiving operation: entry={}", entry.getIpAddress(), entry);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Archive node
|
||||||
|
archived.add(entry);
|
||||||
|
registry.remove(entry.getIpAddress());
|
||||||
|
entry.nodeArchived(null);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -21,8 +21,12 @@ import java.util.*;
|
|||||||
@AllArgsConstructor
|
@AllArgsConstructor
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
public class NodeRegistryEntry {
|
public class NodeRegistryEntry {
|
||||||
public enum STATE { PREREGISTERED, IGNORE_NODE, INSTALLING, NOT_INSTALLED, INSTALLED, INSTALL_ERROR,
|
public enum STATE {
|
||||||
WAITING_REGISTRATION, REGISTERING, REGISTERED, REGISTRATION_ERROR, DISCONNECTED, EXITING, EXITED, NODE_FAILED
|
PREREGISTERED, IGNORE_NODE,
|
||||||
|
INSTALLING, NOT_INSTALLED, INSTALLED, INSTALL_ERROR,
|
||||||
|
WAITING_REGISTRATION, REGISTERING, REGISTERED, REGISTRATION_ERROR,
|
||||||
|
DISCONNECTED, EXITING, EXITED, NODE_FAILED,
|
||||||
|
REMOVING, REMOVED, REMOVE_ERROR, ARCHIVED
|
||||||
};
|
};
|
||||||
@Getter private final String ipAddress;
|
@Getter private final String ipAddress;
|
||||||
@Getter private final String clientId;
|
@Getter private final String clientId;
|
||||||
@ -40,6 +44,8 @@ public class NodeRegistryEntry {
|
|||||||
@JsonIgnore
|
@JsonIgnore
|
||||||
@Getter private transient Map<String, String> registration = new LinkedHashMap<>();
|
@Getter private transient Map<String, String> registration = new LinkedHashMap<>();
|
||||||
@JsonIgnore
|
@JsonIgnore
|
||||||
|
@Getter private transient Map<String, String> removal = new LinkedHashMap<>();
|
||||||
|
@JsonIgnore
|
||||||
@Getter @Setter private transient IClusterZone clusterZone;
|
@Getter @Setter private transient IClusterZone clusterZone;
|
||||||
|
|
||||||
@JsonIgnore
|
@JsonIgnore
|
||||||
@ -70,100 +76,118 @@ public class NodeRegistryEntry {
|
|||||||
|
|
||||||
public void refreshReference() { reference = UUID.randomUUID().toString(); }
|
public void refreshReference() { reference = UUID.randomUUID().toString(); }
|
||||||
|
|
||||||
public NodeRegistryEntry nodePreregistration(Map<String,Object> nodeInfo) {
|
public boolean canRecover() {
|
||||||
preregistration.clear();
|
return state != null && switch (state) {
|
||||||
preregistration.putAll(StrUtil.deepFlattenMap(nodeInfo));
|
case PREREGISTERED, IGNORE_NODE -> false;
|
||||||
setState(STATE.PREREGISTERED);
|
case REMOVING, REMOVED, REMOVE_ERROR, ARCHIVED -> false;
|
||||||
|
default -> true;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isArchived() {
|
||||||
|
return state!=null && state==STATE.ARCHIVED;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean canChangeStateTo(@NonNull STATE newState) {
|
||||||
|
return ! isArchived();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _canUpdateEntry(@NonNull STATE newState) {
|
||||||
|
if (! canChangeStateTo(newState)) {
|
||||||
|
throw new IllegalStateException(String.format("Cannot change NodeRegistryEntry state from %s to %s: client-id=%s, client-address=%s",
|
||||||
|
state, newState, clientId, ipAddress));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private NodeRegistryEntry _updateEntry(@NonNull STATE newState, @NonNull Map<String, String> map, boolean clear, Map<String,Object> nodeInfo) {
|
||||||
|
_canUpdateEntry(newState);
|
||||||
|
if (clear) map.clear();
|
||||||
|
map.putAll(StrUtil.deepFlattenMap(nodeInfo));
|
||||||
|
setState(newState);
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
public NodeRegistryEntry nodeIgnore(Object nodeInfo) {
|
private NodeRegistryEntry _updateEntry(@NonNull STATE newState, @NonNull Map<String, String> map, boolean clear, @NonNull String key, Object val, String defVal) {
|
||||||
installation.clear();
|
_canUpdateEntry(newState);
|
||||||
installation.put("ignore-node", nodeInfo!=null ? nodeInfo.toString() : null);
|
if (clear) map.clear();
|
||||||
|
map.put(key, val!=null ? val.toString() : defVal);
|
||||||
setState(STATE.IGNORE_NODE);
|
setState(STATE.IGNORE_NODE);
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public NodeRegistryEntry nodePreregistration(Map<String,Object> nodeInfo) {
|
||||||
|
return _updateEntry(STATE.PREREGISTERED, preregistration, true, nodeInfo);
|
||||||
|
}
|
||||||
|
|
||||||
|
public NodeRegistryEntry nodeIgnore(Object nodeInfo) {
|
||||||
|
return _updateEntry(STATE.IGNORE_NODE, installation, true, "ignore-node", nodeInfo, null);
|
||||||
|
}
|
||||||
|
|
||||||
public NodeRegistryEntry nodeInstalling(Object nodeInfo) {
|
public NodeRegistryEntry nodeInstalling(Object nodeInfo) {
|
||||||
installation.clear();
|
return _updateEntry(STATE.INSTALLING, installation, true, "installation-task", nodeInfo, "INSTALLING");
|
||||||
installation.put("installation-task", nodeInfo!=null ? nodeInfo.toString() : "INSTALLING");
|
|
||||||
setState(STATE.INSTALLING);
|
|
||||||
return this;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public NodeRegistryEntry nodeNotInstalled(Object nodeInfo) {
|
public NodeRegistryEntry nodeNotInstalled(Object nodeInfo) {
|
||||||
installation.clear();
|
return _updateEntry(STATE.NOT_INSTALLED, installation, true, "installation-task-result", nodeInfo, "NOT_INSTALLED");
|
||||||
installation.put("installation-task-result", nodeInfo!=null ? nodeInfo.toString() : "NOT_INSTALLED");
|
|
||||||
setState(STATE.NOT_INSTALLED);
|
|
||||||
return this;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public NodeRegistryEntry nodeInstallationComplete(Object nodeInfo) {
|
public NodeRegistryEntry nodeInstallationComplete(Object nodeInfo) {
|
||||||
installation.put("installation-task-result", nodeInfo!=null ? nodeInfo.toString() : "SUCCESS");
|
return _updateEntry(STATE.INSTALLED, installation, false, "installation-task-result", nodeInfo, "SUCCESS");
|
||||||
setState(STATE.INSTALLED);
|
|
||||||
return this;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public NodeRegistryEntry nodeInstallationError(Object nodeInfo) {
|
public NodeRegistryEntry nodeInstallationError(Object nodeInfo) {
|
||||||
installation.put("installation-task-result", nodeInfo!=null ? nodeInfo.toString() : "ERROR");
|
return _updateEntry(STATE.INSTALL_ERROR, installation, false, "installation-task-result", nodeInfo, "ERROR");
|
||||||
setState(STATE.INSTALL_ERROR);
|
|
||||||
return this;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public NodeRegistryEntry nodeRegistering(Map<String,Object> nodeInfo) {
|
public NodeRegistryEntry nodeRegistering(Map<String,Object> nodeInfo) {
|
||||||
registration.clear();
|
return _updateEntry(STATE.REGISTERING, registration, true, nodeInfo);
|
||||||
registration.putAll(StrUtil.deepFlattenMap(nodeInfo));
|
|
||||||
setState(STATE.REGISTERING);
|
|
||||||
return this;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public NodeRegistryEntry nodeRegistered(Map<String,Object> nodeInfo) {
|
public NodeRegistryEntry nodeRegistered(Map<String,Object> nodeInfo) {
|
||||||
//registration.clear();
|
return _updateEntry(STATE.REGISTERED, registration, false, nodeInfo);
|
||||||
registration.putAll(StrUtil.deepFlattenMap(nodeInfo));
|
|
||||||
setState(STATE.REGISTERED);
|
|
||||||
return this;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public NodeRegistryEntry nodeRegistrationError(Map<String,Object> nodeInfo) {
|
public NodeRegistryEntry nodeRegistrationError(Map<String,Object> nodeInfo) {
|
||||||
registration.putAll(StrUtil.deepFlattenMap(nodeInfo));
|
return _updateEntry(STATE.REGISTRATION_ERROR, registration, false, nodeInfo);
|
||||||
setState(STATE.REGISTRATION_ERROR);
|
|
||||||
return this;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public NodeRegistryEntry nodeRegistrationError(Throwable t) {
|
public NodeRegistryEntry nodeRegistrationError(Throwable t) {
|
||||||
registration.putAll(StrUtil.deepFlattenMap(Collections.singletonMap("exception", t)));
|
return _updateEntry(STATE.REGISTRATION_ERROR, registration, false, Collections.singletonMap("exception", t));
|
||||||
setState(STATE.REGISTRATION_ERROR);
|
|
||||||
return this;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public NodeRegistryEntry nodeDisconnected(Map<String,Object> nodeInfo) {
|
public NodeRegistryEntry nodeDisconnected(Map<String,Object> nodeInfo) {
|
||||||
registration.putAll(StrUtil.deepFlattenMap(nodeInfo));
|
return _updateEntry(STATE.DISCONNECTED, registration, false, nodeInfo);
|
||||||
setState(STATE.DISCONNECTED);
|
|
||||||
return this;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public NodeRegistryEntry nodeDisconnected(Throwable t) {
|
public NodeRegistryEntry nodeDisconnected(Throwable t) {
|
||||||
registration.putAll(StrUtil.deepFlattenMap(Collections.singletonMap("exception", t)));
|
return _updateEntry(STATE.DISCONNECTED, registration, false, Collections.singletonMap("exception", t));
|
||||||
setState(STATE.DISCONNECTED);
|
|
||||||
return this;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public NodeRegistryEntry nodeExiting(Map<String,Object> nodeInfo) {
|
public NodeRegistryEntry nodeExiting(Map<String,Object> nodeInfo) {
|
||||||
registration.putAll(StrUtil.deepFlattenMap(nodeInfo));
|
return _updateEntry(STATE.EXITING, registration, false, nodeInfo);
|
||||||
setState(STATE.EXITING);
|
|
||||||
return this;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public NodeRegistryEntry nodeExited(Map<String,Object> nodeInfo) {
|
public NodeRegistryEntry nodeExited(Map<String,Object> nodeInfo) {
|
||||||
registration.putAll(StrUtil.deepFlattenMap(nodeInfo));
|
return _updateEntry(STATE.EXITED, registration, false, nodeInfo);
|
||||||
setState(STATE.EXITED);
|
|
||||||
return this;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public NodeRegistryEntry nodeFailed(Map<String,Object> failInfo) {
|
public NodeRegistryEntry nodeFailed(Map<String,Object> failInfo) {
|
||||||
if (failInfo!=null)
|
return _updateEntry(STATE.NODE_FAILED, registration, false, failInfo);
|
||||||
registration.putAll(StrUtil.deepFlattenMap(failInfo));
|
}
|
||||||
setState(STATE.NODE_FAILED);
|
|
||||||
return this;
|
public NodeRegistryEntry nodeRemoving(Map<String,Object> nodeInfo) {
|
||||||
|
return _updateEntry(STATE.REMOVING, removal, false, nodeInfo);
|
||||||
|
}
|
||||||
|
|
||||||
|
public NodeRegistryEntry nodeRemoved(Map<String,Object> nodeInfo) {
|
||||||
|
return _updateEntry(STATE.REMOVED, removal, false, nodeInfo);
|
||||||
|
}
|
||||||
|
|
||||||
|
public NodeRegistryEntry nodeRemoveError(Map<String,Object> failInfo) {
|
||||||
|
return _updateEntry(STATE.REMOVE_ERROR, removal, false, failInfo);
|
||||||
|
}
|
||||||
|
|
||||||
|
public NodeRegistryEntry nodeArchived(Map<String,Object> nodeInfo) {
|
||||||
|
return _updateEntry(STATE.ARCHIVED, removal, false, nodeInfo);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -53,7 +53,7 @@ public class NodeRegistrationController {
|
|||||||
return "OK";
|
return "OK";
|
||||||
}
|
}
|
||||||
|
|
||||||
@RequestMapping(value = "/baguette/registerNode", method = POST,
|
@RequestMapping(value = { "/baguette/registerNode", "/baguette/node/register" }, method = POST,
|
||||||
consumes = MediaType.APPLICATION_JSON_VALUE)
|
consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||||
public String baguetteRegisterNode(@RequestBody String jsonNode, HttpServletRequest request) throws Exception {
|
public String baguetteRegisterNode(@RequestBody String jsonNode, HttpServletRequest request) throws Exception {
|
||||||
log.info("NodeRegistrationController.baguetteRegisterNode(): Invoked");
|
log.info("NodeRegistrationController.baguetteRegisterNode(): Invoked");
|
||||||
@ -80,6 +80,26 @@ public class NodeRegistrationController {
|
|||||||
return response;
|
return response;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@RequestMapping(value = "/baguette/node/unregister/{ipAddress:.+}", method = {GET, POST},
|
||||||
|
produces = MediaType.TEXT_PLAIN_VALUE)
|
||||||
|
public String baguetteUnregisterNode(@PathVariable String ipAddress, HttpServletRequest request) throws Exception {
|
||||||
|
log.info("NodeRegistrationController.baguetteUnregisterNode(): Invoked");
|
||||||
|
log.debug("NodeRegistrationController.baguetteUnregisterNode(): Node IP address:\n{}", ipAddress);
|
||||||
|
|
||||||
|
String response;
|
||||||
|
try {
|
||||||
|
response = nodeRegistrationCoordinator.unregisterNode(ipAddress,
|
||||||
|
coordinator.getTranslationContextOfAppModel(coordinator.getCurrentAppModelId()));
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.error("NodeRegistrationController.baguetteUnregisterNode(): EXCEPTION while unregistering node: address={}\n", ipAddress, e);
|
||||||
|
response = "ERROR "+e.getMessage();
|
||||||
|
}
|
||||||
|
|
||||||
|
log.info("NodeRegistrationController.baguetteUnregisterNode(): Node unregistered: node-address={}", ipAddress);
|
||||||
|
log.debug("NodeRegistrationController.baguetteUnregisterNode(): address={}, json={}", ipAddress, response);
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
|
||||||
@RequestMapping(value = "/baguette/node/list", method = GET)
|
@RequestMapping(value = "/baguette/node/list", method = GET)
|
||||||
public Collection<String> baguetteNodeList() throws Exception {
|
public Collection<String> baguetteNodeList() throws Exception {
|
||||||
log.info("NodeRegistrationController.baguetteNodeList(): Invoked");
|
log.info("NodeRegistrationController.baguetteNodeList(): Invoked");
|
||||||
|
@ -12,15 +12,19 @@ package gr.iccs.imu.ems.control.controller;
|
|||||||
import gr.iccs.imu.ems.baguette.client.install.ClientInstallationTask;
|
import gr.iccs.imu.ems.baguette.client.install.ClientInstallationTask;
|
||||||
import gr.iccs.imu.ems.baguette.client.install.ClientInstaller;
|
import gr.iccs.imu.ems.baguette.client.install.ClientInstaller;
|
||||||
import gr.iccs.imu.ems.baguette.client.install.helper.InstallationHelperFactory;
|
import gr.iccs.imu.ems.baguette.client.install.helper.InstallationHelperFactory;
|
||||||
|
import gr.iccs.imu.ems.baguette.client.selfhealing.ClientRecoveryPlugin;
|
||||||
import gr.iccs.imu.ems.baguette.server.BaguetteServer;
|
import gr.iccs.imu.ems.baguette.server.BaguetteServer;
|
||||||
|
import gr.iccs.imu.ems.baguette.server.ClientShellCommand;
|
||||||
import gr.iccs.imu.ems.baguette.server.NodeRegistryEntry;
|
import gr.iccs.imu.ems.baguette.server.NodeRegistryEntry;
|
||||||
import gr.iccs.imu.ems.control.properties.ControlServiceProperties;
|
import gr.iccs.imu.ems.control.properties.ControlServiceProperties;
|
||||||
import gr.iccs.imu.ems.control.properties.StaticResourceProperties;
|
import gr.iccs.imu.ems.control.properties.StaticResourceProperties;
|
||||||
import gr.iccs.imu.ems.translate.TranslationContext;
|
import gr.iccs.imu.ems.translate.TranslationContext;
|
||||||
|
import gr.iccs.imu.ems.util.EventBus;
|
||||||
import gr.iccs.imu.ems.util.NetUtil;
|
import gr.iccs.imu.ems.util.NetUtil;
|
||||||
import gr.iccs.imu.ems.util.StrUtil;
|
import gr.iccs.imu.ems.util.StrUtil;
|
||||||
import jakarta.servlet.http.HttpServletRequest;
|
import jakarta.servlet.http.HttpServletRequest;
|
||||||
import lombok.Getter;
|
import lombok.Getter;
|
||||||
|
import lombok.NonNull;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.SneakyThrows;
|
import lombok.SneakyThrows;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
@ -35,6 +39,7 @@ import org.springframework.web.servlet.support.ServletUriComponentsBuilder;
|
|||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.Callable;
|
||||||
import java.util.concurrent.atomic.AtomicBoolean;
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
|
|
||||||
@Slf4j
|
@Slf4j
|
||||||
@ -51,6 +56,8 @@ public class NodeRegistrationCoordinator implements InitializingBean {
|
|||||||
private final ServerProperties serverProperties;
|
private final ServerProperties serverProperties;
|
||||||
private final ServletWebServerApplicationContext webServerAppCtxt;
|
private final ServletWebServerApplicationContext webServerAppCtxt;
|
||||||
|
|
||||||
|
private final EventBus<String,Object,Object> eventBus;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void afterPropertiesSet() throws Exception {
|
public void afterPropertiesSet() throws Exception {
|
||||||
}
|
}
|
||||||
@ -130,6 +137,60 @@ public class NodeRegistrationCoordinator implements InitializingBean {
|
|||||||
return response;
|
return response;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public String unregisterNode(@NonNull String ipAddress, TranslationContext translationContext) throws Exception {
|
||||||
|
log.debug("NodeRegistrationCoordinator.unregisterNode(): BEGIN: ip-address={}", ipAddress);
|
||||||
|
if (StringUtils.isBlank(ipAddress)) {
|
||||||
|
log.error("NodeRegistrationCoordinator.unregisterNode(): Blank IP address provided");
|
||||||
|
throw new IllegalArgumentException("Blank IP address provided");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Retrieve node entry from registry
|
||||||
|
NodeRegistryEntry entry = baguetteServer.getNodeRegistry().getNodeByAddress(ipAddress);
|
||||||
|
if (entry==null) {
|
||||||
|
log.error("NodeRegistrationCoordinator.unregisterNode(): Node not found in registry: address={}", ipAddress);
|
||||||
|
throw new IllegalArgumentException("Node not found in registry: address="+ipAddress);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set node state to REMOVING in order to prevent self-healing
|
||||||
|
entry.nodeRemoving(null);
|
||||||
|
|
||||||
|
// Signal self-healing plugin to cancel any pending recovery tasks
|
||||||
|
log.debug("NodeRegistrationCoordinator.unregisterNode(): Notifying Self-healing to remove any pending recovery tasks: address={}", ipAddress);
|
||||||
|
eventBus.send(ClientRecoveryPlugin.CLIENT_REMOVED_TOPIC, entry);
|
||||||
|
|
||||||
|
// Close CSC connection
|
||||||
|
ClientShellCommand csc = ClientShellCommand.getActiveByIpAddress(entry.getIpAddress());
|
||||||
|
if (csc!=null) {
|
||||||
|
log.info("NodeRegistrationCoordinator.unregisterNode(): Closing connection to EMS client: address={}", ipAddress);
|
||||||
|
csc.stop("REMOVING NODE");
|
||||||
|
} else
|
||||||
|
log.warn("NodeRegistrationCoordinator.unregisterNode(): CSC is null. Cannot close connection to EMS client. Probably connection has already closed: address={}", ipAddress);
|
||||||
|
|
||||||
|
// Continue processing according to ExecutionWare type
|
||||||
|
String response;
|
||||||
|
log.info("NodeRegistrationCoordinator.unregisterNode(): ExecutionWare: {}", properties.getExecutionware());
|
||||||
|
if (properties.getExecutionware()==ControlServiceProperties.ExecutionWare.CLOUDIATOR) {
|
||||||
|
response = "NOT SUPPORTED";
|
||||||
|
} else {
|
||||||
|
response = createClientUninstallTask(entry, translationContext, () -> {
|
||||||
|
// Unregister and Archive node
|
||||||
|
try {
|
||||||
|
entry.nodeRemoved(null);
|
||||||
|
baguetteServer.unregisterClient(entry);
|
||||||
|
return "OK";
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.error("NodeRegistrationCoordinator.unregisterNode(): EXCEPTION while unregistering node: address={}, entry={}\n", ipAddress, entry, e);
|
||||||
|
entry.nodeRemoveError(Map.of(
|
||||||
|
"error", e.getMessage()
|
||||||
|
));
|
||||||
|
return "ERROR " + e.getMessage();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
|
||||||
void updateRegistrationInfo(Map<String, String> nodeMap, String baseUrl) {
|
void updateRegistrationInfo(Map<String, String> nodeMap, String baseUrl) {
|
||||||
// Set OS info
|
// Set OS info
|
||||||
String os = StringUtils.isNotBlank(nodeMap.get("operatingSystem.name"))
|
String os = StringUtils.isNotBlank(nodeMap.get("operatingSystem.name"))
|
||||||
@ -211,14 +272,32 @@ public class NodeRegistrationCoordinator implements InitializingBean {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public String createClientInstallationTask(NodeRegistryEntry entry, TranslationContext translationContext) throws Exception {
|
public String createClientInstallationTask(NodeRegistryEntry entry, TranslationContext translationContext) throws Exception {
|
||||||
//log.info("ControlServiceController.baguetteRegisterNodeForProactive(): INPUT: node-map: {}", nodeMap);
|
return createClientInstallationTask(entry, translationContext, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
public String createClientInstallationTask(NodeRegistryEntry entry, TranslationContext translationContext, Callable<String> callback) throws Exception {
|
||||||
ClientInstallationTask installationTask = InstallationHelperFactory.getInstance()
|
ClientInstallationTask installationTask = InstallationHelperFactory.getInstance()
|
||||||
.createInstallationHelper(entry)
|
.createInstallationHelper(entry)
|
||||||
.createClientInstallationTask(entry, translationContext);
|
.createClientInstallationTask(entry, translationContext);
|
||||||
|
installationTask.setCallback(callback);
|
||||||
ClientInstaller.instance().addTask(installationTask);
|
ClientInstaller.instance().addTask(installationTask);
|
||||||
log.debug("NodeRegistrationCoordinator.createClientInstallationTask(): New installation-task: {}", installationTask);
|
log.debug("NodeRegistrationCoordinator.createClientInstallationTask(): New installation-task: {}", installationTask);
|
||||||
|
|
||||||
return "OK";
|
return "OK";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public String createClientUninstallTask(NodeRegistryEntry entry, TranslationContext translationContext) throws Exception {
|
||||||
|
return createClientUninstallTask(entry, translationContext, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
public String createClientUninstallTask(NodeRegistryEntry entry, TranslationContext translationContext, Callable<String> callback) throws Exception {
|
||||||
|
ClientInstallationTask uninstallTask = InstallationHelperFactory.getInstance()
|
||||||
|
.createInstallationHelper(entry)
|
||||||
|
.createClientUninstallTask(entry, translationContext);
|
||||||
|
uninstallTask.setCallback(callback);
|
||||||
|
ClientInstaller.instance().addTask(uninstallTask);
|
||||||
|
log.debug("NodeRegistrationCoordinator.createClientUninstallTask(): New uninstall-task: {}", uninstallTask);
|
||||||
|
|
||||||
|
return "OK";
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -36,4 +36,11 @@ public class NodeRegistrationHelper implements InitializingBean, INodeRegistrati
|
|||||||
baseUrl, nodeInfo, translationContext);
|
baseUrl, nodeInfo, translationContext);
|
||||||
return nodeRegistrationCoordinator.registerNode(baseUrl, nodeInfo, translationContext);
|
return nodeRegistrationCoordinator.registerNode(baseUrl, nodeInfo, translationContext);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String unregisterNode(String nodeAddress, TranslationContext translationContext) throws Exception {
|
||||||
|
log.debug("NodeRegistrationHelper: Invoking unregisterNode: node-address={}, TC={}",
|
||||||
|
nodeAddress, translationContext);
|
||||||
|
return nodeRegistrationCoordinator.unregisterNode(nodeAddress, translationContext);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
Loading…
x
Reference in New Issue
Block a user