use of org.ow2.proactive.resourcemanager.core.RMCore in project scheduling by ow2-proactive.
the class RMCore method executeScript.
/**
* {@inheritDoc}
*/
public <T> List<ScriptResult<T>> executeScript(Script<T> script, String targetType, Set<String> targets) {
// Depending on the target type, select nodes for script execution
final TargetType tType = TargetType.valueOf(targetType);
final HashSet<RMNode> selectedRMNodes = new HashSet<>();
switch(tType) {
case NODESOURCE_NAME:
// If target is a nodesource name select all its nodes
for (String target : targets) {
NodeSource nodeSource = this.deployedNodeSources.get(target);
if (nodeSource != null) {
for (RMNode candidateNode : this.allNodes.values()) {
if (candidateNode.getNodeSource().equals(nodeSource)) {
this.selectCandidateNode(selectedRMNodes, candidateNode);
}
}
}
}
break;
case NODE_URL:
// If target is node url select the node
for (String target : targets) {
RMNode candidateNode = this.allNodes.get(target);
if (candidateNode != null) {
this.selectCandidateNode(selectedRMNodes, candidateNode);
}
}
break;
case HOSTNAME:
// If target is hostname select first node from that host
for (String target : targets) {
for (RMNode node : this.allNodes.values()) {
if (node.getHostName().equals(target)) {
this.selectCandidateNode(selectedRMNodes, node);
break;
}
}
}
break;
default:
throw new IllegalArgumentException("Unable to execute script, unknown target type: " + targetType);
}
// Return a ProActive future on the list of results
return this.selectionManager.executeScript(script, selectedRMNodes, null);
// To avoid blocking rmcore ao the call is delegated to the selection
// manager ao and each node is unlocked as soon as the script has
// finished it's execution.
}
use of org.ow2.proactive.resourcemanager.core.RMCore in project scheduling by ow2-proactive.
the class NodeSource method detectedPingedDownNode.
/**
* Marks node as down. Remove it from node source node set. It remains in rmcore nodes list until
* user decides to remove them or node source is shutdown.
* @see NodeSource#detectedPingedDownNode(String, String)
*/
public void detectedPingedDownNode(String nodeName, String nodeUrl) {
if (toShutdown) {
logger.warn("[" + name + "] detectedPingedDownNode request discarded because node source is shutting down");
return;
}
logger.warn("[" + name + "] Detected down node: " + nodeUrl);
Node downNode = nodes.remove(nodeUrl);
if (downNode != null) {
downNodes.put(nodeUrl, downNode);
try {
RMCore.topologyManager.removeNode(downNode);
infrastructureManager.internalNotifyDownNode(nodeName, nodeUrl, downNode);
} catch (RMException e) {
logger.error("Error while removing down node: " + nodeUrl, e);
}
} else {
// almost no information about the node apart from its name and url
try {
infrastructureManager.internalNotifyDownNode(nodeName, nodeUrl, null);
} catch (RMException e) {
logger.error("New empty node " + nodeUrl + " could not be created to handle down node", e);
}
}
rmcore.setDownNode(nodeUrl);
}
use of org.ow2.proactive.resourcemanager.core.RMCore in project scheduling by ow2-proactive.
the class BatchJobInfrastructure method startNode.
/**
* Builds the command line to execute on the PBS frontend and wait for every
* launched nodes to register. If the node doesn't register (ie. runs
* {@link #internalRegisterAcquiredNode(Node)} isn't called) before the
* timeout (configurable) value, an exception is raised. If the qSub command
* submitted to the PBS frontend fails, the node supposed to be launched is
* not expected anymore and will be discarded at registration time.
*
* @throws RMException
*/
private void startNode() throws RMException {
CommandLineBuilder clb = new CommandLineBuilder();
// generate the node name
// current rmcore shortID should be added to ensure uniqueness
String nodeName = getBatchinJobSystemName() + "-" + nodeSource.getName() + "-" + ProActiveCounter.getUniqID();
clb.setNodeName(nodeName);
clb.setJavaPath(this.javaPath);
clb.setRmURL(getRmUrl());
clb.setRmHome(this.schedulingPath);
clb.setSourceName(this.nodeSource.getName());
clb.setPaProperties(this.javaOptions);
try {
clb.setCredentialsValueAndNullOthers(new String(getCredentials().getBase64()));
} catch (KeyException e) {
this.handleFailedDeployment(clb, e);
}
InetAddress host = null;
try {
host = InetAddress.getByName(this.serverName);
} catch (UnknownHostException e) {
this.handleFailedDeployment(clb, e);
}
String deleteCmd = getDeleteJobCommand();
String submitCmd = getSubmitJobCommand();
// build the command: echo "script.sh params"|qsub params
String cmd = null;
String obfuscatedCmd = null;
try {
cmd = "echo \\\"" + clb.buildCommandLine(true).replace("\"", "\\\"") + "\\\" | " + submitCmd + " " + this.submitJobOpt;
obfuscatedCmd = "echo \\\"" + clb.buildCommandLine(false).replace("\"", "\\\"") + "\\\" | " + submitCmd + " " + this.submitJobOpt;
} catch (IOException e) {
this.handleFailedDeployment(clb, e);
}
// add an deploying node.
final String dnURL = super.addDeployingNode(nodeName, obfuscatedCmd, "Deploying node on " + getBatchinJobSystemName() + " scheduler", this.nodeTimeOut);
putPnTimeout(dnURL, Boolean.FALSE);
// executing the command
Process p;
try {
p = Utils.runSSHCommand(host, cmd, this.sshOptions);
} catch (IOException e1) {
throw new RMException("Cannot execute ssh command: " + cmd + " on host: " + this.serverName, e1);
}
// recover the Job ID through stdout
String id = "";
InputStream in = p.getInputStream();
int b = -1;
try {
while ((b = in.read()) > -1) {
id += (char) b;
}
} catch (IOException e) {
}
// check for registration
// at this point, the ssh process should have already exited because it
// only handle the job submission, not the execution... furthermore
// the "id" is defined
String lf = System.lineSeparator();
final long timeout = nodeTimeOut;
long t1 = System.currentTimeMillis();
// Hack. SSHClient fails but qSub succeeds.
boolean isJobIDValid = false;
// Tries to wait for this node
// registration...
int circuitBreakerThreshold = 5;
while (!getPnTimeout(dnURL) && circuitBreakerThreshold > 0) {
try {
int exitCode = p.exitValue();
if (exitCode != 0 && !isJobIDValid) {
logger.warn("SSH subprocess at " + host.getHostName() + " exit code != 0 but IM tries to recover from this error...Current submit command's output: " + id + " and associated node's name: " + nodeName);
String extractedID = this.extractSubmitOutput(id);
String errput = this.extractProcessErrput(p);
final String description = "SSH command failed to launch node on " + getBatchinJobSystemName() + " scheduler" + lf + " >Error code: " + exitCode + lf + " >Errput: " + errput + " >Output: " + id;
// registration...
if (extractedID != null && !extractedID.equals("")) {
isJobIDValid = true;
}
// defines how to recover from this state
// throws a RMException if we can't
handleWrongJobTermination(isJobIDValid, nodeName, dnURL, host, id, description, exitCode, submitCmd, deleteCmd);
}
} catch (IllegalThreadStateException e) {
// process has not returned yet
logger.trace("Waiting for ssh process to exit in BatchJobInfrastructure");
}
if (super.checkNodeIsAcquiredAndDo(nodeName, null, null)) {
// registration is ok
p.destroy();
addNodeAndDecrementDeployingNode(nodeName, this.extractSubmitOutput(id));
return;
}
try {
logger.debug("Waiting for node " + nodeName + " registration... time to timeout: " + (timeout - (System.currentTimeMillis() - t1)));
Thread.sleep(BatchJobInfrastructure.NODE_ACQUISITION_CHECK_RATE);
} catch (Exception e) {
circuitBreakerThreshold--;
logger.error("While monitoring ssh subprocess.", e);
}
}
// end of while loop, either deploying node timeout/removed of
// threshold reached
// the node is not expected anymore
atomicRemovePnTimeoutAndJob(nodeName, dnURL, p, id);
if (circuitBreakerThreshold <= 0) {
logger.error("Circuit breaker threshold reached while monitoring ssh subprocess.");
throw new RMException("Several exceptions occurred while monitoring ssh subprocess.");
}
// if we are here we reached an invalid state
throw new RMException("Invalid state, exit from a control loop with threshold > 0 and expected deploying node");
}
use of org.ow2.proactive.resourcemanager.core.RMCore in project scheduling by ow2-proactive.
the class InfrastructureManager method internalRegisterAcquiredNode.
/**
* This method is called by the RMCore to notify the InfrastructureManager
* that a new node was added to the core. If the IM throws an exception, the
* node is not added. Note that this method is in mutual exclusion with
* {@link #checkNodeIsAcquiredAndDo(String, Runnable, Runnable)} At this
* point, if a previous call to
* {@link InfrastructureManager#addDeployingNode(String, String, String, long)}
* was made (means that implementor uses deploying nodes features), this
* method ensures that the implementation method (see
* {@link InfrastructureManager#notifyAcquiredNode(Node)} is only called if
* no timeout has occurred for the associated deploying node.
*
* @param node
* the newly added node
* @throws RMException
*/
public final RMDeployingNode internalRegisterAcquiredNode(Node node) throws RMException {
if (!isUsingDeployingNode()) {
this.notifyAcquiredNode(node);
return null;
}
// here we use deploying nodes and timeout
RMDeployingNode pn;
// we build the url of the associated deploying node
String deployingNodeURL = this.buildDeployingNodeURL(node.getNodeInformation().getName());
writeLock.lock();
try {
pn = removeDeployingNodeWithLockAndPersist(deployingNodeURL);
// implementation callback
if (pn != null) {
RMNodeEvent event = pn.createNodeEvent(RMEventType.NODE_REMOVED, pn.getState(), pn.getProvider().getName());
emitEvent(event);
this.notifyAcquiredNode(node);
// if everything went well with the new node, caching it
addAcquiredNodeNameWithLockAndPersist(node.getNodeInformation().getName());
} else {
String url = node.getNodeInformation().getURL();
logger.warn("Not expected node registered, discarding it: " + url);
throw new RMException("Not expected node registered, discarding it: " + url);
}
} catch (RuntimeException e) {
logger.error("Exception while moving deploying node to acquired node", e);
throw e;
} finally {
writeLock.unlock();
}
return pn;
}
use of org.ow2.proactive.resourcemanager.core.RMCore in project scheduling by ow2-proactive.
the class SSHInfrastructure method startNodeImpl.
/**
* Internal node acquisition method
* <p>
* Starts a PA runtime on remote host using SSH, register it manually in the
* nodesource.
*
* @param hostTracker The host on which one the node will be started
* @param nbNodes number of nodes to deploy
* @param depNodeURLs list of deploying or lost nodes urls created
* @throws RMException
* acquisition failed
*/
protected void startNodeImpl(HostTracker hostTracker, int nbNodes, final List<String> depNodeURLs) throws RMException {
String fs = getTargetOSObj().fs;
CommandLineBuilder clb = super.getDefaultCommandLineBuilder(getTargetOSObj());
// we take care of spaces in java path
clb.setJavaPath(this.javaPath);
// we set the rm.home prop
clb.setRmHome(schedulingPath);
// we set the java security policy file
StringBuilder sb = new StringBuilder();
final boolean containsSpace = schedulingPath.contains(" ");
String securitycmd = CentralPAPropertyRepository.JAVA_SECURITY_POLICY.getCmdLine();
if (!this.javaOptions.contains(securitycmd)) {
sb.append(securitycmd);
if (containsSpace) {
sb.append("\"");
}
sb.append(schedulingPath);
sb.append(fs);
sb.append("config");
sb.append(fs);
sb.append("security.java.policy-client");
if (containsSpace) {
sb.append("\"");
}
sb.append(" ");
}
// we set the log4j configuration file
String log4jcmd = CentralPAPropertyRepository.LOG4J.getCmdLine();
if (!this.javaOptions.contains(log4jcmd)) {
sb.append(log4jcmd);
if (containsSpace) {
sb.append("\"");
}
// log4j only understands urls
sb.append("file:");
if (!schedulingPath.startsWith("/")) {
sb.append("/" + schedulingPath.replace("\\", "/"));
} else {
sb.append(schedulingPath.replace("\\", "/"));
}
sb.append("/");
sb.append("config");
sb.append("/");
sb.append("log");
sb.append("/");
sb.append("node.properties");
if (containsSpace) {
sb.append("\"");
}
sb.append(" ");
}
// we add extra java/PA configuration
sb.append(this.javaOptions);
clb.setPaProperties(sb.toString());
// afterwards, node's name
// generate the node name
// current rmcore shortID should be added to ensure uniqueness
final String nodeName = nodeNameBuilder.generateNodeName(hostTracker);
clb.setNodeName(nodeName);
clb.setNumberOfNodes(nbNodes);
// finally, the credential's value
String credString = null;
try {
credString = new String(getCredentials().getBase64());
} catch (KeyException e1) {
throw new RMException("Could not get base64 credentials", e1);
}
clb.setCredentialsValueAndNullOthers(credString);
// add an expected node. every unexpected node will be discarded
String cmdLine;
String obfuscatedCmdLine;
try {
cmdLine = clb.buildCommandLine(true);
obfuscatedCmdLine = clb.buildCommandLine(false);
} catch (IOException e2) {
throw new RMException("Cannot build the " + RMNodeStarter.class.getSimpleName() + "'s command line.", e2);
}
// one escape the command to make it runnable through ssh
if (cmdLine.contains("\"")) {
cmdLine = cmdLine.replaceAll("\"", "\\\\\"");
}
// we create a new deploying node before ssh command ran
final List<String> createdNodeNames = RMNodeStarter.getWorkersNodeNames(nodeName, nbNodes);
depNodeURLs.addAll(addMultipleDeployingNodes(createdNodeNames, obfuscatedCmdLine, "Deploying nodes on host " + hostTracker.getResolvedAddress(), super.nodeTimeOut));
addTimeouts(depNodeURLs);
Process p = null;
try {
p = Utils.runSSHCommand(hostTracker.getResolvedAddress(), cmdLine, sshOptions);
} catch (IOException e1) {
multipleDeclareDeployingNodeLost(depNodeURLs, "Cannot run command: " + cmdLine + ", with ssh options: " + sshOptions + " -\n The following exception occutred:\n " + getStackTraceAsString(e1));
throw new RMException("Cannot run command: " + cmdLine + ", with ssh options: " + sshOptions, e1);
}
String lf = System.lineSeparator();
int circuitBreakerThreshold = 5;
while (!anyTimedOut(depNodeURLs) && circuitBreakerThreshold > 0) {
try {
int exitCode = p.exitValue();
if (exitCode != 0) {
logger.error("SSH subprocess at " + hostTracker.getResolvedAddress().getHostName() + " exited abnormally (" + exitCode + ").");
} else {
logger.error("Launching node process has exited normally whereas it shouldn't.");
}
String pOutPut = Utils.extractProcessOutput(p);
String pErrPut = Utils.extractProcessErrput(p);
final String description = "SSH command failed to launch node on host " + hostTracker.getResolvedAddress().getHostName() + lf + " >Error code: " + exitCode + lf + " >Errput: " + pErrPut + " >Output: " + pOutPut;
logger.error(description);
if (super.checkAllNodesAreAcquiredAndDo(createdNodeNames, null, new Runnable() {
public void run() {
SSHInfrastructure.this.multipleDeclareDeployingNodeLost(depNodeURLs, description);
}
})) {
return;
} else {
// there isn't any race regarding node registration
throw new RMException("SSH Node " + nodeName + " is not expected anymore because of an error.");
}
} catch (IllegalThreadStateException e) {
logger.trace("IllegalThreadStateException while waiting for " + nodeName + " registration");
}
if (super.checkNodeIsAcquiredAndDo(nodeName, null, null)) {
// registration is ok, we destroy the process
p.destroy();
return;
}
try {
Thread.sleep(1000);
} catch (Exception e) {
circuitBreakerThreshold--;
logger.trace("An exception occurred while monitoring ssh subprocess", e);
}
}
// if we exit because of a timeout
if (anyTimedOut(depNodeURLs)) {
// we remove it
removeTimeouts(depNodeURLs);
// we destroy the process
p.destroy();
throw new RMException("Deploying Node " + nodeName + " not expected any more");
}
if (circuitBreakerThreshold <= 0) {
logger.error("Circuit breaker threshold reached while monitoring ssh subprocess.");
throw new RMException("Several exceptions occurred while monitoring ssh subprocess.");
}
}
Aggregations