use of java.security.KeyException in project scheduling by ow2-proactive.
the class SchedulerStateRest method getCreateCredential.
/**
* generates a credential file from user provided credentials
*
* @return the credential file generated by the scheduler
* @throws LoginException
* @throws SchedulerRestException
*/
@Override
@POST
@Consumes(MediaType.MULTIPART_FORM_DATA)
@Path("createcredential")
@Produces("*/*")
public byte[] getCreateCredential(@MultipartForm LoginForm multipart) throws LoginException, SchedulerRestException {
String username = multipart.getUsername();
String password = multipart.getPassword();
byte[] privKey = multipart.getSshKey();
try {
String url = PortalConfiguration.SCHEDULER_URL.getValueAsString();
SchedulerAuthenticationInterface auth = SchedulerConnection.join(url);
PublicKey pubKey = auth.getPublicKey();
sessionStore.create(username);
Credentials cred = Credentials.createCredentials(new CredData(CredData.parseLogin(username), CredData.parseDomain(username), password, privKey), pubKey);
return cred.getBase64();
} catch (ConnectionException | KeyException e) {
throw new SchedulerRestException(e);
}
}
use of java.security.KeyException in project scheduling by ow2-proactive.
the class AutoUpdateInfrastructure method startNodeImpl.
/**
* Internal node acquisition method
* <p>
* Starts a PA runtime on remote host using a custom script, register it
* manually in the nodesource.
*
* @param hostTracker The host on which one the node will be started
* @param nbNodes number of nodes to deploy
* @param depNodeURLs list of deploying or lost nodes urls created
* @throws org.ow2.proactive.resourcemanager.exception.RMException
* acquisition failed
*/
protected void startNodeImpl(HostTracker hostTracker, int nbNodes, final List<String> depNodeURLs) throws RMException {
final String nodeName = this.nodeSource.getName() + "-" + ProActiveCounter.getUniqID();
String credentials = "";
try {
credentials = new String(nodeSource.getAdministrator().getCredentials().getBase64());
} catch (KeyException e) {
logger.error("Invalid credentials");
return;
}
Properties localProperties = new Properties();
localProperties.put(NODE_NAME, nodeName);
localProperties.put(HOST_NAME, hostTracker.getResolvedAddress().getHostName());
localProperties.put(NODESOURCE_CREDENTIALS, credentials);
localProperties.put(NODESOURCE_NAME, nodeSource.getName());
localProperties.put(NB_NODES, nbNodes);
String filledCommand = replaceProperties(command, localProperties);
filledCommand = replaceProperties(filledCommand, System.getProperties());
final List<String> createdNodeNames = RMNodeStarter.getWorkersNodeNames(nodeName, nbNodes);
depNodeURLs.addAll(addMultipleDeployingNodes(createdNodeNames, filledCommand, "Deploying node on host " + hostTracker.getResolvedAddress(), this.nodeTimeOut));
addTimeouts(depNodeURLs);
Process p;
try {
logger.debug("Deploying node: " + nodeName);
logger.debug("Launching the command: " + filledCommand);
p = Runtime.getRuntime().exec(new String[] { "bash", "-c", filledCommand });
} catch (IOException e1) {
multipleDeclareDeployingNodeLost(depNodeURLs, "Cannot run command: " + filledCommand + " - \n The following exception occurred: " + getStackTraceAsString(e1));
throw new RMException("Cannot run command: " + filledCommand, e1);
}
String lf = System.lineSeparator();
int circuitBreakerThreshold = 5;
while (!anyTimedOut(depNodeURLs) && circuitBreakerThreshold > 0) {
try {
int exitCode = p.exitValue();
if (exitCode != 0) {
logger.error("Child process at " + hostTracker.getResolvedAddress().getHostName() + " exited abnormally (" + exitCode + ").");
} else {
logger.error("Launching node script has exited normally whereas it shouldn't.");
}
String pOutPut = Utils.extractProcessOutput(p);
String pErrPut = Utils.extractProcessErrput(p);
final String description = "Script failed to launch a node on host " + hostTracker.getResolvedAddress().getHostName() + lf + " >Error code: " + exitCode + lf + " >Errput: " + pErrPut + " >Output: " + pOutPut;
logger.error(description);
if (super.checkNodeIsAcquiredAndDo(nodeName, null, new Runnable() {
public void run() {
multipleDeclareDeployingNodeLost(depNodeURLs, description);
}
})) {
return;
} else {
// there isn't any race regarding node registration
throw new RMException("A node " + nodeName + " is not expected anymore because of an error.");
}
} catch (IllegalThreadStateException e) {
logger.trace("IllegalThreadStateException while waiting for " + nodeName + " registration");
}
if (super.checkNodeIsAcquiredAndDo(nodeName, null, null)) {
// registration is ok, we destroy the process
logger.debug("Destroying the process: " + p);
try {
ProcessTree.get().get(p).kill();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
return;
}
try {
Thread.sleep(1000);
} catch (Exception e) {
circuitBreakerThreshold--;
logger.trace("An exception occurred while monitoring a child process", e);
}
}
// if we exit because of a timeout
if (anyTimedOut(depNodeURLs)) {
// we remove it
removeTimeouts(depNodeURLs);
// we destroy the process
p.destroy();
throw new RMException("Deploying Node " + nodeName + " not expected any more");
}
if (circuitBreakerThreshold <= 0) {
logger.error("Circuit breaker threshold reached while monitoring a child process.");
throw new RMException("Several exceptions occurred while monitoring a child process.");
}
}
use of java.security.KeyException in project scheduling by ow2-proactive.
the class BatchJobInfrastructure method startNode.
/**
* Builds the command line to execute on the PBS frontend and wait for every
* launched nodes to register. If the node doesn't register (ie. runs
* {@link #internalRegisterAcquiredNode(Node)} isn't called) before the
* timeout (configurable) value, an exception is raised. If the qSub command
* submitted to the PBS frontend fails, the node supposed to be launched is
* not expected anymore and will be discarded at registration time.
*
* @throws RMException
*/
private void startNode() throws RMException {
CommandLineBuilder clb = new CommandLineBuilder();
// generate the node name
// current rmcore shortID should be added to ensure uniqueness
String nodeName = getBatchinJobSystemName() + "-" + nodeSource.getName() + "-" + ProActiveCounter.getUniqID();
clb.setNodeName(nodeName);
clb.setJavaPath(this.javaPath);
clb.setRmURL(getRmUrl());
clb.setRmHome(this.schedulingPath);
clb.setSourceName(this.nodeSource.getName());
clb.setPaProperties(this.javaOptions);
try {
clb.setCredentialsValueAndNullOthers(new String(getCredentials().getBase64()));
} catch (KeyException e) {
this.handleFailedDeployment(clb, e);
}
InetAddress host = null;
try {
host = InetAddress.getByName(this.serverName);
} catch (UnknownHostException e) {
this.handleFailedDeployment(clb, e);
}
String deleteCmd = getDeleteJobCommand();
String submitCmd = getSubmitJobCommand();
// build the command: echo "script.sh params"|qsub params
String cmd = null;
String obfuscatedCmd = null;
try {
cmd = "echo \\\"" + clb.buildCommandLine(true).replace("\"", "\\\"") + "\\\" | " + submitCmd + " " + this.submitJobOpt;
obfuscatedCmd = "echo \\\"" + clb.buildCommandLine(false).replace("\"", "\\\"") + "\\\" | " + submitCmd + " " + this.submitJobOpt;
} catch (IOException e) {
this.handleFailedDeployment(clb, e);
}
// add an deploying node.
final String dnURL = super.addDeployingNode(nodeName, obfuscatedCmd, "Deploying node on " + getBatchinJobSystemName() + " scheduler", this.nodeTimeOut);
putPnTimeout(dnURL, Boolean.FALSE);
// executing the command
Process p;
try {
p = Utils.runSSHCommand(host, cmd, this.sshOptions);
} catch (IOException e1) {
throw new RMException("Cannot execute ssh command: " + cmd + " on host: " + this.serverName, e1);
}
// recover the Job ID through stdout
String id = "";
InputStream in = p.getInputStream();
int b = -1;
try {
while ((b = in.read()) > -1) {
id += (char) b;
}
} catch (IOException e) {
}
// check for registration
// at this point, the ssh process should have already exited because it
// only handle the job submission, not the execution... furthermore
// the "id" is defined
String lf = System.lineSeparator();
final long timeout = nodeTimeOut;
long t1 = System.currentTimeMillis();
// Hack. SSHClient fails but qSub succeeds.
boolean isJobIDValid = false;
// Tries to wait for this node
// registration...
int circuitBreakerThreshold = 5;
while (!getPnTimeout(dnURL) && circuitBreakerThreshold > 0) {
try {
int exitCode = p.exitValue();
if (exitCode != 0 && !isJobIDValid) {
logger.warn("SSH subprocess at " + host.getHostName() + " exit code != 0 but IM tries to recover from this error...Current submit command's output: " + id + " and associated node's name: " + nodeName);
String extractedID = this.extractSubmitOutput(id);
String errput = this.extractProcessErrput(p);
final String description = "SSH command failed to launch node on " + getBatchinJobSystemName() + " scheduler" + lf + " >Error code: " + exitCode + lf + " >Errput: " + errput + " >Output: " + id;
// registration...
if (extractedID != null && !extractedID.equals("")) {
isJobIDValid = true;
}
// defines how to recover from this state
// throws a RMException if we can't
handleWrongJobTermination(isJobIDValid, nodeName, dnURL, host, id, description, exitCode, submitCmd, deleteCmd);
}
} catch (IllegalThreadStateException e) {
// process has not returned yet
logger.trace("Waiting for ssh process to exit in BatchJobInfrastructure");
}
if (super.checkNodeIsAcquiredAndDo(nodeName, null, null)) {
// registration is ok
p.destroy();
addNodeAndDecrementDeployingNode(nodeName, this.extractSubmitOutput(id));
return;
}
try {
logger.debug("Waiting for node " + nodeName + " registration... time to timeout: " + (timeout - (System.currentTimeMillis() - t1)));
Thread.sleep(BatchJobInfrastructure.NODE_ACQUISITION_CHECK_RATE);
} catch (Exception e) {
circuitBreakerThreshold--;
logger.error("While monitoring ssh subprocess.", e);
}
}
// end of while loop, either deploying node timeout/removed of
// threshold reached
// the node is not expected anymore
atomicRemovePnTimeoutAndJob(nodeName, dnURL, p, id);
if (circuitBreakerThreshold <= 0) {
logger.error("Circuit breaker threshold reached while monitoring ssh subprocess.");
throw new RMException("Several exceptions occurred while monitoring ssh subprocess.");
}
// if we are here we reached an invalid state
throw new RMException("Invalid state, exit from a control loop with threshold > 0 and expected deploying node");
}
use of java.security.KeyException in project scheduling by ow2-proactive.
the class LocalInfrastructure method startNodeProcess.
private void startNodeProcess(int numberOfNodes) {
int currentIndex = getIndexAndIncrement();
String baseNodeName = "local-" + this.nodeSource.getName() + "-" + currentIndex;
OperatingSystem os = OperatingSystem.UNIX;
// assuming no cygwin, windows or the "others"...
if (System.getProperty("os.name").contains("Windows")) {
os = OperatingSystem.WINDOWS;
}
String rmHome = PAResourceManagerProperties.RM_HOME.getValueAsString();
if (!rmHome.endsWith(os.fs)) {
rmHome += os.fs;
}
CommandLineBuilder clb = this.getDefaultCommandLineBuilder(os);
// RM_Home set in bin/unix/env script
clb.setRmHome(rmHome);
ArrayList<String> paPropList = new ArrayList<>();
if (!this.paProperties.contains(CentralPAPropertyRepository.JAVA_SECURITY_POLICY.getName())) {
paPropList.add(CentralPAPropertyRepository.JAVA_SECURITY_POLICY.getCmdLine() + rmHome + "config" + os.fs + "security.java.policy-client");
}
if (!this.paProperties.contains(CentralPAPropertyRepository.PA_CONFIGURATION_FILE.getName())) {
paPropList.add(CentralPAPropertyRepository.PA_CONFIGURATION_FILE.getCmdLine() + rmHome + "config" + os.fs + "network" + os.fs + "node.ini");
}
if (!this.paProperties.contains(PAResourceManagerProperties.RM_HOME.getKey())) {
paPropList.add(PAResourceManagerProperties.RM_HOME.getCmdLine() + rmHome);
}
if (!this.paProperties.contains("java.library.path")) {
paPropList.add("-Djava.library.path=" + System.getProperty("java.library.path"));
}
if (!paProperties.isEmpty()) {
Collections.addAll(paPropList, this.paProperties.split(" "));
}
clb.setPaProperties(paPropList);
clb.setNodeName(baseNodeName);
clb.setNumberOfNodes(numberOfNodes);
try {
clb.setCredentialsValueAndNullOthers(new String(this.credentials.getBase64()));
} catch (KeyException e) {
createLostNodes(baseNodeName, numberOfNodes, "Cannot decrypt credentials value", e);
return;
}
List<String> cmd;
try {
cmd = clb.buildCommandLineAsList(false);
} catch (IOException e) {
createLostNodes(baseNodeName, numberOfNodes, "Cannot build command line", e);
return;
}
// The printed cmd with obfuscated credentials
final String obfuscatedCmd = Joiner.on(' ').join(cmd);
List<String> depNodeURLs = new ArrayList<>(numberOfNodes);
final List<String> createdNodeNames = RMNodeStarter.getWorkersNodeNames(baseNodeName, numberOfNodes);
ProcessExecutor processExecutor = null;
try {
depNodeURLs.addAll(addMultipleDeployingNodes(createdNodeNames, obfuscatedCmd, "Node launched locally", this.nodeTimeout));
// Deobfuscate the cred value
Collections.replaceAll(cmd, CommandLineBuilder.OBFUSC, clb.getCredentialsValue());
processExecutor = new ProcessExecutor(baseNodeName, cmd, false, true);
processExecutor.start();
processExecutors.put(processExecutor, depNodeURLs);
final ProcessExecutor tmpProcessExecutor = processExecutor;
Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
@Override
public void run() {
if (tmpProcessExecutor != null && !tmpProcessExecutor.isProcessFinished()) {
tmpProcessExecutor.killProcess();
}
}
}));
logger.info("Local Nodes command started : " + obfuscatedCmd);
} catch (IOException e) {
String lf = System.lineSeparator();
String mess = "Cannot launch rm node " + baseNodeName + lf + Throwables.getStackTraceAsString(e);
multipleDeclareDeployingNodeLost(depNodeURLs, mess);
if (processExecutor != null) {
cleanProcess(processExecutor);
}
return;
}
// watching process
int threshold = 10;
while (!allNodesAcquiredOrLost()) {
if (processExecutor.isProcessFinished()) {
int exit = processExecutor.getExitCode();
if (exit != 0) {
String lf = System.lineSeparator();
String message = "RMNode exit code == " + exit + lf;
message += "Command: " + obfuscatedCmd + lf;
String out = Joiner.on('\n').join(processExecutor.getOutput());
String err = Joiner.on('\n').join(processExecutor.getErrorOutput());
message += "stdout: " + out + lf + "stderr: " + err;
multipleDeclareDeployingNodeLost(depNodeURLs, message);
}
} else {
logger.debug("Waiting for nodes " + baseNodeName + " acquisition");
}
try {
Thread.sleep(500);
} catch (InterruptedException e) {
logger.warn("Interrupted while waiting for local process status", e);
threshold--;
if (threshold <= 0) {
break;
}
}
}
logger.debug("Local Infrastructure manager exits watching loop for nodes " + baseNodeName);
logNodeOutput(baseNodeName + " stdout: ", processExecutor.getOutput());
logNodeOutput(baseNodeName + " stderr: ", processExecutor.getErrorOutput());
if (allNodesLost(numberOfNodes)) {
// clean up the process
cleanProcess(processExecutor);
}
}
use of java.security.KeyException in project scheduling by ow2-proactive.
the class SSHInfrastructure method startNodeImpl.
/**
* Internal node acquisition method
* <p>
* Starts a PA runtime on remote host using SSH, register it manually in the
* nodesource.
*
* @param hostTracker The host on which one the node will be started
* @param nbNodes number of nodes to deploy
* @param depNodeURLs list of deploying or lost nodes urls created
* @throws RMException
* acquisition failed
*/
protected void startNodeImpl(HostTracker hostTracker, int nbNodes, final List<String> depNodeURLs) throws RMException {
String fs = getTargetOSObj().fs;
CommandLineBuilder clb = super.getDefaultCommandLineBuilder(getTargetOSObj());
// we take care of spaces in java path
clb.setJavaPath(this.javaPath);
// we set the rm.home prop
clb.setRmHome(schedulingPath);
// we set the java security policy file
StringBuilder sb = new StringBuilder();
final boolean containsSpace = schedulingPath.contains(" ");
String securitycmd = CentralPAPropertyRepository.JAVA_SECURITY_POLICY.getCmdLine();
if (!this.javaOptions.contains(securitycmd)) {
sb.append(securitycmd);
if (containsSpace) {
sb.append("\"");
}
sb.append(schedulingPath);
sb.append(fs);
sb.append("config");
sb.append(fs);
sb.append("security.java.policy-client");
if (containsSpace) {
sb.append("\"");
}
sb.append(" ");
}
// we set the log4j configuration file
String log4jcmd = CentralPAPropertyRepository.LOG4J.getCmdLine();
if (!this.javaOptions.contains(log4jcmd)) {
sb.append(log4jcmd);
if (containsSpace) {
sb.append("\"");
}
// log4j only understands urls
sb.append("file:");
if (!schedulingPath.startsWith("/")) {
sb.append("/" + schedulingPath.replace("\\", "/"));
} else {
sb.append(schedulingPath.replace("\\", "/"));
}
sb.append("/");
sb.append("config");
sb.append("/");
sb.append("log");
sb.append("/");
sb.append("node.properties");
if (containsSpace) {
sb.append("\"");
}
sb.append(" ");
}
// we add extra java/PA configuration
sb.append(this.javaOptions);
clb.setPaProperties(sb.toString());
// afterwards, node's name
// generate the node name
// current rmcore shortID should be added to ensure uniqueness
final String nodeName = nodeNameBuilder.generateNodeName(hostTracker);
clb.setNodeName(nodeName);
clb.setNumberOfNodes(nbNodes);
// finally, the credential's value
String credString = null;
try {
credString = new String(getCredentials().getBase64());
} catch (KeyException e1) {
throw new RMException("Could not get base64 credentials", e1);
}
clb.setCredentialsValueAndNullOthers(credString);
// add an expected node. every unexpected node will be discarded
String cmdLine;
String obfuscatedCmdLine;
try {
cmdLine = clb.buildCommandLine(true);
obfuscatedCmdLine = clb.buildCommandLine(false);
} catch (IOException e2) {
throw new RMException("Cannot build the " + RMNodeStarter.class.getSimpleName() + "'s command line.", e2);
}
// one escape the command to make it runnable through ssh
if (cmdLine.contains("\"")) {
cmdLine = cmdLine.replaceAll("\"", "\\\\\"");
}
// we create a new deploying node before ssh command ran
final List<String> createdNodeNames = RMNodeStarter.getWorkersNodeNames(nodeName, nbNodes);
depNodeURLs.addAll(addMultipleDeployingNodes(createdNodeNames, obfuscatedCmdLine, "Deploying nodes on host " + hostTracker.getResolvedAddress(), super.nodeTimeOut));
addTimeouts(depNodeURLs);
Process p = null;
try {
p = Utils.runSSHCommand(hostTracker.getResolvedAddress(), cmdLine, sshOptions);
} catch (IOException e1) {
multipleDeclareDeployingNodeLost(depNodeURLs, "Cannot run command: " + cmdLine + ", with ssh options: " + sshOptions + " -\n The following exception occutred:\n " + getStackTraceAsString(e1));
throw new RMException("Cannot run command: " + cmdLine + ", with ssh options: " + sshOptions, e1);
}
String lf = System.lineSeparator();
int circuitBreakerThreshold = 5;
while (!anyTimedOut(depNodeURLs) && circuitBreakerThreshold > 0) {
try {
int exitCode = p.exitValue();
if (exitCode != 0) {
logger.error("SSH subprocess at " + hostTracker.getResolvedAddress().getHostName() + " exited abnormally (" + exitCode + ").");
} else {
logger.error("Launching node process has exited normally whereas it shouldn't.");
}
String pOutPut = Utils.extractProcessOutput(p);
String pErrPut = Utils.extractProcessErrput(p);
final String description = "SSH command failed to launch node on host " + hostTracker.getResolvedAddress().getHostName() + lf + " >Error code: " + exitCode + lf + " >Errput: " + pErrPut + " >Output: " + pOutPut;
logger.error(description);
if (super.checkAllNodesAreAcquiredAndDo(createdNodeNames, null, new Runnable() {
public void run() {
SSHInfrastructure.this.multipleDeclareDeployingNodeLost(depNodeURLs, description);
}
})) {
return;
} else {
// there isn't any race regarding node registration
throw new RMException("SSH Node " + nodeName + " is not expected anymore because of an error.");
}
} catch (IllegalThreadStateException e) {
logger.trace("IllegalThreadStateException while waiting for " + nodeName + " registration");
}
if (super.checkNodeIsAcquiredAndDo(nodeName, null, null)) {
// registration is ok, we destroy the process
p.destroy();
return;
}
try {
Thread.sleep(1000);
} catch (Exception e) {
circuitBreakerThreshold--;
logger.trace("An exception occurred while monitoring ssh subprocess", e);
}
}
// if we exit because of a timeout
if (anyTimedOut(depNodeURLs)) {
// we remove it
removeTimeouts(depNodeURLs);
// we destroy the process
p.destroy();
throw new RMException("Deploying Node " + nodeName + " not expected any more");
}
if (circuitBreakerThreshold <= 0) {
logger.error("Circuit breaker threshold reached while monitoring ssh subprocess.");
throw new RMException("Several exceptions occurred while monitoring ssh subprocess.");
}
}
Aggregations