use of org.apache.commons.exec.PumpStreamHandler in project zeppelin by apache.
the class MongoDbInterpreter method interpret.
@Override
public InterpreterResult interpret(String script, InterpreterContext context) {
LOGGER.debug("Run MongoDB script: {}", script);
if (StringUtils.isEmpty(script)) {
return new InterpreterResult(Code.SUCCESS);
}
String paragraphId = context.getParagraphId();
// Write script in a temporary file
// The script is enriched with extensions
final File scriptFile = new File(getScriptFileName(paragraphId));
try {
FileUtils.write(scriptFile, shellExtension + script);
} catch (IOException e) {
LOGGER.error("Can not write script in temp file", e);
return new InterpreterResult(Code.ERROR, e.getMessage());
}
InterpreterResult result = new InterpreterResult(InterpreterResult.Code.SUCCESS);
final DefaultExecutor executor = new DefaultExecutor();
final ByteArrayOutputStream errorStream = new ByteArrayOutputStream();
executor.setStreamHandler(new PumpStreamHandler(context.out, errorStream));
executor.setWatchdog(new ExecuteWatchdog(commandTimeout));
final CommandLine cmdLine = CommandLine.parse(getProperty("mongo.shell.path"));
cmdLine.addArgument("--quiet", false);
cmdLine.addArgument(dbAddress, false);
cmdLine.addArgument(scriptFile.getAbsolutePath(), false);
try {
executor.execute(cmdLine);
runningProcesses.put(paragraphId, executor);
} catch (ExecuteException e) {
LOGGER.error("Can not run script in paragraph {}", paragraphId, e);
final int exitValue = e.getExitValue();
Code code = Code.ERROR;
String msg = errorStream.toString();
if (exitValue == SIGTERM_CODE) {
code = Code.INCOMPLETE;
msg = msg + "Paragraph received a SIGTERM.\n";
LOGGER.info("The paragraph {} stopped executing: {}", paragraphId, msg);
}
msg += "ExitValue: " + exitValue;
result = new InterpreterResult(code, msg);
} catch (IOException e) {
LOGGER.error("Can not run script in paragraph {}", paragraphId, e);
result = new InterpreterResult(Code.ERROR, e.getMessage());
} finally {
FileUtils.deleteQuietly(scriptFile);
stopProcess(paragraphId);
}
return result;
}
use of org.apache.commons.exec.PumpStreamHandler in project zeppelin by apache.
the class TensorboardRunThread method run.
public void run() {
SubmarineUI submarineUI = submarineJob.getSubmarineUI();
boolean tryLock = lockRunning.tryLock();
try {
Properties properties = submarineJob.getProperties();
String tensorboardName = SubmarineUtils.getTensorboardName(submarineJob.getUserName());
if (true == running.get()) {
String message = String.format("tensorboard %s already running.", tensorboardName);
submarineUI.outputLog("WARN", message);
LOGGER.warn(message);
return;
}
running.set(true);
HashMap jinjaParams = SubmarineUtils.propertiesToJinjaParams(properties, submarineJob, false);
// update jobName -> tensorboardName
jinjaParams.put(SubmarineConstants.JOB_NAME, tensorboardName);
URL urlTemplate = Resources.getResource(SubmarineJob.SUBMARINE_TENSORBOARD_JINJA);
String template = Resources.toString(urlTemplate, Charsets.UTF_8);
Jinjava jinjava = new Jinjava();
String submarineCmd = jinjava.render(template, jinjaParams);
// If the first line is a newline, delete the newline
int firstLineIsNewline = submarineCmd.indexOf("\n");
if (firstLineIsNewline == 0) {
submarineCmd = submarineCmd.replaceFirst("\n", "");
}
StringBuffer sbLogs = new StringBuffer(submarineCmd);
submarineUI.outputLog("Submarine submit command", sbLogs.toString());
long timeout = Long.valueOf(properties.getProperty(SubmarineJob.TIMEOUT_PROPERTY, SubmarineJob.defaultTimeout));
CommandLine cmdLine = CommandLine.parse(SubmarineJob.shell);
cmdLine.addArgument(submarineCmd, false);
DefaultExecutor executor = new DefaultExecutor();
ExecuteWatchdog watchDog = new ExecuteWatchdog(timeout);
executor.setWatchdog(watchDog);
StringBuffer sbLogOutput = new StringBuffer();
executor.setStreamHandler(new PumpStreamHandler(new LogOutputStream() {
@Override
protected void processLine(String line, int level) {
line = line.trim();
if (!StringUtils.isEmpty(line)) {
sbLogOutput.append(line + "\n");
}
}
}));
if (Boolean.valueOf(properties.getProperty(SubmarineJob.DIRECTORY_USER_HOME))) {
executor.setWorkingDirectory(new File(System.getProperty("user.home")));
}
Map<String, String> env = new HashMap<>();
String launchMode = (String) jinjaParams.get(SubmarineConstants.INTERPRETER_LAUNCH_MODE);
if (StringUtils.equals(launchMode, "yarn")) {
// Set environment variables in the container
String javaHome, hadoopHome, hadoopConf;
javaHome = (String) jinjaParams.get(SubmarineConstants.DOCKER_JAVA_HOME);
hadoopHome = (String) jinjaParams.get(SubmarineConstants.DOCKER_HADOOP_HDFS_HOME);
hadoopConf = (String) jinjaParams.get(SubmarineConstants.SUBMARINE_HADOOP_CONF_DIR);
env.put("JAVA_HOME", javaHome);
env.put("HADOOP_HOME", hadoopHome);
env.put("HADOOP_HDFS_HOME", hadoopHome);
env.put("HADOOP_CONF_DIR", hadoopConf);
env.put("YARN_CONF_DIR", hadoopConf);
env.put("CLASSPATH", "`$HADOOP_HDFS_HOME/bin/hadoop classpath --glob`");
}
LOGGER.info("Execute EVN: {}, Command: {} ", env.toString(), submarineCmd);
AtomicBoolean cmdLineRunning = new AtomicBoolean(true);
executor.execute(cmdLine, env, new DefaultExecuteResultHandler() {
@Override
public void onProcessComplete(int exitValue) {
String message = String.format("jobName %s ProcessComplete exit value is : %d", tensorboardName, exitValue);
LOGGER.info(message);
submarineUI.outputLog("TENSORBOARD RUN COMPLETE", message);
cmdLineRunning.set(false);
}
@Override
public void onProcessFailed(ExecuteException e) {
String message = String.format("jobName %s ProcessFailed exit value is : %d, exception is : %s", tensorboardName, e.getExitValue(), e.getMessage());
LOGGER.error(message);
submarineUI.outputLog("TENSORBOARD RUN FAILED", message);
cmdLineRunning.set(false);
}
});
int loopCount = 100;
while ((loopCount-- > 0) && cmdLineRunning.get() && running.get()) {
Thread.sleep(1000);
}
if (watchDog.isWatching()) {
watchDog.destroyProcess();
Thread.sleep(1000);
}
if (watchDog.isWatching()) {
watchDog.killedProcess();
}
// Check if it has been submitted to YARN
Map<String, Object> jobState = submarineJob.getJobStateByYarn(tensorboardName);
loopCount = 50;
while ((loopCount-- > 0) && !jobState.containsKey("state") && running.get()) {
Thread.sleep(3000);
jobState = submarineJob.getJobStateByYarn(tensorboardName);
}
if (!jobState.containsKey("state")) {
String message = String.format("tensorboard %s was not submitted to YARN!", tensorboardName);
LOGGER.error(message);
submarineUI.outputLog("JOR RUN FAILED", message);
}
} catch (Exception e) {
LOGGER.error(e.getMessage(), e);
submarineUI.outputLog("Exception", e.getMessage());
} finally {
running.set(false);
lockRunning.unlock();
}
}
use of org.apache.commons.exec.PumpStreamHandler in project zeppelin by apache.
the class JobRunThread method run.
public void run() {
boolean tryLock = lockRunning.tryLock();
if (false == tryLock) {
LOGGER.warn("Can not get JobRunThread lockRunning!");
return;
}
SubmarineUI submarineUI = submarineJob.getSubmarineUI();
try {
InterpreterContext intpContext = submarineJob.getIntpContext();
String noteId = intpContext.getNoteId();
String userName = intpContext.getAuthenticationInfo().getUser();
String jobName = SubmarineUtils.getJobName(userName, noteId);
if (true == running.get()) {
String message = String.format("Job %s already running.", jobName);
submarineUI.outputLog("WARN", message);
LOGGER.warn(message);
return;
}
running.set(true);
Properties properties = submarineJob.getProperties();
HdfsClient hdfsClient = submarineJob.getHdfsClient();
File pythonWorkDir = submarineJob.getPythonWorkDir();
submarineJob.setCurrentJobState(EXECUTE_SUBMARINE);
String algorithmPath = properties.getProperty(SubmarineConstants.SUBMARINE_ALGORITHM_HDFS_PATH, "");
if (!algorithmPath.startsWith("hdfs://")) {
String message = "Algorithm file upload HDFS path, " + "Must be `hdfs://` prefix. now setting " + algorithmPath;
submarineUI.outputLog("Configuration error", message);
return;
}
List<ParagraphInfo> paragraphInfos = intpContext.getIntpEventClient().getParagraphList(userName, noteId);
String outputMsg = hdfsClient.saveParagraphToFiles(noteId, paragraphInfos, pythonWorkDir == null ? "" : pythonWorkDir.getAbsolutePath(), properties);
if (!StringUtils.isEmpty(outputMsg)) {
submarineUI.outputLog("Save algorithm file", outputMsg);
}
HashMap jinjaParams = SubmarineUtils.propertiesToJinjaParams(properties, submarineJob, true);
URL urlTemplate = Resources.getResource(SubmarineJob.SUBMARINE_JOBRUN_TF_JINJA);
String template = Resources.toString(urlTemplate, Charsets.UTF_8);
Jinjava jinjava = new Jinjava();
String submarineCmd = jinjava.render(template, jinjaParams);
// If the first line is a newline, delete the newline
int firstLineIsNewline = submarineCmd.indexOf("\n");
if (firstLineIsNewline == 0) {
submarineCmd = submarineCmd.replaceFirst("\n", "");
}
StringBuffer sbLogs = new StringBuffer(submarineCmd);
submarineUI.outputLog("Submarine submit command", sbLogs.toString());
long timeout = Long.valueOf(properties.getProperty(SubmarineJob.TIMEOUT_PROPERTY, SubmarineJob.defaultTimeout));
CommandLine cmdLine = CommandLine.parse(SubmarineJob.shell);
cmdLine.addArgument(submarineCmd, false);
DefaultExecutor executor = new DefaultExecutor();
ExecuteWatchdog watchDog = new ExecuteWatchdog(timeout);
executor.setWatchdog(watchDog);
StringBuffer sbLogOutput = new StringBuffer();
executor.setStreamHandler(new PumpStreamHandler(new LogOutputStream() {
@Override
protected void processLine(String line, int level) {
line = line.trim();
if (!StringUtils.isEmpty(line)) {
sbLogOutput.append(line + "\n");
}
}
}));
if (Boolean.valueOf(properties.getProperty(SubmarineJob.DIRECTORY_USER_HOME))) {
executor.setWorkingDirectory(new File(System.getProperty("user.home")));
}
Map<String, String> env = new HashMap<>();
String launchMode = (String) jinjaParams.get(SubmarineConstants.INTERPRETER_LAUNCH_MODE);
if (StringUtils.equals(launchMode, "yarn")) {
// Set environment variables in the submarine interpreter container run on yarn
String javaHome, hadoopHome, hadoopConf;
javaHome = (String) jinjaParams.get(SubmarineConstants.DOCKER_JAVA_HOME);
hadoopHome = (String) jinjaParams.get(SubmarineConstants.DOCKER_HADOOP_HDFS_HOME);
hadoopConf = (String) jinjaParams.get(SubmarineConstants.SUBMARINE_HADOOP_CONF_DIR);
env.put("JAVA_HOME", javaHome);
env.put("HADOOP_HOME", hadoopHome);
env.put("HADOOP_HDFS_HOME", hadoopHome);
env.put("HADOOP_CONF_DIR", hadoopConf);
env.put("YARN_CONF_DIR", hadoopConf);
env.put("CLASSPATH", "`$HADOOP_HDFS_HOME/bin/hadoop classpath --glob`");
env.put("ZEPPELIN_FORCE_STOP", "true");
}
LOGGER.info("Execute EVN: {}, Command: {} ", env.toString(), submarineCmd);
AtomicBoolean cmdLineRunning = new AtomicBoolean(true);
executor.execute(cmdLine, env, new DefaultExecuteResultHandler() {
@Override
public void onProcessComplete(int exitValue) {
String message = String.format("jobName %s ProcessComplete exit value is : %d", jobName, exitValue);
LOGGER.info(message);
submarineUI.outputLog("JOR RUN COMPLETE", message);
cmdLineRunning.set(false);
submarineJob.setCurrentJobState(EXECUTE_SUBMARINE_FINISHED);
}
@Override
public void onProcessFailed(ExecuteException e) {
String message = String.format("jobName %s ProcessFailed exit value is : %d, exception is : %s", jobName, e.getExitValue(), e.getMessage());
LOGGER.error(message);
submarineUI.outputLog("JOR RUN FAILED", message);
cmdLineRunning.set(false);
submarineJob.setCurrentJobState(EXECUTE_SUBMARINE_ERROR);
}
});
int loopCount = 100;
while ((loopCount-- > 0) && cmdLineRunning.get() && running.get()) {
Thread.sleep(1000);
}
if (watchDog.isWatching()) {
watchDog.destroyProcess();
Thread.sleep(1000);
}
if (watchDog.isWatching()) {
watchDog.killedProcess();
}
// Check if it has been submitted to YARN
Map<String, Object> jobState = submarineJob.getJobStateByYarn(jobName);
loopCount = 50;
while ((loopCount-- > 0) && !jobState.containsKey("state") && running.get()) {
Thread.sleep(3000);
jobState = submarineJob.getJobStateByYarn(jobName);
}
if (!jobState.containsKey("state")) {
String message = String.format("JOB %s was not submitted to YARN!", jobName);
LOGGER.error(message);
submarineUI.outputLog("JOR RUN FAILED", message);
submarineJob.setCurrentJobState(EXECUTE_SUBMARINE_ERROR);
}
} catch (Exception e) {
LOGGER.error(e.getMessage(), e);
submarineJob.setCurrentJobState(EXECUTE_SUBMARINE_ERROR);
submarineUI.outputLog("Exception", e.getMessage());
} finally {
running.set(false);
lockRunning.unlock();
}
}
use of org.apache.commons.exec.PumpStreamHandler in project zeppelin by apache.
the class AbstractTestRestApi method ps.
public static void ps() {
DefaultExecutor executor = new DefaultExecutor();
executor.setStreamHandler(new PumpStreamHandler(System.out, System.err));
CommandLine cmd = CommandLine.parse("ps");
cmd.addArgument("aux", false);
try {
executor.execute(cmd);
} catch (IOException e) {
LOG.error(e.getMessage(), e);
}
}
use of org.apache.commons.exec.PumpStreamHandler in project zeppelin by apache.
the class ShellInterpreter method internalInterpret.
@Override
public InterpreterResult internalInterpret(String cmd, InterpreterContext context) {
LOGGER.debug("Run shell command '{}'", cmd);
CommandLine cmdLine = CommandLine.parse(shell);
// they need to be delimited by '&&' instead
if (isWindows) {
String[] lines = StringUtils.split(cmd, "\n");
cmd = StringUtils.join(lines, " && ");
}
cmdLine.addArgument(cmd, false);
try {
contextMap.put(context.getParagraphId(), context);
DefaultExecutor executor = new DefaultExecutor();
executor.setStreamHandler(new PumpStreamHandler(context.out, context.out));
executor.setWatchdog(new ExecuteWatchdog(Long.MAX_VALUE));
executorMap.put(context.getParagraphId(), executor);
if (Boolean.valueOf(getProperty(DIRECTORY_USER_HOME))) {
executor.setWorkingDirectory(new File(System.getProperty("user.home")));
}
int exitVal = executor.execute(cmdLine);
LOGGER.info("Paragraph {} return with exit value: {}", context.getParagraphId(), exitVal);
if (exitVal == 0) {
return new InterpreterResult(Code.SUCCESS);
} else {
return new InterpreterResult(Code.ERROR);
}
} catch (ExecuteException e) {
int exitValue = e.getExitValue();
LOGGER.error("Can not run command: " + cmd, e);
Code code = Code.ERROR;
StringBuilder messageBuilder = new StringBuilder();
if (exitValue == 143) {
code = Code.INCOMPLETE;
messageBuilder.append("Paragraph received a SIGTERM\n");
LOGGER.info("The paragraph {} stopped executing: {}", context.getParagraphId(), messageBuilder.toString());
}
messageBuilder.append("ExitValue: " + exitValue);
return new InterpreterResult(code, messageBuilder.toString());
} catch (IOException e) {
LOGGER.error("Can not run command: " + cmd, e);
return new InterpreterResult(Code.ERROR, e.getMessage());
} finally {
executorMap.remove(context.getParagraphId());
contextMap.remove(context.getParagraphId());
}
}
Aggregations