use of org.apache.commons.exec.DefaultExecuteResultHandler in project alliance by codice.
the class MpegTsUdpClient method executeFFmpeg.
private static DefaultExecuteResultHandler executeFFmpeg(final CommandLine command, final int timeoutSeconds, final PumpStreamHandler streamHandler) throws IOException {
final ExecuteWatchdog watchdog = new ExecuteWatchdog(timeoutSeconds * 1000);
final Executor executor = new DefaultExecutor();
executor.setWatchdog(watchdog);
if (streamHandler != null) {
executor.setStreamHandler(streamHandler);
}
final DefaultExecuteResultHandler resultHandler = new DefaultExecuteResultHandler();
executor.execute(command, resultHandler);
return resultHandler;
}
use of org.apache.commons.exec.DefaultExecuteResultHandler in project ddf by codice.
the class VideoThumbnailPlugin method getVideoDuration.
private Duration getVideoDuration(final String videoFilePath) throws IOException, InterruptedException {
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
final PumpStreamHandler streamHandler = new PumpStreamHandler(outputStream);
final CommandLine command = getFFmpegInfoCommand(videoFilePath);
final DefaultExecuteResultHandler resultHandler = executeFFmpeg(command, 15, streamHandler);
resultHandler.waitFor();
return parseVideoDuration(outputStream.toString(StandardCharsets.UTF_8.name()));
}
use of org.apache.commons.exec.DefaultExecuteResultHandler in project Robot by fo0.
the class Commander method execute.
public String execute(boolean wait, boolean shell, String homedir, boolean quoting, List<String> cmds) {
if (cmds == null || cmds.isEmpty()) {
Logger.info("stopped cmd command is empty");
return null;
}
CommandLine cmdLine = null;
if (shell) {
switch(OSCheck.getOperatingSystemType()) {
case Windows:
cmdLine = new CommandLine("cmd");
if (shell) {
cmdLine.addArgument("/c");
}
break;
case Linux:
cmdLine = new CommandLine("/bin/bash");
if (shell) {
cmdLine.addArgument("-c");
}
break;
}
cmdLine.addArguments(cmds.stream().toArray(String[]::new), quoting);
} else {
cmdLine = new CommandLine(cmds.get(0));
cmdLine.addArguments(cmds.stream().skip(1).toArray(String[]::new), quoting);
}
Logger.debug("HomeDir: '" + Paths.get(homedir).toAbsolutePath() + "' => " + cmdLine.getExecutable() + ", " + StringUtils.join(cmdLine.getArguments(), ","));
try {
Executor executor = createDefaultExecutor();
executor.setStreamHandler(new PumpStreamHandler(new OutputStream() {
@Override
public void write(int b) throws IOException {
try {
String str = String.valueOf((char) b);
if (listener != null)
listener.event(str);
buffer.append(str);
} catch (Exception e) {
Logger.debug(e.getMessage());
}
}
}, new OutputStream() {
@Override
public void write(int b) throws IOException {
error = true;
try {
String str = String.valueOf((char) b);
if (listener != null)
listener.event(str);
buffer.append(str);
} catch (Exception e) {
Logger.debug(e.getMessage());
}
}
}));
// configure timeout
// executor.setWatchdog(new ExecuteWatchdog(-1));
executor.setWorkingDirectory(new File(homedir));
if (wait) {
executor.execute(cmdLine);
} else {
DefaultExecuteResultHandler resultHandler = new DefaultExecuteResultHandler();
executor.execute(cmdLine, resultHandler);
}
} catch (Exception e) {
error = true;
Logger.error("failed commander in Cmd: " + cmdLine + " | " + e);
Logger.debug(e.getMessage());
}
return buffer.toString();
}
use of org.apache.commons.exec.DefaultExecuteResultHandler in project zeppelin by apache.
the class TensorboardRunThread method run.
public void run() {
SubmarineUI submarineUI = submarineJob.getSubmarineUI();
boolean tryLock = lockRunning.tryLock();
try {
Properties properties = submarineJob.getProperties();
String tensorboardName = SubmarineUtils.getTensorboardName(submarineJob.getUserName());
if (true == running.get()) {
String message = String.format("tensorboard %s already running.", tensorboardName);
submarineUI.outputLog("WARN", message);
LOGGER.warn(message);
return;
}
running.set(true);
HashMap jinjaParams = SubmarineUtils.propertiesToJinjaParams(properties, submarineJob, false);
// update jobName -> tensorboardName
jinjaParams.put(SubmarineConstants.JOB_NAME, tensorboardName);
URL urlTemplate = Resources.getResource(SubmarineJob.SUBMARINE_TENSORBOARD_JINJA);
String template = Resources.toString(urlTemplate, Charsets.UTF_8);
Jinjava jinjava = new Jinjava();
String submarineCmd = jinjava.render(template, jinjaParams);
// If the first line is a newline, delete the newline
int firstLineIsNewline = submarineCmd.indexOf("\n");
if (firstLineIsNewline == 0) {
submarineCmd = submarineCmd.replaceFirst("\n", "");
}
StringBuffer sbLogs = new StringBuffer(submarineCmd);
submarineUI.outputLog("Submarine submit command", sbLogs.toString());
long timeout = Long.valueOf(properties.getProperty(SubmarineJob.TIMEOUT_PROPERTY, SubmarineJob.defaultTimeout));
CommandLine cmdLine = CommandLine.parse(SubmarineJob.shell);
cmdLine.addArgument(submarineCmd, false);
DefaultExecutor executor = new DefaultExecutor();
ExecuteWatchdog watchDog = new ExecuteWatchdog(timeout);
executor.setWatchdog(watchDog);
StringBuffer sbLogOutput = new StringBuffer();
executor.setStreamHandler(new PumpStreamHandler(new LogOutputStream() {
@Override
protected void processLine(String line, int level) {
line = line.trim();
if (!StringUtils.isEmpty(line)) {
sbLogOutput.append(line + "\n");
}
}
}));
if (Boolean.valueOf(properties.getProperty(SubmarineJob.DIRECTORY_USER_HOME))) {
executor.setWorkingDirectory(new File(System.getProperty("user.home")));
}
Map<String, String> env = new HashMap<>();
String launchMode = (String) jinjaParams.get(SubmarineConstants.INTERPRETER_LAUNCH_MODE);
if (StringUtils.equals(launchMode, "yarn")) {
// Set environment variables in the container
String javaHome, hadoopHome, hadoopConf;
javaHome = (String) jinjaParams.get(SubmarineConstants.DOCKER_JAVA_HOME);
hadoopHome = (String) jinjaParams.get(SubmarineConstants.DOCKER_HADOOP_HDFS_HOME);
hadoopConf = (String) jinjaParams.get(SubmarineConstants.SUBMARINE_HADOOP_CONF_DIR);
env.put("JAVA_HOME", javaHome);
env.put("HADOOP_HOME", hadoopHome);
env.put("HADOOP_HDFS_HOME", hadoopHome);
env.put("HADOOP_CONF_DIR", hadoopConf);
env.put("YARN_CONF_DIR", hadoopConf);
env.put("CLASSPATH", "`$HADOOP_HDFS_HOME/bin/hadoop classpath --glob`");
}
LOGGER.info("Execute EVN: {}, Command: {} ", env.toString(), submarineCmd);
AtomicBoolean cmdLineRunning = new AtomicBoolean(true);
executor.execute(cmdLine, env, new DefaultExecuteResultHandler() {
@Override
public void onProcessComplete(int exitValue) {
String message = String.format("jobName %s ProcessComplete exit value is : %d", tensorboardName, exitValue);
LOGGER.info(message);
submarineUI.outputLog("TENSORBOARD RUN COMPLETE", message);
cmdLineRunning.set(false);
}
@Override
public void onProcessFailed(ExecuteException e) {
String message = String.format("jobName %s ProcessFailed exit value is : %d, exception is : %s", tensorboardName, e.getExitValue(), e.getMessage());
LOGGER.error(message);
submarineUI.outputLog("TENSORBOARD RUN FAILED", message);
cmdLineRunning.set(false);
}
});
int loopCount = 100;
while ((loopCount-- > 0) && cmdLineRunning.get() && running.get()) {
Thread.sleep(1000);
}
if (watchDog.isWatching()) {
watchDog.destroyProcess();
Thread.sleep(1000);
}
if (watchDog.isWatching()) {
watchDog.killedProcess();
}
// Check if it has been submitted to YARN
Map<String, Object> jobState = submarineJob.getJobStateByYarn(tensorboardName);
loopCount = 50;
while ((loopCount-- > 0) && !jobState.containsKey("state") && running.get()) {
Thread.sleep(3000);
jobState = submarineJob.getJobStateByYarn(tensorboardName);
}
if (!jobState.containsKey("state")) {
String message = String.format("tensorboard %s was not submitted to YARN!", tensorboardName);
LOGGER.error(message);
submarineUI.outputLog("JOR RUN FAILED", message);
}
} catch (Exception e) {
LOGGER.error(e.getMessage(), e);
submarineUI.outputLog("Exception", e.getMessage());
} finally {
running.set(false);
lockRunning.unlock();
}
}
use of org.apache.commons.exec.DefaultExecuteResultHandler in project zeppelin by apache.
the class JobRunThread method run.
public void run() {
boolean tryLock = lockRunning.tryLock();
if (false == tryLock) {
LOGGER.warn("Can not get JobRunThread lockRunning!");
return;
}
SubmarineUI submarineUI = submarineJob.getSubmarineUI();
try {
InterpreterContext intpContext = submarineJob.getIntpContext();
String noteId = intpContext.getNoteId();
String userName = intpContext.getAuthenticationInfo().getUser();
String jobName = SubmarineUtils.getJobName(userName, noteId);
if (true == running.get()) {
String message = String.format("Job %s already running.", jobName);
submarineUI.outputLog("WARN", message);
LOGGER.warn(message);
return;
}
running.set(true);
Properties properties = submarineJob.getProperties();
HdfsClient hdfsClient = submarineJob.getHdfsClient();
File pythonWorkDir = submarineJob.getPythonWorkDir();
submarineJob.setCurrentJobState(EXECUTE_SUBMARINE);
String algorithmPath = properties.getProperty(SubmarineConstants.SUBMARINE_ALGORITHM_HDFS_PATH, "");
if (!algorithmPath.startsWith("hdfs://")) {
String message = "Algorithm file upload HDFS path, " + "Must be `hdfs://` prefix. now setting " + algorithmPath;
submarineUI.outputLog("Configuration error", message);
return;
}
List<ParagraphInfo> paragraphInfos = intpContext.getIntpEventClient().getParagraphList(userName, noteId);
String outputMsg = hdfsClient.saveParagraphToFiles(noteId, paragraphInfos, pythonWorkDir == null ? "" : pythonWorkDir.getAbsolutePath(), properties);
if (!StringUtils.isEmpty(outputMsg)) {
submarineUI.outputLog("Save algorithm file", outputMsg);
}
HashMap jinjaParams = SubmarineUtils.propertiesToJinjaParams(properties, submarineJob, true);
URL urlTemplate = Resources.getResource(SubmarineJob.SUBMARINE_JOBRUN_TF_JINJA);
String template = Resources.toString(urlTemplate, Charsets.UTF_8);
Jinjava jinjava = new Jinjava();
String submarineCmd = jinjava.render(template, jinjaParams);
// If the first line is a newline, delete the newline
int firstLineIsNewline = submarineCmd.indexOf("\n");
if (firstLineIsNewline == 0) {
submarineCmd = submarineCmd.replaceFirst("\n", "");
}
StringBuffer sbLogs = new StringBuffer(submarineCmd);
submarineUI.outputLog("Submarine submit command", sbLogs.toString());
long timeout = Long.valueOf(properties.getProperty(SubmarineJob.TIMEOUT_PROPERTY, SubmarineJob.defaultTimeout));
CommandLine cmdLine = CommandLine.parse(SubmarineJob.shell);
cmdLine.addArgument(submarineCmd, false);
DefaultExecutor executor = new DefaultExecutor();
ExecuteWatchdog watchDog = new ExecuteWatchdog(timeout);
executor.setWatchdog(watchDog);
StringBuffer sbLogOutput = new StringBuffer();
executor.setStreamHandler(new PumpStreamHandler(new LogOutputStream() {
@Override
protected void processLine(String line, int level) {
line = line.trim();
if (!StringUtils.isEmpty(line)) {
sbLogOutput.append(line + "\n");
}
}
}));
if (Boolean.valueOf(properties.getProperty(SubmarineJob.DIRECTORY_USER_HOME))) {
executor.setWorkingDirectory(new File(System.getProperty("user.home")));
}
Map<String, String> env = new HashMap<>();
String launchMode = (String) jinjaParams.get(SubmarineConstants.INTERPRETER_LAUNCH_MODE);
if (StringUtils.equals(launchMode, "yarn")) {
// Set environment variables in the submarine interpreter container run on yarn
String javaHome, hadoopHome, hadoopConf;
javaHome = (String) jinjaParams.get(SubmarineConstants.DOCKER_JAVA_HOME);
hadoopHome = (String) jinjaParams.get(SubmarineConstants.DOCKER_HADOOP_HDFS_HOME);
hadoopConf = (String) jinjaParams.get(SubmarineConstants.SUBMARINE_HADOOP_CONF_DIR);
env.put("JAVA_HOME", javaHome);
env.put("HADOOP_HOME", hadoopHome);
env.put("HADOOP_HDFS_HOME", hadoopHome);
env.put("HADOOP_CONF_DIR", hadoopConf);
env.put("YARN_CONF_DIR", hadoopConf);
env.put("CLASSPATH", "`$HADOOP_HDFS_HOME/bin/hadoop classpath --glob`");
env.put("ZEPPELIN_FORCE_STOP", "true");
}
LOGGER.info("Execute EVN: {}, Command: {} ", env.toString(), submarineCmd);
AtomicBoolean cmdLineRunning = new AtomicBoolean(true);
executor.execute(cmdLine, env, new DefaultExecuteResultHandler() {
@Override
public void onProcessComplete(int exitValue) {
String message = String.format("jobName %s ProcessComplete exit value is : %d", jobName, exitValue);
LOGGER.info(message);
submarineUI.outputLog("JOR RUN COMPLETE", message);
cmdLineRunning.set(false);
submarineJob.setCurrentJobState(EXECUTE_SUBMARINE_FINISHED);
}
@Override
public void onProcessFailed(ExecuteException e) {
String message = String.format("jobName %s ProcessFailed exit value is : %d, exception is : %s", jobName, e.getExitValue(), e.getMessage());
LOGGER.error(message);
submarineUI.outputLog("JOR RUN FAILED", message);
cmdLineRunning.set(false);
submarineJob.setCurrentJobState(EXECUTE_SUBMARINE_ERROR);
}
});
int loopCount = 100;
while ((loopCount-- > 0) && cmdLineRunning.get() && running.get()) {
Thread.sleep(1000);
}
if (watchDog.isWatching()) {
watchDog.destroyProcess();
Thread.sleep(1000);
}
if (watchDog.isWatching()) {
watchDog.killedProcess();
}
// Check if it has been submitted to YARN
Map<String, Object> jobState = submarineJob.getJobStateByYarn(jobName);
loopCount = 50;
while ((loopCount-- > 0) && !jobState.containsKey("state") && running.get()) {
Thread.sleep(3000);
jobState = submarineJob.getJobStateByYarn(jobName);
}
if (!jobState.containsKey("state")) {
String message = String.format("JOB %s was not submitted to YARN!", jobName);
LOGGER.error(message);
submarineUI.outputLog("JOR RUN FAILED", message);
submarineJob.setCurrentJobState(EXECUTE_SUBMARINE_ERROR);
}
} catch (Exception e) {
LOGGER.error(e.getMessage(), e);
submarineJob.setCurrentJobState(EXECUTE_SUBMARINE_ERROR);
submarineUI.outputLog("Exception", e.getMessage());
} finally {
running.set(false);
lockRunning.unlock();
}
}
Aggregations