Search in sources :

Example 11 with DefaultExecuteResultHandler

use of org.apache.commons.exec.DefaultExecuteResultHandler in project alliance by codice.

the class MpegTsUdpClient method executeFFmpeg.

private static DefaultExecuteResultHandler executeFFmpeg(final CommandLine command, final int timeoutSeconds, final PumpStreamHandler streamHandler) throws IOException {
    final ExecuteWatchdog watchdog = new ExecuteWatchdog(timeoutSeconds * 1000);
    final Executor executor = new DefaultExecutor();
    executor.setWatchdog(watchdog);
    if (streamHandler != null) {
        executor.setStreamHandler(streamHandler);
    }
    final DefaultExecuteResultHandler resultHandler = new DefaultExecuteResultHandler();
    executor.execute(command, resultHandler);
    return resultHandler;
}
Also used : Executor(org.apache.commons.exec.Executor) DefaultExecutor(org.apache.commons.exec.DefaultExecutor) DefaultExecuteResultHandler(org.apache.commons.exec.DefaultExecuteResultHandler) DefaultExecutor(org.apache.commons.exec.DefaultExecutor) ExecuteWatchdog(org.apache.commons.exec.ExecuteWatchdog)

Example 12 with DefaultExecuteResultHandler

use of org.apache.commons.exec.DefaultExecuteResultHandler in project ddf by codice.

the class VideoThumbnailPlugin method getVideoDuration.

private Duration getVideoDuration(final String videoFilePath) throws IOException, InterruptedException {
    final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
    final PumpStreamHandler streamHandler = new PumpStreamHandler(outputStream);
    final CommandLine command = getFFmpegInfoCommand(videoFilePath);
    final DefaultExecuteResultHandler resultHandler = executeFFmpeg(command, 15, streamHandler);
    resultHandler.waitFor();
    return parseVideoDuration(outputStream.toString(StandardCharsets.UTF_8.name()));
}
Also used : CommandLine(org.apache.commons.exec.CommandLine) PumpStreamHandler(org.apache.commons.exec.PumpStreamHandler) DefaultExecuteResultHandler(org.apache.commons.exec.DefaultExecuteResultHandler) ByteArrayOutputStream(java.io.ByteArrayOutputStream)

Example 13 with DefaultExecuteResultHandler

use of org.apache.commons.exec.DefaultExecuteResultHandler in project Robot by fo0.

the class Commander method execute.

public String execute(boolean wait, boolean shell, String homedir, boolean quoting, List<String> cmds) {
    if (cmds == null || cmds.isEmpty()) {
        Logger.info("stopped cmd command is empty");
        return null;
    }
    CommandLine cmdLine = null;
    if (shell) {
        switch(OSCheck.getOperatingSystemType()) {
            case Windows:
                cmdLine = new CommandLine("cmd");
                if (shell) {
                    cmdLine.addArgument("/c");
                }
                break;
            case Linux:
                cmdLine = new CommandLine("/bin/bash");
                if (shell) {
                    cmdLine.addArgument("-c");
                }
                break;
        }
        cmdLine.addArguments(cmds.stream().toArray(String[]::new), quoting);
    } else {
        cmdLine = new CommandLine(cmds.get(0));
        cmdLine.addArguments(cmds.stream().skip(1).toArray(String[]::new), quoting);
    }
    Logger.debug("HomeDir: '" + Paths.get(homedir).toAbsolutePath() + "' => " + cmdLine.getExecutable() + ", " + StringUtils.join(cmdLine.getArguments(), ","));
    try {
        Executor executor = createDefaultExecutor();
        executor.setStreamHandler(new PumpStreamHandler(new OutputStream() {

            @Override
            public void write(int b) throws IOException {
                try {
                    String str = String.valueOf((char) b);
                    if (listener != null)
                        listener.event(str);
                    buffer.append(str);
                } catch (Exception e) {
                    Logger.debug(e.getMessage());
                }
            }
        }, new OutputStream() {

            @Override
            public void write(int b) throws IOException {
                error = true;
                try {
                    String str = String.valueOf((char) b);
                    if (listener != null)
                        listener.event(str);
                    buffer.append(str);
                } catch (Exception e) {
                    Logger.debug(e.getMessage());
                }
            }
        }));
        // configure timeout
        // executor.setWatchdog(new ExecuteWatchdog(-1));
        executor.setWorkingDirectory(new File(homedir));
        if (wait) {
            executor.execute(cmdLine);
        } else {
            DefaultExecuteResultHandler resultHandler = new DefaultExecuteResultHandler();
            executor.execute(cmdLine, resultHandler);
        }
    } catch (Exception e) {
        error = true;
        Logger.error("failed commander in Cmd: " + cmdLine + " | " + e);
        Logger.debug(e.getMessage());
    }
    return buffer.toString();
}
Also used : CommandLine(org.apache.commons.exec.CommandLine) Executor(org.apache.commons.exec.Executor) DefaultExecutor(org.apache.commons.exec.DefaultExecutor) PumpStreamHandler(org.apache.commons.exec.PumpStreamHandler) DefaultExecuteResultHandler(org.apache.commons.exec.DefaultExecuteResultHandler) OutputStream(java.io.OutputStream) File(java.io.File) IOException(java.io.IOException)

Example 14 with DefaultExecuteResultHandler

use of org.apache.commons.exec.DefaultExecuteResultHandler in project zeppelin by apache.

the class TensorboardRunThread method run.

public void run() {
    SubmarineUI submarineUI = submarineJob.getSubmarineUI();
    boolean tryLock = lockRunning.tryLock();
    try {
        Properties properties = submarineJob.getProperties();
        String tensorboardName = SubmarineUtils.getTensorboardName(submarineJob.getUserName());
        if (true == running.get()) {
            String message = String.format("tensorboard %s already running.", tensorboardName);
            submarineUI.outputLog("WARN", message);
            LOGGER.warn(message);
            return;
        }
        running.set(true);
        HashMap jinjaParams = SubmarineUtils.propertiesToJinjaParams(properties, submarineJob, false);
        // update jobName -> tensorboardName
        jinjaParams.put(SubmarineConstants.JOB_NAME, tensorboardName);
        URL urlTemplate = Resources.getResource(SubmarineJob.SUBMARINE_TENSORBOARD_JINJA);
        String template = Resources.toString(urlTemplate, Charsets.UTF_8);
        Jinjava jinjava = new Jinjava();
        String submarineCmd = jinjava.render(template, jinjaParams);
        // If the first line is a newline, delete the newline
        int firstLineIsNewline = submarineCmd.indexOf("\n");
        if (firstLineIsNewline == 0) {
            submarineCmd = submarineCmd.replaceFirst("\n", "");
        }
        StringBuffer sbLogs = new StringBuffer(submarineCmd);
        submarineUI.outputLog("Submarine submit command", sbLogs.toString());
        long timeout = Long.valueOf(properties.getProperty(SubmarineJob.TIMEOUT_PROPERTY, SubmarineJob.defaultTimeout));
        CommandLine cmdLine = CommandLine.parse(SubmarineJob.shell);
        cmdLine.addArgument(submarineCmd, false);
        DefaultExecutor executor = new DefaultExecutor();
        ExecuteWatchdog watchDog = new ExecuteWatchdog(timeout);
        executor.setWatchdog(watchDog);
        StringBuffer sbLogOutput = new StringBuffer();
        executor.setStreamHandler(new PumpStreamHandler(new LogOutputStream() {

            @Override
            protected void processLine(String line, int level) {
                line = line.trim();
                if (!StringUtils.isEmpty(line)) {
                    sbLogOutput.append(line + "\n");
                }
            }
        }));
        if (Boolean.valueOf(properties.getProperty(SubmarineJob.DIRECTORY_USER_HOME))) {
            executor.setWorkingDirectory(new File(System.getProperty("user.home")));
        }
        Map<String, String> env = new HashMap<>();
        String launchMode = (String) jinjaParams.get(SubmarineConstants.INTERPRETER_LAUNCH_MODE);
        if (StringUtils.equals(launchMode, "yarn")) {
            // Set environment variables in the container
            String javaHome, hadoopHome, hadoopConf;
            javaHome = (String) jinjaParams.get(SubmarineConstants.DOCKER_JAVA_HOME);
            hadoopHome = (String) jinjaParams.get(SubmarineConstants.DOCKER_HADOOP_HDFS_HOME);
            hadoopConf = (String) jinjaParams.get(SubmarineConstants.SUBMARINE_HADOOP_CONF_DIR);
            env.put("JAVA_HOME", javaHome);
            env.put("HADOOP_HOME", hadoopHome);
            env.put("HADOOP_HDFS_HOME", hadoopHome);
            env.put("HADOOP_CONF_DIR", hadoopConf);
            env.put("YARN_CONF_DIR", hadoopConf);
            env.put("CLASSPATH", "`$HADOOP_HDFS_HOME/bin/hadoop classpath --glob`");
        }
        LOGGER.info("Execute EVN: {}, Command: {} ", env.toString(), submarineCmd);
        AtomicBoolean cmdLineRunning = new AtomicBoolean(true);
        executor.execute(cmdLine, env, new DefaultExecuteResultHandler() {

            @Override
            public void onProcessComplete(int exitValue) {
                String message = String.format("jobName %s ProcessComplete exit value is : %d", tensorboardName, exitValue);
                LOGGER.info(message);
                submarineUI.outputLog("TENSORBOARD RUN COMPLETE", message);
                cmdLineRunning.set(false);
            }

            @Override
            public void onProcessFailed(ExecuteException e) {
                String message = String.format("jobName %s ProcessFailed exit value is : %d, exception is : %s", tensorboardName, e.getExitValue(), e.getMessage());
                LOGGER.error(message);
                submarineUI.outputLog("TENSORBOARD RUN FAILED", message);
                cmdLineRunning.set(false);
            }
        });
        int loopCount = 100;
        while ((loopCount-- > 0) && cmdLineRunning.get() && running.get()) {
            Thread.sleep(1000);
        }
        if (watchDog.isWatching()) {
            watchDog.destroyProcess();
            Thread.sleep(1000);
        }
        if (watchDog.isWatching()) {
            watchDog.killedProcess();
        }
        // Check if it has been submitted to YARN
        Map<String, Object> jobState = submarineJob.getJobStateByYarn(tensorboardName);
        loopCount = 50;
        while ((loopCount-- > 0) && !jobState.containsKey("state") && running.get()) {
            Thread.sleep(3000);
            jobState = submarineJob.getJobStateByYarn(tensorboardName);
        }
        if (!jobState.containsKey("state")) {
            String message = String.format("tensorboard %s was not submitted to YARN!", tensorboardName);
            LOGGER.error(message);
            submarineUI.outputLog("JOR RUN FAILED", message);
        }
    } catch (Exception e) {
        LOGGER.error(e.getMessage(), e);
        submarineUI.outputLog("Exception", e.getMessage());
    } finally {
        running.set(false);
        lockRunning.unlock();
    }
}
Also used : DefaultExecuteResultHandler(org.apache.commons.exec.DefaultExecuteResultHandler) HashMap(java.util.HashMap) DefaultExecutor(org.apache.commons.exec.DefaultExecutor) ExecuteWatchdog(org.apache.commons.exec.ExecuteWatchdog) Jinjava(com.hubspot.jinjava.Jinjava) Properties(java.util.Properties) URL(java.net.URL) LogOutputStream(org.apache.commons.exec.LogOutputStream) ExecuteException(org.apache.commons.exec.ExecuteException) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) CommandLine(org.apache.commons.exec.CommandLine) PumpStreamHandler(org.apache.commons.exec.PumpStreamHandler) ExecuteException(org.apache.commons.exec.ExecuteException) File(java.io.File) SubmarineUI(org.apache.zeppelin.submarine.commons.SubmarineUI)

Example 15 with DefaultExecuteResultHandler

use of org.apache.commons.exec.DefaultExecuteResultHandler in project zeppelin by apache.

the class JobRunThread method run.

public void run() {
    boolean tryLock = lockRunning.tryLock();
    if (false == tryLock) {
        LOGGER.warn("Can not get JobRunThread lockRunning!");
        return;
    }
    SubmarineUI submarineUI = submarineJob.getSubmarineUI();
    try {
        InterpreterContext intpContext = submarineJob.getIntpContext();
        String noteId = intpContext.getNoteId();
        String userName = intpContext.getAuthenticationInfo().getUser();
        String jobName = SubmarineUtils.getJobName(userName, noteId);
        if (true == running.get()) {
            String message = String.format("Job %s already running.", jobName);
            submarineUI.outputLog("WARN", message);
            LOGGER.warn(message);
            return;
        }
        running.set(true);
        Properties properties = submarineJob.getProperties();
        HdfsClient hdfsClient = submarineJob.getHdfsClient();
        File pythonWorkDir = submarineJob.getPythonWorkDir();
        submarineJob.setCurrentJobState(EXECUTE_SUBMARINE);
        String algorithmPath = properties.getProperty(SubmarineConstants.SUBMARINE_ALGORITHM_HDFS_PATH, "");
        if (!algorithmPath.startsWith("hdfs://")) {
            String message = "Algorithm file upload HDFS path, " + "Must be `hdfs://` prefix. now setting " + algorithmPath;
            submarineUI.outputLog("Configuration error", message);
            return;
        }
        List<ParagraphInfo> paragraphInfos = intpContext.getIntpEventClient().getParagraphList(userName, noteId);
        String outputMsg = hdfsClient.saveParagraphToFiles(noteId, paragraphInfos, pythonWorkDir == null ? "" : pythonWorkDir.getAbsolutePath(), properties);
        if (!StringUtils.isEmpty(outputMsg)) {
            submarineUI.outputLog("Save algorithm file", outputMsg);
        }
        HashMap jinjaParams = SubmarineUtils.propertiesToJinjaParams(properties, submarineJob, true);
        URL urlTemplate = Resources.getResource(SubmarineJob.SUBMARINE_JOBRUN_TF_JINJA);
        String template = Resources.toString(urlTemplate, Charsets.UTF_8);
        Jinjava jinjava = new Jinjava();
        String submarineCmd = jinjava.render(template, jinjaParams);
        // If the first line is a newline, delete the newline
        int firstLineIsNewline = submarineCmd.indexOf("\n");
        if (firstLineIsNewline == 0) {
            submarineCmd = submarineCmd.replaceFirst("\n", "");
        }
        StringBuffer sbLogs = new StringBuffer(submarineCmd);
        submarineUI.outputLog("Submarine submit command", sbLogs.toString());
        long timeout = Long.valueOf(properties.getProperty(SubmarineJob.TIMEOUT_PROPERTY, SubmarineJob.defaultTimeout));
        CommandLine cmdLine = CommandLine.parse(SubmarineJob.shell);
        cmdLine.addArgument(submarineCmd, false);
        DefaultExecutor executor = new DefaultExecutor();
        ExecuteWatchdog watchDog = new ExecuteWatchdog(timeout);
        executor.setWatchdog(watchDog);
        StringBuffer sbLogOutput = new StringBuffer();
        executor.setStreamHandler(new PumpStreamHandler(new LogOutputStream() {

            @Override
            protected void processLine(String line, int level) {
                line = line.trim();
                if (!StringUtils.isEmpty(line)) {
                    sbLogOutput.append(line + "\n");
                }
            }
        }));
        if (Boolean.valueOf(properties.getProperty(SubmarineJob.DIRECTORY_USER_HOME))) {
            executor.setWorkingDirectory(new File(System.getProperty("user.home")));
        }
        Map<String, String> env = new HashMap<>();
        String launchMode = (String) jinjaParams.get(SubmarineConstants.INTERPRETER_LAUNCH_MODE);
        if (StringUtils.equals(launchMode, "yarn")) {
            // Set environment variables in the submarine interpreter container run on yarn
            String javaHome, hadoopHome, hadoopConf;
            javaHome = (String) jinjaParams.get(SubmarineConstants.DOCKER_JAVA_HOME);
            hadoopHome = (String) jinjaParams.get(SubmarineConstants.DOCKER_HADOOP_HDFS_HOME);
            hadoopConf = (String) jinjaParams.get(SubmarineConstants.SUBMARINE_HADOOP_CONF_DIR);
            env.put("JAVA_HOME", javaHome);
            env.put("HADOOP_HOME", hadoopHome);
            env.put("HADOOP_HDFS_HOME", hadoopHome);
            env.put("HADOOP_CONF_DIR", hadoopConf);
            env.put("YARN_CONF_DIR", hadoopConf);
            env.put("CLASSPATH", "`$HADOOP_HDFS_HOME/bin/hadoop classpath --glob`");
            env.put("ZEPPELIN_FORCE_STOP", "true");
        }
        LOGGER.info("Execute EVN: {}, Command: {} ", env.toString(), submarineCmd);
        AtomicBoolean cmdLineRunning = new AtomicBoolean(true);
        executor.execute(cmdLine, env, new DefaultExecuteResultHandler() {

            @Override
            public void onProcessComplete(int exitValue) {
                String message = String.format("jobName %s ProcessComplete exit value is : %d", jobName, exitValue);
                LOGGER.info(message);
                submarineUI.outputLog("JOR RUN COMPLETE", message);
                cmdLineRunning.set(false);
                submarineJob.setCurrentJobState(EXECUTE_SUBMARINE_FINISHED);
            }

            @Override
            public void onProcessFailed(ExecuteException e) {
                String message = String.format("jobName %s ProcessFailed exit value is : %d, exception is : %s", jobName, e.getExitValue(), e.getMessage());
                LOGGER.error(message);
                submarineUI.outputLog("JOR RUN FAILED", message);
                cmdLineRunning.set(false);
                submarineJob.setCurrentJobState(EXECUTE_SUBMARINE_ERROR);
            }
        });
        int loopCount = 100;
        while ((loopCount-- > 0) && cmdLineRunning.get() && running.get()) {
            Thread.sleep(1000);
        }
        if (watchDog.isWatching()) {
            watchDog.destroyProcess();
            Thread.sleep(1000);
        }
        if (watchDog.isWatching()) {
            watchDog.killedProcess();
        }
        // Check if it has been submitted to YARN
        Map<String, Object> jobState = submarineJob.getJobStateByYarn(jobName);
        loopCount = 50;
        while ((loopCount-- > 0) && !jobState.containsKey("state") && running.get()) {
            Thread.sleep(3000);
            jobState = submarineJob.getJobStateByYarn(jobName);
        }
        if (!jobState.containsKey("state")) {
            String message = String.format("JOB %s was not submitted to YARN!", jobName);
            LOGGER.error(message);
            submarineUI.outputLog("JOR RUN FAILED", message);
            submarineJob.setCurrentJobState(EXECUTE_SUBMARINE_ERROR);
        }
    } catch (Exception e) {
        LOGGER.error(e.getMessage(), e);
        submarineJob.setCurrentJobState(EXECUTE_SUBMARINE_ERROR);
        submarineUI.outputLog("Exception", e.getMessage());
    } finally {
        running.set(false);
        lockRunning.unlock();
    }
}
Also used : HashMap(java.util.HashMap) Properties(java.util.Properties) URL(java.net.URL) PumpStreamHandler(org.apache.commons.exec.PumpStreamHandler) ExecuteException(org.apache.commons.exec.ExecuteException) InterpreterContext(org.apache.zeppelin.interpreter.InterpreterContext) DefaultExecuteResultHandler(org.apache.commons.exec.DefaultExecuteResultHandler) DefaultExecutor(org.apache.commons.exec.DefaultExecutor) ExecuteWatchdog(org.apache.commons.exec.ExecuteWatchdog) Jinjava(com.hubspot.jinjava.Jinjava) HdfsClient(org.apache.zeppelin.submarine.hadoop.HdfsClient) LogOutputStream(org.apache.commons.exec.LogOutputStream) ExecuteException(org.apache.commons.exec.ExecuteException) ParagraphInfo(org.apache.zeppelin.interpreter.thrift.ParagraphInfo) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) CommandLine(org.apache.commons.exec.CommandLine) File(java.io.File) SubmarineUI(org.apache.zeppelin.submarine.commons.SubmarineUI)

Aggregations

DefaultExecuteResultHandler (org.apache.commons.exec.DefaultExecuteResultHandler)16 CommandLine (org.apache.commons.exec.CommandLine)12 DefaultExecutor (org.apache.commons.exec.DefaultExecutor)10 PumpStreamHandler (org.apache.commons.exec.PumpStreamHandler)9 ExecuteWatchdog (org.apache.commons.exec.ExecuteWatchdog)8 ExecuteException (org.apache.commons.exec.ExecuteException)6 Executor (org.apache.commons.exec.Executor)6 File (java.io.File)5 IOException (java.io.IOException)5 ByteArrayOutputStream (java.io.ByteArrayOutputStream)4 HashMap (java.util.HashMap)4 URL (java.net.URL)3 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)3 LogOutputStream (org.apache.commons.exec.LogOutputStream)3 Jinjava (com.hubspot.jinjava.Jinjava)2 Properties (java.util.Properties)2 ShutdownHookProcessDestroyer (org.apache.commons.exec.ShutdownHookProcessDestroyer)2 SubmarineUI (org.apache.zeppelin.submarine.commons.SubmarineUI)2 ConfigConstants (io.georocket.constants.ConfigConstants)1 RxUtils (io.georocket.util.RxUtils)1