Examples with ExecuteWatchdog org.apache.commons.exec.ExecuteWatchdog used on opensource projects

Example 31 with ExecuteWatchdog

use of org.apache.commons.exec.ExecuteWatchdog in project zeppelin by apache.

the class MongoDbInterpreter method interpret.

@Override
public InterpreterResult interpret(String script, InterpreterContext context) {
    LOGGER.debug("Run MongoDB script: {}", script);
    if (StringUtils.isEmpty(script)) {
        return new InterpreterResult(Code.SUCCESS);
    }
    String paragraphId = context.getParagraphId();
    // Write script in a temporary file
    // The script is enriched with extensions
    final File scriptFile = new File(getScriptFileName(paragraphId));
    try {
        FileUtils.write(scriptFile, shellExtension + script);
    } catch (IOException e) {
        LOGGER.error("Can not write script in temp file", e);
        return new InterpreterResult(Code.ERROR, e.getMessage());
    }
    InterpreterResult result = new InterpreterResult(InterpreterResult.Code.SUCCESS);
    final DefaultExecutor executor = new DefaultExecutor();
    final ByteArrayOutputStream errorStream = new ByteArrayOutputStream();
    executor.setStreamHandler(new PumpStreamHandler(context.out, errorStream));
    executor.setWatchdog(new ExecuteWatchdog(commandTimeout));
    final CommandLine cmdLine = CommandLine.parse(getProperty("mongo.shell.path"));
    cmdLine.addArgument("--quiet", false);
    cmdLine.addArgument(dbAddress, false);
    cmdLine.addArgument(scriptFile.getAbsolutePath(), false);
    try {
        executor.execute(cmdLine);
        runningProcesses.put(paragraphId, executor);
    } catch (ExecuteException e) {
        LOGGER.error("Can not run script in paragraph {}", paragraphId, e);
        final int exitValue = e.getExitValue();
        Code code = Code.ERROR;
        String msg = errorStream.toString();
        if (exitValue == SIGTERM_CODE) {
            code = Code.INCOMPLETE;
            msg = msg + "Paragraph received a SIGTERM.\n";
            LOGGER.info("The paragraph {} stopped executing: {}", paragraphId, msg);
        }
        msg += "ExitValue: " + exitValue;
        result = new InterpreterResult(code, msg);
    } catch (IOException e) {
        LOGGER.error("Can not run script in paragraph {}", paragraphId, e);
        result = new InterpreterResult(Code.ERROR, e.getMessage());
    } finally {
        FileUtils.deleteQuietly(scriptFile);
        stopProcess(paragraphId);
    }
    return result;
}

Example 32 with ExecuteWatchdog

use of org.apache.commons.exec.ExecuteWatchdog in project zeppelin by apache.

the class TensorboardRunThread method run.

public void run() {
    SubmarineUI submarineUI = submarineJob.getSubmarineUI();
    boolean tryLock = lockRunning.tryLock();
    try {
        Properties properties = submarineJob.getProperties();
        String tensorboardName = SubmarineUtils.getTensorboardName(submarineJob.getUserName());
        if (true == running.get()) {
            String message = String.format("tensorboard %s already running.", tensorboardName);
            submarineUI.outputLog("WARN", message);
            LOGGER.warn(message);
            return;
        }
        running.set(true);
        HashMap jinjaParams = SubmarineUtils.propertiesToJinjaParams(properties, submarineJob, false);
        // update jobName -> tensorboardName
        jinjaParams.put(SubmarineConstants.JOB_NAME, tensorboardName);
        URL urlTemplate = Resources.getResource(SubmarineJob.SUBMARINE_TENSORBOARD_JINJA);
        String template = Resources.toString(urlTemplate, Charsets.UTF_8);
        Jinjava jinjava = new Jinjava();
        String submarineCmd = jinjava.render(template, jinjaParams);
        // If the first line is a newline, delete the newline
        int firstLineIsNewline = submarineCmd.indexOf("\n");
        if (firstLineIsNewline == 0) {
            submarineCmd = submarineCmd.replaceFirst("\n", "");
        }
        StringBuffer sbLogs = new StringBuffer(submarineCmd);
        submarineUI.outputLog("Submarine submit command", sbLogs.toString());
        long timeout = Long.valueOf(properties.getProperty(SubmarineJob.TIMEOUT_PROPERTY, SubmarineJob.defaultTimeout));
        CommandLine cmdLine = CommandLine.parse(SubmarineJob.shell);
        cmdLine.addArgument(submarineCmd, false);
        DefaultExecutor executor = new DefaultExecutor();
        ExecuteWatchdog watchDog = new ExecuteWatchdog(timeout);
        executor.setWatchdog(watchDog);
        StringBuffer sbLogOutput = new StringBuffer();
        executor.setStreamHandler(new PumpStreamHandler(new LogOutputStream() {

            @Override
            protected void processLine(String line, int level) {
                line = line.trim();
                if (!StringUtils.isEmpty(line)) {
                    sbLogOutput.append(line + "\n");
                }
            }
        }));
        if (Boolean.valueOf(properties.getProperty(SubmarineJob.DIRECTORY_USER_HOME))) {
            executor.setWorkingDirectory(new File(System.getProperty("user.home")));
        }
        Map<String, String> env = new HashMap<>();
        String launchMode = (String) jinjaParams.get(SubmarineConstants.INTERPRETER_LAUNCH_MODE);
        if (StringUtils.equals(launchMode, "yarn")) {
            // Set environment variables in the container
            String javaHome, hadoopHome, hadoopConf;
            javaHome = (String) jinjaParams.get(SubmarineConstants.DOCKER_JAVA_HOME);
            hadoopHome = (String) jinjaParams.get(SubmarineConstants.DOCKER_HADOOP_HDFS_HOME);
            hadoopConf = (String) jinjaParams.get(SubmarineConstants.SUBMARINE_HADOOP_CONF_DIR);
            env.put("JAVA_HOME", javaHome);
            env.put("HADOOP_HOME", hadoopHome);
            env.put("HADOOP_HDFS_HOME", hadoopHome);
            env.put("HADOOP_CONF_DIR", hadoopConf);
            env.put("YARN_CONF_DIR", hadoopConf);
            env.put("CLASSPATH", "`$HADOOP_HDFS_HOME/bin/hadoop classpath --glob`");
        }
        LOGGER.info("Execute EVN: {}, Command: {} ", env.toString(), submarineCmd);
        AtomicBoolean cmdLineRunning = new AtomicBoolean(true);
        executor.execute(cmdLine, env, new DefaultExecuteResultHandler() {

            @Override
            public void onProcessComplete(int exitValue) {
                String message = String.format("jobName %s ProcessComplete exit value is : %d", tensorboardName, exitValue);
                LOGGER.info(message);
                submarineUI.outputLog("TENSORBOARD RUN COMPLETE", message);
                cmdLineRunning.set(false);
            }

            @Override
            public void onProcessFailed(ExecuteException e) {
                String message = String.format("jobName %s ProcessFailed exit value is : %d, exception is : %s", tensorboardName, e.getExitValue(), e.getMessage());
                LOGGER.error(message);
                submarineUI.outputLog("TENSORBOARD RUN FAILED", message);
                cmdLineRunning.set(false);
            }
        });
        int loopCount = 100;
        while ((loopCount-- > 0) && cmdLineRunning.get() && running.get()) {
            Thread.sleep(1000);
        }
        if (watchDog.isWatching()) {
            watchDog.destroyProcess();
            Thread.sleep(1000);
        }
        if (watchDog.isWatching()) {
            watchDog.killedProcess();
        }
        // Check if it has been submitted to YARN
        Map<String, Object> jobState = submarineJob.getJobStateByYarn(tensorboardName);
        loopCount = 50;
        while ((loopCount-- > 0) && !jobState.containsKey("state") && running.get()) {
            Thread.sleep(3000);
            jobState = submarineJob.getJobStateByYarn(tensorboardName);
        }
        if (!jobState.containsKey("state")) {
            String message = String.format("tensorboard %s was not submitted to YARN!", tensorboardName);
            LOGGER.error(message);
            submarineUI.outputLog("JOR RUN FAILED", message);
        }
    } catch (Exception e) {
        LOGGER.error(e.getMessage(), e);
        submarineUI.outputLog("Exception", e.getMessage());
    } finally {
        running.set(false);
        lockRunning.unlock();
    }
}

Example 33 with ExecuteWatchdog

use of org.apache.commons.exec.ExecuteWatchdog in project zeppelin by apache.

the class JobRunThread method run.

public void run() {
    boolean tryLock = lockRunning.tryLock();
    if (false == tryLock) {
        LOGGER.warn("Can not get JobRunThread lockRunning!");
        return;
    }
    SubmarineUI submarineUI = submarineJob.getSubmarineUI();
    try {
        InterpreterContext intpContext = submarineJob.getIntpContext();
        String noteId = intpContext.getNoteId();
        String userName = intpContext.getAuthenticationInfo().getUser();
        String jobName = SubmarineUtils.getJobName(userName, noteId);
        if (true == running.get()) {
            String message = String.format("Job %s already running.", jobName);
            submarineUI.outputLog("WARN", message);
            LOGGER.warn(message);
            return;
        }
        running.set(true);
        Properties properties = submarineJob.getProperties();
        HdfsClient hdfsClient = submarineJob.getHdfsClient();
        File pythonWorkDir = submarineJob.getPythonWorkDir();
        submarineJob.setCurrentJobState(EXECUTE_SUBMARINE);
        String algorithmPath = properties.getProperty(SubmarineConstants.SUBMARINE_ALGORITHM_HDFS_PATH, "");
        if (!algorithmPath.startsWith("hdfs://")) {
            String message = "Algorithm file upload HDFS path, " + "Must be `hdfs://` prefix. now setting " + algorithmPath;
            submarineUI.outputLog("Configuration error", message);
            return;
        }
        List<ParagraphInfo> paragraphInfos = intpContext.getIntpEventClient().getParagraphList(userName, noteId);
        String outputMsg = hdfsClient.saveParagraphToFiles(noteId, paragraphInfos, pythonWorkDir == null ? "" : pythonWorkDir.getAbsolutePath(), properties);
        if (!StringUtils.isEmpty(outputMsg)) {
            submarineUI.outputLog("Save algorithm file", outputMsg);
        }
        HashMap jinjaParams = SubmarineUtils.propertiesToJinjaParams(properties, submarineJob, true);
        URL urlTemplate = Resources.getResource(SubmarineJob.SUBMARINE_JOBRUN_TF_JINJA);
        String template = Resources.toString(urlTemplate, Charsets.UTF_8);
        Jinjava jinjava = new Jinjava();
        String submarineCmd = jinjava.render(template, jinjaParams);
        // If the first line is a newline, delete the newline
        int firstLineIsNewline = submarineCmd.indexOf("\n");
        if (firstLineIsNewline == 0) {
            submarineCmd = submarineCmd.replaceFirst("\n", "");
        }
        StringBuffer sbLogs = new StringBuffer(submarineCmd);
        submarineUI.outputLog("Submarine submit command", sbLogs.toString());
        long timeout = Long.valueOf(properties.getProperty(SubmarineJob.TIMEOUT_PROPERTY, SubmarineJob.defaultTimeout));
        CommandLine cmdLine = CommandLine.parse(SubmarineJob.shell);
        cmdLine.addArgument(submarineCmd, false);
        DefaultExecutor executor = new DefaultExecutor();
        ExecuteWatchdog watchDog = new ExecuteWatchdog(timeout);
        executor.setWatchdog(watchDog);
        StringBuffer sbLogOutput = new StringBuffer();
        executor.setStreamHandler(new PumpStreamHandler(new LogOutputStream() {

            @Override
            protected void processLine(String line, int level) {
                line = line.trim();
                if (!StringUtils.isEmpty(line)) {
                    sbLogOutput.append(line + "\n");
                }
            }
        }));
        if (Boolean.valueOf(properties.getProperty(SubmarineJob.DIRECTORY_USER_HOME))) {
            executor.setWorkingDirectory(new File(System.getProperty("user.home")));
        }
        Map<String, String> env = new HashMap<>();
        String launchMode = (String) jinjaParams.get(SubmarineConstants.INTERPRETER_LAUNCH_MODE);
        if (StringUtils.equals(launchMode, "yarn")) {
            // Set environment variables in the submarine interpreter container run on yarn
            String javaHome, hadoopHome, hadoopConf;
            javaHome = (String) jinjaParams.get(SubmarineConstants.DOCKER_JAVA_HOME);
            hadoopHome = (String) jinjaParams.get(SubmarineConstants.DOCKER_HADOOP_HDFS_HOME);
            hadoopConf = (String) jinjaParams.get(SubmarineConstants.SUBMARINE_HADOOP_CONF_DIR);
            env.put("JAVA_HOME", javaHome);
            env.put("HADOOP_HOME", hadoopHome);
            env.put("HADOOP_HDFS_HOME", hadoopHome);
            env.put("HADOOP_CONF_DIR", hadoopConf);
            env.put("YARN_CONF_DIR", hadoopConf);
            env.put("CLASSPATH", "`$HADOOP_HDFS_HOME/bin/hadoop classpath --glob`");
            env.put("ZEPPELIN_FORCE_STOP", "true");
        }
        LOGGER.info("Execute EVN: {}, Command: {} ", env.toString(), submarineCmd);
        AtomicBoolean cmdLineRunning = new AtomicBoolean(true);
        executor.execute(cmdLine, env, new DefaultExecuteResultHandler() {

            @Override
            public void onProcessComplete(int exitValue) {
                String message = String.format("jobName %s ProcessComplete exit value is : %d", jobName, exitValue);
                LOGGER.info(message);
                submarineUI.outputLog("JOR RUN COMPLETE", message);
                cmdLineRunning.set(false);
                submarineJob.setCurrentJobState(EXECUTE_SUBMARINE_FINISHED);
            }

            @Override
            public void onProcessFailed(ExecuteException e) {
                String message = String.format("jobName %s ProcessFailed exit value is : %d, exception is : %s", jobName, e.getExitValue(), e.getMessage());
                LOGGER.error(message);
                submarineUI.outputLog("JOR RUN FAILED", message);
                cmdLineRunning.set(false);
                submarineJob.setCurrentJobState(EXECUTE_SUBMARINE_ERROR);
            }
        });
        int loopCount = 100;
        while ((loopCount-- > 0) && cmdLineRunning.get() && running.get()) {
            Thread.sleep(1000);
        }
        if (watchDog.isWatching()) {
            watchDog.destroyProcess();
            Thread.sleep(1000);
        }
        if (watchDog.isWatching()) {
            watchDog.killedProcess();
        }
        // Check if it has been submitted to YARN
        Map<String, Object> jobState = submarineJob.getJobStateByYarn(jobName);
        loopCount = 50;
        while ((loopCount-- > 0) && !jobState.containsKey("state") && running.get()) {
            Thread.sleep(3000);
            jobState = submarineJob.getJobStateByYarn(jobName);
        }
        if (!jobState.containsKey("state")) {
            String message = String.format("JOB %s was not submitted to YARN!", jobName);
            LOGGER.error(message);
            submarineUI.outputLog("JOR RUN FAILED", message);
            submarineJob.setCurrentJobState(EXECUTE_SUBMARINE_ERROR);
        }
    } catch (Exception e) {
        LOGGER.error(e.getMessage(), e);
        submarineJob.setCurrentJobState(EXECUTE_SUBMARINE_ERROR);
        submarineUI.outputLog("Exception", e.getMessage());
    } finally {
        running.set(false);
        lockRunning.unlock();
    }
}

Example 34 with ExecuteWatchdog

use of org.apache.commons.exec.ExecuteWatchdog in project zeppelin by apache.

the class ShellInterpreter method internalInterpret.

@Override
public InterpreterResult internalInterpret(String cmd, InterpreterContext context) {
    LOGGER.debug("Run shell command '{}'", cmd);
    CommandLine cmdLine = CommandLine.parse(shell);
    // they need to be delimited by '&&' instead
    if (isWindows) {
        String[] lines = StringUtils.split(cmd, "\n");
        cmd = StringUtils.join(lines, " && ");
    }
    cmdLine.addArgument(cmd, false);
    try {
        contextMap.put(context.getParagraphId(), context);
        DefaultExecutor executor = new DefaultExecutor();
        executor.setStreamHandler(new PumpStreamHandler(context.out, context.out));
        executor.setWatchdog(new ExecuteWatchdog(Long.MAX_VALUE));
        executorMap.put(context.getParagraphId(), executor);
        if (Boolean.valueOf(getProperty(DIRECTORY_USER_HOME))) {
            executor.setWorkingDirectory(new File(System.getProperty("user.home")));
        }
        int exitVal = executor.execute(cmdLine);
        LOGGER.info("Paragraph {} return with exit value: {}", context.getParagraphId(), exitVal);
        if (exitVal == 0) {
            return new InterpreterResult(Code.SUCCESS);
        } else {
            return new InterpreterResult(Code.ERROR);
        }
    } catch (ExecuteException e) {
        int exitValue = e.getExitValue();
        LOGGER.error("Can not run command: " + cmd, e);
        Code code = Code.ERROR;
        StringBuilder messageBuilder = new StringBuilder();
        if (exitValue == 143) {
            code = Code.INCOMPLETE;
            messageBuilder.append("Paragraph received a SIGTERM\n");
            LOGGER.info("The paragraph {} stopped executing: {}", context.getParagraphId(), messageBuilder.toString());
        }
        messageBuilder.append("ExitValue: " + exitValue);
        return new InterpreterResult(code, messageBuilder.toString());
    } catch (IOException e) {
        LOGGER.error("Can not run command: " + cmd, e);
        return new InterpreterResult(Code.ERROR, e.getMessage());
    } finally {
        executorMap.remove(context.getParagraphId());
        contextMap.remove(context.getParagraphId());
    }
}

Example 35 with ExecuteWatchdog

use of org.apache.commons.exec.ExecuteWatchdog in project tycho by eclipse.

the class JdkLibraryInfoProvider method generateLibraryInfo.

private LibraryInfo generateLibraryInfo(String javaHome) {
    String executable = javaHome + File.separator + "bin" + File.separator + "java";
    if (File.separatorChar == '\\') {
        executable = executable + ".exe";
    }
    if (!new File(executable).isFile()) {
        getLog().warn(executable + " not found. Fallback to scan " + javaHome + "/lib/*.jar and " + javaHome + "/lib/ext/*.jar for bootclasspath");
        return new LibraryInfo("unknown", scanLibFolders(javaHome), new String[0], new String[0]);
    }
    CommandLine cli = new CommandLine(executable);
    cli.addArguments(new String[] { "-classpath", getLibDetectorJar().getAbsolutePath(), "org.eclipse.tycho.libdetector.LibraryDetector" }, false);
    DefaultExecutor executor = new DefaultExecutor();
    ExecuteWatchdog watchdog = new ExecuteWatchdog(30 * 1000L);
    executor.setWatchdog(watchdog);
    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
    PumpStreamHandler handler = new PumpStreamHandler(outputStream);
    executor.setStreamHandler(handler);
    int exitValue = -1;
    try {
        exitValue = executor.execute(cli);
    } catch (ExecuteException e) {
        if (watchdog.killedProcess()) {
            throw new RuntimeException("Timeout 30 s exceeded. Commandline " + cli.toString() + " was killed. Output: " + outputStream.toString());
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    if (exitValue == 0) {
        return parseLibraryInfo(outputStream.toString(), javaHome);
    } else {
        throw new RuntimeException(cli.toString() + " process exit code was " + exitValue + ". Output: " + outputStream.toString());
    }
}