Examples with PumpStreamHandler org.apache.commons.exec.PumpStreamHandler used on opensource projects

Example 11 with PumpStreamHandler

use of org.apache.commons.exec.PumpStreamHandler in project opennms by OpenNMS.

the class RScriptExecutor method exec.

/**
 * Executes by given script by:
 *   - Searching both the classpath and the filesystem for the path
 *   - Copying the script at the given path to a temporary file and
 *     performing variable substitution with the arguments using Freemarker.
 *   - Invoking the script with commons-exec
 *   - Converting the input table to CSV and passing this to the process via stdin
 *   - Parsing stdout, expecting CSV output, and converting this to an immutable table
 */
public RScriptOutput exec(String script, RScriptInput input) throws RScriptException {
    Preconditions.checkNotNull(script, "script argument");
    Preconditions.checkNotNull(input, "input argument");
    // Grab the script/template
    Template template;
    try {
        template = m_freemarkerConfiguration.getTemplate(script);
    } catch (IOException e) {
        throw new RScriptException("Failed to read the script.", e);
    }
    // Create a temporary file
    File scriptOnDisk;
    try {
        scriptOnDisk = File.createTempFile("Rcsript", "R");
        scriptOnDisk.deleteOnExit();
    } catch (IOException e) {
        throw new RScriptException("Failed to create a temporary file.", e);
    }
    // Perform variable substitution and write the results to the temporary file
    try (FileOutputStream fos = new FileOutputStream(scriptOnDisk);
        Writer out = new OutputStreamWriter(fos)) {
        template.process(input.getArguments(), out);
    } catch (IOException | TemplateException e) {
        scriptOnDisk.delete();
        throw new RScriptException("Failed to process the template.", e);
    }
    // Convert the input matrix to a CSV string which will be passed to the script via stdin.
    // The table may be large, so we try and avoid writing it to disk
    final StringBuilder inputTableAsCsv;
    try {
        inputTableAsCsv = toCsv(input.getTable());
    } catch (IOException e) {
        scriptOnDisk.delete();
        throw new RScriptException("Failed to convert the input table to CSV.", e);
    }
    // Invoke Rscript against the script (located in a temporary file)
    CommandLine cmdLine = new CommandLine(RSCRIPT_BINARY);
    cmdLine.addArgument(scriptOnDisk.getAbsolutePath());
    // Use commons-exec to execute the process
    DefaultExecutor executor = new DefaultExecutor();
    // Use the CharSequenceInputStream in order to avoid explicitly converting
    // the StringBuilder a string and then an array of bytes.
    InputStream stdin = new CharSequenceInputStream(inputTableAsCsv, StandardCharsets.UTF_8);
    ByteArrayOutputStream stderr = new ByteArrayOutputStream();
    ByteArrayOutputStream stdout = new ByteArrayOutputStream();
    executor.setStreamHandler(new PumpStreamHandler(stdout, stderr, stdin));
    // Fail if we get a non-zero exit code
    executor.setExitValue(0);
    // Fail if the process takes too long
    ExecuteWatchdog watchdog = new ExecuteWatchdog(SCRIPT_TIMEOUT_MS);
    executor.setWatchdog(watchdog);
    // Execute
    try {
        executor.execute(cmdLine);
    } catch (IOException e) {
        scriptOnDisk.delete();
        throw new RScriptException("An error occured while executing Rscript, or the requested script.", inputTableAsCsv.toString(), stderr.toString(), stdout.toString(), e);
    }
    // Parse and return the results
    try {
        ImmutableTable<Long, String, Double> table = fromCsv(stdout.toString());
        return new RScriptOutput(table);
    } catch (Throwable t) {
        throw new RScriptException("Failed to parse the script's output.", inputTableAsCsv.toString(), stderr.toString(), stdout.toString(), t);
    } finally {
        scriptOnDisk.delete();
    }
}

Example 12 with PumpStreamHandler

use of org.apache.commons.exec.PumpStreamHandler in project zeppelin by apache.

the class PySparkInterpreter method createGatewayServerAndStartScript.

private void createGatewayServerAndStartScript() {
    // create python script
    createPythonScript();
    port = findRandomOpenPortOnAllLocalInterfaces();
    gatewayServer = new GatewayServer(this, port);
    gatewayServer.start();
    // Run python shell
    // Choose python in the order of
    // PYSPARK_DRIVER_PYTHON > PYSPARK_PYTHON > zeppelin.pyspark.python
    String pythonExec = getProperty("zeppelin.pyspark.python");
    if (System.getenv("PYSPARK_PYTHON") != null) {
        pythonExec = System.getenv("PYSPARK_PYTHON");
    }
    if (System.getenv("PYSPARK_DRIVER_PYTHON") != null) {
        pythonExec = System.getenv("PYSPARK_DRIVER_PYTHON");
    }
    CommandLine cmd = CommandLine.parse(pythonExec);
    cmd.addArgument(scriptPath, false);
    cmd.addArgument(Integer.toString(port), false);
    cmd.addArgument(Integer.toString(getSparkInterpreter().getSparkVersion().toNumber()), false);
    executor = new DefaultExecutor();
    outputStream = new InterpreterOutputStream(logger);
    PipedOutputStream ps = new PipedOutputStream();
    in = null;
    try {
        in = new PipedInputStream(ps);
    } catch (IOException e1) {
        throw new InterpreterException(e1);
    }
    ins = new BufferedWriter(new OutputStreamWriter(ps));
    input = new ByteArrayOutputStream();
    PumpStreamHandler streamHandler = new PumpStreamHandler(outputStream, outputStream, in);
    executor.setStreamHandler(streamHandler);
    executor.setWatchdog(new ExecuteWatchdog(ExecuteWatchdog.INFINITE_TIMEOUT));
    try {
        Map env = setupPySparkEnv();
        executor.execute(cmd, env, this);
        pythonscriptRunning = true;
    } catch (IOException e) {
        throw new InterpreterException(e);
    }
    try {
        input.write("import sys, getopt\n".getBytes());
        ins.flush();
    } catch (IOException e) {
        throw new InterpreterException(e);
    }
}

Example 13 with PumpStreamHandler

use of org.apache.commons.exec.PumpStreamHandler in project zeppelin by apache.

the class PythonInterpreter method createGatewayServerAndStartScript.

private void createGatewayServerAndStartScript() throws UnknownHostException {
    createPythonScript();
    if (System.getenv("ZEPPELIN_HOME") != null) {
        py4jLibPath = System.getenv("ZEPPELIN_HOME") + File.separator + ZEPPELIN_PY4JPATH;
        pythonLibPath = System.getenv("ZEPPELIN_HOME") + File.separator + ZEPPELIN_PYTHON_LIBS;
    } else {
        Path workingPath = Paths.get("..").toAbsolutePath();
        py4jLibPath = workingPath + File.separator + ZEPPELIN_PY4JPATH;
        pythonLibPath = workingPath + File.separator + ZEPPELIN_PYTHON_LIBS;
    }
    port = findRandomOpenPortOnAllLocalInterfaces();
    gatewayServer = new GatewayServer(this, port, GatewayServer.DEFAULT_PYTHON_PORT, InetAddress.getByName("0.0.0.0"), InetAddress.getByName("0.0.0.0"), GatewayServer.DEFAULT_CONNECT_TIMEOUT, GatewayServer.DEFAULT_READ_TIMEOUT, (List) null);
    gatewayServer.start();
    // Run python shell
    String pythonCmd = getPythonCommand();
    CommandLine cmd = CommandLine.parse(pythonCmd);
    if (!pythonCmd.endsWith(".py")) {
        // PythonDockerInterpreter set pythoncmd with script
        cmd.addArgument(getScriptPath(), false);
    }
    cmd.addArgument(Integer.toString(port), false);
    cmd.addArgument(getLocalIp(), false);
    executor = new DefaultExecutor();
    outputStream = new InterpreterOutputStream(logger);
    PipedOutputStream ps = new PipedOutputStream();
    in = null;
    try {
        in = new PipedInputStream(ps);
    } catch (IOException e1) {
        throw new InterpreterException(e1);
    }
    ins = new BufferedWriter(new OutputStreamWriter(ps));
    input = new ByteArrayOutputStream();
    PumpStreamHandler streamHandler = new PumpStreamHandler(outputStream, outputStream, in);
    executor.setStreamHandler(streamHandler);
    executor.setWatchdog(new ExecuteWatchdog(ExecuteWatchdog.INFINITE_TIMEOUT));
    try {
        Map env = EnvironmentUtils.getProcEnvironment();
        if (!env.containsKey("PYTHONPATH")) {
            env.put("PYTHONPATH", py4jLibPath + File.pathSeparator + pythonLibPath);
        } else {
            env.put("PYTHONPATH", env.get("PYTHONPATH") + File.pathSeparator + py4jLibPath + File.pathSeparator + pythonLibPath);
        }
        logger.info("cmd = {}", cmd.toString());
        executor.execute(cmd, env, this);
        pythonscriptRunning = true;
    } catch (IOException e) {
        throw new InterpreterException(e);
    }
    try {
        input.write("import sys, getopt\n".getBytes());
        ins.flush();
    } catch (IOException e) {
        throw new InterpreterException(e);
    }
}

Example 14 with PumpStreamHandler

use of org.apache.commons.exec.PumpStreamHandler in project frontend-maven-plugin by eirslett.

the class ProcessExecutor method execute.

private int execute(final Logger logger, final OutputStream stdout, final OutputStream stderr) throws ProcessExecutionException {
    logger.debug("Executing command line {}", commandLine);
    try {
        ExecuteStreamHandler streamHandler = new PumpStreamHandler(stdout, stderr);
        executor.setStreamHandler(streamHandler);
        int exitValue = executor.execute(commandLine, environment);
        logger.debug("Exit value {}", exitValue);
        return exitValue;
    } catch (ExecuteException e) {
        if (executor.getWatchdog() != null && executor.getWatchdog().killedProcess()) {
            throw new ProcessExecutionException("Process killed after timeout");
        }
        throw new ProcessExecutionException(e);
    } catch (IOException e) {
        throw new ProcessExecutionException(e);
    }
}

Example 15 with PumpStreamHandler

use of org.apache.commons.exec.PumpStreamHandler in project zeppelin by apache.

the class AbstractTestRestApi method ps.

public static void ps() {
    DefaultExecutor executor = new DefaultExecutor();
    executor.setStreamHandler(new PumpStreamHandler(System.out, System.err));
    CommandLine cmd = CommandLine.parse("ps");
    cmd.addArgument("aux", false);
    try {
        executor.execute(cmd);
    } catch (IOException e) {
        LOG.error(e.getMessage(), e);
    }
}