Examples with InterpreterOutputStream org.apache.zeppelin.interpreter.util.InterpreterOutputStream used on opensource projects

Example 1 with InterpreterOutputStream

use of org.apache.zeppelin.interpreter.util.InterpreterOutputStream in project zeppelin by apache.

the class ZeppelinR method open.

/**
   * Start R repl
   * @throws IOException
   */
public void open() throws IOException {
    createRScript();
    zeppelinR.put(hashCode(), this);
    CommandLine cmd = CommandLine.parse(rCmdPath);
    cmd.addArgument("--no-save");
    cmd.addArgument("--no-restore");
    cmd.addArgument("-f");
    cmd.addArgument(scriptPath);
    cmd.addArgument("--args");
    cmd.addArgument(Integer.toString(hashCode()));
    cmd.addArgument(Integer.toString(port));
    cmd.addArgument(libPath);
    cmd.addArgument(Integer.toString(sparkVersion.toNumber()));
    // dump out the R command to facilitate manually running it, e.g. for fault diagnosis purposes
    logger.debug(cmd.toString());
    executor = new DefaultExecutor();
    outputStream = new InterpreterOutputStream(logger);
    input = new PipedOutputStream();
    PipedInputStream in = new PipedInputStream(input);
    PumpStreamHandler streamHandler = new PumpStreamHandler(outputStream, outputStream, in);
    executor.setWatchdog(new ExecuteWatchdog(ExecuteWatchdog.INFINITE_TIMEOUT));
    executor.setStreamHandler(streamHandler);
    Map env = EnvironmentUtils.getProcEnvironment();
    initialOutput = new InterpreterOutput(null);
    outputStream.setInterpreterOutput(initialOutput);
    executor.execute(cmd, env, this);
    rScriptRunning = true;
    // flush output
    eval("cat('')");
}

Example 2 with InterpreterOutputStream

use of org.apache.zeppelin.interpreter.util.InterpreterOutputStream in project zeppelin by apache.

the class PySparkInterpreter method createGatewayServerAndStartScript.

private void createGatewayServerAndStartScript() {
    // create python script
    createPythonScript();
    port = findRandomOpenPortOnAllLocalInterfaces();
    gatewayServer = new GatewayServer(this, port);
    gatewayServer.start();
    // Run python shell
    // Choose python in the order of
    // PYSPARK_DRIVER_PYTHON > PYSPARK_PYTHON > zeppelin.pyspark.python
    String pythonExec = getProperty("zeppelin.pyspark.python");
    if (System.getenv("PYSPARK_PYTHON") != null) {
        pythonExec = System.getenv("PYSPARK_PYTHON");
    }
    if (System.getenv("PYSPARK_DRIVER_PYTHON") != null) {
        pythonExec = System.getenv("PYSPARK_DRIVER_PYTHON");
    }
    CommandLine cmd = CommandLine.parse(pythonExec);
    cmd.addArgument(scriptPath, false);
    cmd.addArgument(Integer.toString(port), false);
    cmd.addArgument(Integer.toString(getSparkInterpreter().getSparkVersion().toNumber()), false);
    executor = new DefaultExecutor();
    outputStream = new InterpreterOutputStream(logger);
    PipedOutputStream ps = new PipedOutputStream();
    in = null;
    try {
        in = new PipedInputStream(ps);
    } catch (IOException e1) {
        throw new InterpreterException(e1);
    }
    ins = new BufferedWriter(new OutputStreamWriter(ps));
    input = new ByteArrayOutputStream();
    PumpStreamHandler streamHandler = new PumpStreamHandler(outputStream, outputStream, in);
    executor.setStreamHandler(streamHandler);
    executor.setWatchdog(new ExecuteWatchdog(ExecuteWatchdog.INFINITE_TIMEOUT));
    try {
        Map env = setupPySparkEnv();
        executor.execute(cmd, env, this);
        pythonscriptRunning = true;
    } catch (IOException e) {
        throw new InterpreterException(e);
    }
    try {
        input.write("import sys, getopt\n".getBytes());
        ins.flush();
    } catch (IOException e) {
        throw new InterpreterException(e);
    }
}

Example 3 with InterpreterOutputStream

use of org.apache.zeppelin.interpreter.util.InterpreterOutputStream in project zeppelin by apache.

the class PythonInterpreter method createGatewayServerAndStartScript.

private void createGatewayServerAndStartScript() throws UnknownHostException {
    createPythonScript();
    if (System.getenv("ZEPPELIN_HOME") != null) {
        py4jLibPath = System.getenv("ZEPPELIN_HOME") + File.separator + ZEPPELIN_PY4JPATH;
        pythonLibPath = System.getenv("ZEPPELIN_HOME") + File.separator + ZEPPELIN_PYTHON_LIBS;
    } else {
        Path workingPath = Paths.get("..").toAbsolutePath();
        py4jLibPath = workingPath + File.separator + ZEPPELIN_PY4JPATH;
        pythonLibPath = workingPath + File.separator + ZEPPELIN_PYTHON_LIBS;
    }
    port = findRandomOpenPortOnAllLocalInterfaces();
    gatewayServer = new GatewayServer(this, port, GatewayServer.DEFAULT_PYTHON_PORT, InetAddress.getByName("0.0.0.0"), InetAddress.getByName("0.0.0.0"), GatewayServer.DEFAULT_CONNECT_TIMEOUT, GatewayServer.DEFAULT_READ_TIMEOUT, (List) null);
    gatewayServer.start();
    // Run python shell
    String pythonCmd = getPythonCommand();
    CommandLine cmd = CommandLine.parse(pythonCmd);
    if (!pythonCmd.endsWith(".py")) {
        // PythonDockerInterpreter set pythoncmd with script
        cmd.addArgument(getScriptPath(), false);
    }
    cmd.addArgument(Integer.toString(port), false);
    cmd.addArgument(getLocalIp(), false);
    executor = new DefaultExecutor();
    outputStream = new InterpreterOutputStream(logger);
    PipedOutputStream ps = new PipedOutputStream();
    in = null;
    try {
        in = new PipedInputStream(ps);
    } catch (IOException e1) {
        throw new InterpreterException(e1);
    }
    ins = new BufferedWriter(new OutputStreamWriter(ps));
    input = new ByteArrayOutputStream();
    PumpStreamHandler streamHandler = new PumpStreamHandler(outputStream, outputStream, in);
    executor.setStreamHandler(streamHandler);
    executor.setWatchdog(new ExecuteWatchdog(ExecuteWatchdog.INFINITE_TIMEOUT));
    try {
        Map env = EnvironmentUtils.getProcEnvironment();
        if (!env.containsKey("PYTHONPATH")) {
            env.put("PYTHONPATH", py4jLibPath + File.pathSeparator + pythonLibPath);
        } else {
            env.put("PYTHONPATH", env.get("PYTHONPATH") + File.pathSeparator + py4jLibPath + File.pathSeparator + pythonLibPath);
        }
        logger.info("cmd = {}", cmd.toString());
        executor.execute(cmd, env, this);
        pythonscriptRunning = true;
    } catch (IOException e) {
        throw new InterpreterException(e);
    }
    try {
        input.write("import sys, getopt\n".getBytes());
        ins.flush();
    } catch (IOException e) {
        throw new InterpreterException(e);
    }
}