Examples with GatewayServer py4j.GatewayServer used on opensource projects

Example 11 with GatewayServer

use of py4j.GatewayServer in project org.csstudio.display.builder by kasemir.

the class PythonGatewaySupport method run.

/**
 * Run a Python script, with a map of Java objects made accessible through a
 * gateway to Python. The port to which the gateway server is listening is
 * passed as an argument to the script.
 *
 * @param map Map which is to be accessed by the script
 * @param script Path (including name) of script which is to be run
 * @throws Exception If unable to execute command to run script
 */
public static void run(Map<String, Object> map, String script) throws Exception {
    GatewayServer server = new GatewayServer(new MapWrapper(map), 0);
    server.start();
    int port = server.getListeningPort();
    if (port == -1) {
        server.shutdown();
        throw new Exception("Exception instantiating PythonGatewaySupport: GatewayServer not listening");
    }
    // start Python process, passing port used to connect to Py4J Java Gateway
    final Process process = new ProcessBuilder("python", script, Integer.toString(port)).start();
    final Thread error_log = new LogWriter(process.getErrorStream(), "PythonErrors", Level.WARNING);
    final Thread python_out = new LogWriter(process.getInputStream(), "PythonOutput", Level.INFO);
    error_log.start();
    python_out.start();
    try {
        process.waitFor();
    } catch (InterruptedException ex) {
    // ignore; closing display creates interruption
    } finally {
        process.destroyForcibly();
    }
    error_log.join();
    python_out.join();
    server.shutdown();
}

Example 12 with GatewayServer

use of py4j.GatewayServer in project angel by Tencent.

the class PythonRunner method runPythonProcess.

private void runPythonProcess() {
    String pythonFileName = jconf.get(AngelConf.PYANGEL_PYFILE);
    String pyFilename = jconf.get(AngelConf.PYANGEL_PYDEPFILES);
    Configuration conf = jconf;
    String pyAngelExec = conf.get(AngelConf.PYANGEL_PYTHON, "python");
    GatewayServer gatewayServer = new py4j.GatewayServer(null, 0);
    Thread thread = new Thread(new Runnable() {

        @Override
        public void run() {
            gatewayServer.start();
        }
    });
    thread.setName("py4j-gateway-init");
    thread.setDaemon(true);
    thread.start();
    try {
        thread.join();
    } catch (InterruptedException ie) {
        LOG.error("failed to to start python server while join daemon thread: " + thread.getName(), ie);
        ie.printStackTrace();
    }
    // Create python path which include angel's jars, the python directory in ANGEL_HOME,
    // and other files submitted by user.
    ArrayList<String> pathItems = new ArrayList<>();
    pathItems.add(PythonUtils.getAngelPythonPath());
    pathItems.add(System.getenv("PYTHONPATH"));
    String pythonPath = String.join(File.separator, pathItems);
    LOG.info("python path is : " + pythonPath);
    // Launch python process
    List<String> pyProcInfoList = Arrays.asList(pyAngelExec, pythonFileName);
    ProcessBuilder pyProcBuilder = new ProcessBuilder(pyProcInfoList);
    Map<String, String> envMap = pyProcBuilder.environment();
    envMap.put("PYTHONPATH", pythonPath);
    envMap.put("PYTHONUNBUFFERED", "YES");
    envMap.put("PYANGEL_GATEWAY_PORT", "" + gatewayServer.getListeningPort());
    envMap.put("PYANGEL_PYTHON", conf.get(AngelConf.PYANGEL_PYTHON, "python"));
    envMap.put("PYTHONHASHSEED", System.getenv("PYTHONHASHSEED"));
    LOG.info("python gateway server port bind on : " + gatewayServer.getListeningPort());
    pyProcBuilder.redirectErrorStream(true);
    try {
        Process pyProc = pyProcBuilder.start();
        InputStream pyIns = pyProc.getInputStream();
        OutputStream pyOus = System.out;
        Thread redirectThread = new Thread(new Runnable() {

            @Override
            public void run() {
                byte[] buf = new byte[1024];
                try {
                    int len = pyIns.read(buf);
                    while (len != -1) {
                        pyOus.write(buf, 0, len);
                        pyOus.flush();
                        len = pyIns.read(buf);
                    }
                } catch (IOException ioe) {
                    LOG.error("EOF: ", ioe);
                }
            }
        });
        redirectThread.start();
        int exitCode = pyProc.waitFor();
        if (exitCode != 0) {
            throw new AngelException("failed to start python process, the Error Code is: " + exitCode);
        }
    } catch (InterruptedException ie) {
        LOG.error("failed to start redirect thread for python process", ie);
    } catch (IOException ioe) {
        LOG.error("EOF: ", ioe);
    } finally {
        gatewayServer.shutdown();
    }
}

Example 13 with GatewayServer

use of py4j.GatewayServer in project flink by apache.

the class PythonDriver method main.

public static void main(String[] args) throws Throwable {
    // e.g. pym a.b [user args]
    if (args.length < 2) {
        LOG.error("Required at least two arguments, only python file or python module is available.");
        System.exit(1);
    }
    // parse args
    final CommandLineParser<PythonDriverOptions> commandLineParser = new CommandLineParser<>(new PythonDriverOptionsParserFactory());
    PythonDriverOptions pythonDriverOptions = null;
    try {
        pythonDriverOptions = commandLineParser.parse(args);
    } catch (Exception e) {
        LOG.error("Could not parse command line arguments {}.", args, e);
        commandLineParser.printHelp(PythonDriver.class.getSimpleName());
        System.exit(1);
    }
    // Get configuration from ContextEnvironment/OptimizerPlanEnvironment. As the configurations
    // of
    // streaming and batch environments are always set at the same time, for streaming jobs we
    // can
    // also get its configuration from batch environments.
    Configuration config = ExecutionEnvironment.getExecutionEnvironment().getConfiguration();
    // start gateway server
    GatewayServer gatewayServer = PythonEnvUtils.startGatewayServer();
    PythonEnvUtils.setGatewayServer(gatewayServer);
    PythonEnvUtils.PythonProcessShutdownHook shutdownHook = null;
    // commands which will be exec in python progress.
    final List<String> commands = constructPythonCommands(pythonDriverOptions);
    try {
        // prepare the exec environment of python progress.
        String tmpDir = System.getProperty("java.io.tmpdir") + File.separator + "pyflink" + File.separator + UUID.randomUUID();
        // start the python process.
        Process pythonProcess = PythonEnvUtils.launchPy4jPythonClient(gatewayServer, config, commands, pythonDriverOptions.getEntryPointScript().orElse(null), tmpDir, true);
        shutdownHook = new PythonEnvUtils.PythonProcessShutdownHook(pythonProcess, gatewayServer, tmpDir);
        Runtime.getRuntime().addShutdownHook(shutdownHook);
        BufferedReader in = new BufferedReader(new InputStreamReader(pythonProcess.getInputStream(), StandardCharsets.UTF_8));
        LOG.info("--------------------------- Python Process Started --------------------------");
        // print the python process output to stdout and log file
        final StringBuilder sb = new StringBuilder();
        try {
            while (true) {
                String line = in.readLine();
                if (line == null) {
                    break;
                } else {
                    System.out.println(line);
                    sb.append(line);
                    sb.append("\n");
                }
            }
        } finally {
            LOG.info(sb.toString());
        }
        int exitCode = pythonProcess.waitFor();
        LOG.info("--------------------------- Python Process Exited ---------------------------");
        if (exitCode != 0) {
            throw new RuntimeException("Python process exits with code: " + exitCode);
        }
    } catch (Throwable e) {
        LOG.error("Run python process failed", e);
        if (PythonEnvUtils.capturedJavaException != null) {
            throw PythonEnvUtils.capturedJavaException;
        } else {
            // there is no harm to throw ProgramAbortException even if it is not the case.
            throw new ProgramAbortException(e);
        }
    } finally {
        PythonEnvUtils.setGatewayServer(null);
        if (shutdownHook != null && Runtime.getRuntime().removeShutdownHook(shutdownHook)) {
            shutdownHook.run();
        }
    }
}

Example 14 with GatewayServer

use of py4j.GatewayServer in project flink by apache.

the class PythonFunctionFactory method createPythonFunctionFactory.

static PythonFunctionFactory createPythonFunctionFactory(ReadableConfig config) throws ExecutionException, InterruptedException, IOException {
    Map<String, Object> entryPoint;
    PythonProcessShutdownHook shutdownHook = null;
    if (getGatewayServer() == null) {
        GatewayServer gatewayServer = null;
        Process pythonProcess = null;
        String tmpDir = null;
        try {
            gatewayServer = startGatewayServer();
            List<String> commands = new ArrayList<>();
            commands.add("-m");
            commands.add("pyflink.pyflink_callback_server");
            tmpDir = System.getProperty("java.io.tmpdir") + File.separator + "pyflink" + File.separator + UUID.randomUUID();
            pythonProcess = launchPy4jPythonClient(gatewayServer, config, commands, null, tmpDir, false);
            entryPoint = (Map<String, Object>) gatewayServer.getGateway().getEntryPoint();
            int i = 0;
            while (!entryPoint.containsKey("PythonFunctionFactory")) {
                if (!pythonProcess.isAlive()) {
                    throw new RuntimeException("Python callback server start failed!");
                }
                try {
                    Thread.sleep(CHECK_INTERVAL);
                } catch (InterruptedException e) {
                    throw new RuntimeException("Interrupted while waiting for the python process to start.", e);
                }
                i++;
                if (i > TIMEOUT_MILLIS / CHECK_INTERVAL) {
                    throw new RuntimeException("Python callback server start failed!");
                }
            }
        } catch (Throwable e) {
            try {
                if (gatewayServer != null) {
                    gatewayServer.shutdown();
                }
            } catch (Throwable e2) {
            // ignore, do not swallow the origin exception.
            }
            try {
                if (pythonProcess != null) {
                    shutdownPythonProcess(pythonProcess, TIMEOUT_MILLIS);
                }
            } catch (Throwable e3) {
            // ignore, do not swallow the origin exception.
            }
            if (tmpDir != null) {
                FileUtils.deleteDirectoryQuietly(new File(tmpDir));
            }
            throw e;
        }
        shutdownHook = new PythonProcessShutdownHook(pythonProcess, gatewayServer, tmpDir);
        Runtime.getRuntime().addShutdownHook(shutdownHook);
    } else {
        entryPoint = (Map<String, Object>) getGatewayServer().getGateway().getEntryPoint();
    }
    return new PythonFunctionFactoryImpl((PythonFunctionFactory) entryPoint.get("PythonFunctionFactory"), shutdownHook);
}

Example 15 with GatewayServer

use of py4j.GatewayServer in project flink by apache.

the class PythonGatewayServer method main.

/**
 * Main method to start a local GatewayServer on a ephemeral port. It tells python side via a
 * file.
 *
 * <p>See: py4j.GatewayServer.main()
 */
public static void main(String[] args) throws IOException, ExecutionException, InterruptedException {
    GatewayServer gatewayServer = PythonEnvUtils.startGatewayServer();
    PythonEnvUtils.setGatewayServer(gatewayServer);
    int boundPort = gatewayServer.getListeningPort();
    Py4JPythonClient callbackClient = gatewayServer.getCallbackClient();
    int callbackPort = callbackClient.getPort();
    if (boundPort == -1) {
        System.out.println("GatewayServer failed to bind; exiting");
        System.exit(1);
    }
    // Tells python side the port of our java rpc server
    String handshakeFilePath = System.getenv("_PYFLINK_CONN_INFO_PATH");
    File handshakeFile = new File(handshakeFilePath);
    File tmpPath = Files.createTempFile(handshakeFile.getParentFile().toPath(), "connection", ".info").toFile();
    FileOutputStream fileOutputStream = new FileOutputStream(tmpPath);
    DataOutputStream stream = new DataOutputStream(fileOutputStream);
    stream.writeInt(boundPort);
    stream.writeInt(callbackPort);
    stream.close();
    fileOutputStream.close();
    if (!tmpPath.renameTo(handshakeFile)) {
        System.out.println("Unable to write connection information to handshake file: " + handshakeFilePath + ", now exit...");
        System.exit(1);
    }
    try {
        // This ensures that the server dies if its parent program dies.
        Map<String, Object> entryPoint = (Map<String, Object>) gatewayServer.getGateway().getEntryPoint();
        for (int i = 0; i < TIMEOUT_MILLIS / CHECK_INTERVAL; i++) {
            if (entryPoint.containsKey("Watchdog")) {
                break;
            }
            Thread.sleep(CHECK_INTERVAL);
        }
        if (!entryPoint.containsKey("Watchdog")) {
            System.out.println("Unable to get the Python watchdog object, now exit.");
            System.exit(1);
        }
        Watchdog watchdog = (Watchdog) entryPoint.get("Watchdog");
        while (watchdog.ping()) {
            Thread.sleep(CHECK_INTERVAL);
        }
        gatewayServer.shutdown();
        System.exit(0);
    } finally {
        System.exit(1);
    }
}