Examples with CallbackClient py4j.CallbackClient used on opensource projects

Example 1 with CallbackClient

use of py4j.CallbackClient in project cdap by caskdata.

the class SparkPythonUtil method setGatewayCallbackPort.

/**
 * Updates the python callback port in the {@link GatewayServer}.
 */
public static void setGatewayCallbackPort(GatewayServer gatewayServer, int port) {
    CallbackClient callbackClient = gatewayServer.getCallbackClient();
    InetAddress address = callbackClient.getAddress();
    callbackClient.shutdown();
    Class<? extends GatewayServer> gatewayServerClass = gatewayServer.getClass();
    try {
        callbackClient = new CallbackClient(port, address);
        Field cbClientField = gatewayServerClass.getDeclaredField("cbClient");
        cbClientField.setAccessible(true);
        cbClientField.set(gatewayServer, callbackClient);
        Field gatewayField = gatewayServerClass.getDeclaredField("gateway");
        gatewayField.setAccessible(true);
        Object gateway = gatewayField.get(gatewayServer);
        Field gatewayCbClientField = gateway.getClass().getDeclaredField("cbClient");
        gatewayCbClientField.setAccessible(true);
        gatewayCbClientField.set(gateway, callbackClient);
        Field pythonPortField = gatewayServerClass.getDeclaredField("pythonPort");
        pythonPortField.setAccessible(true);
        pythonPortField.set(gatewayServer, port);
    } catch (Exception e) {
        LOG.warn("Failed to update python gateway callback port. Callback into Python may not work.", e);
    }
}

Example 2 with CallbackClient

use of py4j.CallbackClient in project flink by apache.

the class PythonEnvUtils method startGatewayServer.

/**
 * Creates a GatewayServer run in a daemon thread.
 *
 * @return The created GatewayServer
 */
static GatewayServer startGatewayServer() throws ExecutionException, InterruptedException {
    CompletableFuture<GatewayServer> gatewayServerFuture = new CompletableFuture<>();
    Thread thread = new Thread(() -> {
        try (NetUtils.Port port = NetUtils.getAvailablePort()) {
            int freePort = port.getPort();
            GatewayServer server = new GatewayServer.GatewayServerBuilder().gateway(new Gateway(new ConcurrentHashMap<String, Object>(), new CallbackClient(freePort))).javaPort(0).build();
            resetCallbackClientExecutorService(server);
            gatewayServerFuture.complete(server);
            server.start(true);
        } catch (Throwable e) {
            gatewayServerFuture.completeExceptionally(e);
        }
    });
    thread.setName("py4j-gateway");
    thread.setDaemon(true);
    thread.start();
    thread.join();
    return gatewayServerFuture.get();
}

Example 3 with CallbackClient

use of py4j.CallbackClient in project flink by apache.

the class PythonEnvUtils method resetCallbackClientExecutorService.

/**
 * Reset a daemon thread to the callback client thread pool so that the callback server can be
 * terminated when gate way server is shutting down. We need to shut down the none-daemon thread
 * firstly, then set a new thread created in a daemon thread to the ExecutorService.
 *
 * @param gatewayServer the gateway which creates the callback server.
 */
private static void resetCallbackClientExecutorService(GatewayServer gatewayServer) throws NoSuchFieldException, IllegalAccessException, NoSuchMethodException, InvocationTargetException {
    CallbackClient callbackClient = (CallbackClient) gatewayServer.getCallbackClient();
    // The Java API of py4j does not provide approach to set "daemonize_connections" parameter.
    // Use reflect to daemonize the connection thread.
    Field executor = CallbackClient.class.getDeclaredField("executor");
    executor.setAccessible(true);
    ((ScheduledExecutorService) executor.get(callbackClient)).shutdown();
    executor.set(callbackClient, Executors.newScheduledThreadPool(1, Thread::new));
    Method setupCleaner = CallbackClient.class.getDeclaredMethod("setupCleaner");
    setupCleaner.setAccessible(true);
    setupCleaner.invoke(callbackClient);
}