Examples with RemoteInterpreterRunningProcess org.apache.zeppelin.interpreter.remote.RemoteInterpreterRunningProcess used on opensource projects

Example 1 with RemoteInterpreterRunningProcess

use of org.apache.zeppelin.interpreter.remote.RemoteInterpreterRunningProcess in project zeppelin by apache.

the class ClusterInterpreterLauncher method launchDirectly.

@Override
public InterpreterClient launchDirectly(InterpreterLaunchContext context) throws IOException {
    LOGGER.info("Launching Interpreter: " + context.getInterpreterSettingGroup());
    this.context = context;
    this.properties = context.getProperties();
    int connectTimeout = getConnectTimeout();
    String intpGroupId = context.getInterpreterGroupId();
    // connect exist Interpreter Process
    InterpreterClient intpClient = clusterServer.getIntpProcessStatus(intpGroupId, 3000, new ClusterCallback<HashMap<String, Object>>() {

        @Override
        public InterpreterClient online(HashMap<String, Object> result) {
            String intpTserverHost = (String) result.get(INTP_TSERVER_HOST);
            int intpTserverPort = (int) result.get(INTP_TSERVER_PORT);
            return new RemoteInterpreterRunningProcess(context.getInterpreterSettingName(), context.getInterpreterGroupId(), connectTimeout, getConnectPoolSize(), context.getIntpEventServerHost(), context.getIntpEventServerPort(), intpTserverHost, intpTserverPort, false);
        }

        @Override
        public void offline() {
            LOGGER.info("interpreter {} is not exist!", intpGroupId);
        }
    });
    if (null != intpClient) {
        return intpClient;
    }
    // No process was found for the InterpreterGroup ID
    String srvHost = null;
    int srvPort = 0;
    HashMap<String, Object> meta = clusterServer.getIdleNodeMeta();
    if (null == meta) {
        LOGGER.error("Don't get idle node meta, launch interpreter on local.");
        InterpreterClient clusterIntpProcess = createInterpreterProcess(context);
        try {
            clusterIntpProcess.start(context.getUserName());
        } catch (IOException e) {
            LOGGER.error(e.getMessage(), e);
            return clusterIntpProcess;
        }
    } else {
        srvHost = (String) meta.get(SERVER_HOST);
        String localhost = RemoteInterpreterUtils.findAvailableHostAddress();
        if (localhost.equalsIgnoreCase(srvHost)) {
            LOGGER.info("launch interpreter on local");
            InterpreterClient clusterIntpProcess = createInterpreterProcess(context);
            try {
                clusterIntpProcess.start(context.getUserName());
            } catch (IOException e) {
                LOGGER.error(e.getMessage(), e);
                return clusterIntpProcess;
            }
        } else {
            // launch interpreter on cluster
            srvPort = (int) meta.get(SERVER_PORT);
            Gson gson = new Gson();
            String sContext = gson.toJson(context);
            Map<String, Object> mapEvent = new HashMap<>();
            mapEvent.put(CLUSTER_EVENT, CREATE_INTP_PROCESS);
            mapEvent.put(CLUSTER_EVENT_MSG, sContext);
            String strEvent = gson.toJson(mapEvent);
            // Notify other server in the cluster that the resource is idle to create an interpreter
            clusterServer.unicastClusterEvent(srvHost, srvPort, ClusterManagerServer.CLUSTER_INTP_EVENT_TOPIC, strEvent);
        }
    }
    // Find the ip and port of thrift registered by the remote interpreter process
    // through the cluster metadata
    String finalSrvHost = srvHost;
    int finalSrvPort = srvPort;
    intpClient = clusterServer.getIntpProcessStatus(intpGroupId, connectTimeout, new ClusterCallback<HashMap<String, Object>>() {

        @Override
        public InterpreterClient online(HashMap<String, Object> result) {
            // connect exist Interpreter Process
            String intpTserverHost = (String) result.get(INTP_TSERVER_HOST);
            int intpTserverPort = (int) result.get(INTP_TSERVER_PORT);
            return new RemoteInterpreterRunningProcess(context.getInterpreterSettingName(), context.getInterpreterGroupId(), connectTimeout, getConnectPoolSize(), context.getIntpEventServerHost(), context.getIntpEventServerPort(), intpTserverHost, intpTserverPort, false);
        }

        @Override
        public void offline() {
            String errorInfo = String.format("Creating process %s failed on remote server %s:%d", intpGroupId, finalSrvHost, finalSrvPort);
            LOGGER.error(errorInfo);
        }
    });
    if (null == intpClient) {
        String errorInfo = String.format("Creating process %s failed on remote server %s:%d", intpGroupId, srvHost, srvPort);
        throw new IOException(errorInfo);
    } else {
        return intpClient;
    }
}

Example 2 with RemoteInterpreterRunningProcess

use of org.apache.zeppelin.interpreter.remote.RemoteInterpreterRunningProcess in project zeppelin by apache.

the class RecoveryUtils method restoreFromRecoveryData.

/**
 * Return interpreterClient from recoveryData of one interpreterSetting.
 *
 * @param recoveryData
 * @param interpreterSettingName
 * @param interpreterSettingManager
 * @param zConf
 * @return
 */
public static Map<String, InterpreterClient> restoreFromRecoveryData(String recoveryData, String interpreterSettingName, InterpreterSettingManager interpreterSettingManager, ZeppelinConfiguration zConf) {
    int connectTimeout = zConf.getInt(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT);
    Properties interpreterProperties = interpreterSettingManager.getByName(interpreterSettingName).getJavaProperties();
    int connectionPoolSize = Integer.parseInt(interpreterProperties.getProperty(ZEPPELIN_INTERPRETER_CONNECTION_POOL_SIZE.getVarName(), ZEPPELIN_INTERPRETER_CONNECTION_POOL_SIZE.getIntValue() + ""));
    Map<String, InterpreterClient> clients = new HashMap<>();
    if (!StringUtils.isBlank(recoveryData)) {
        for (String line : recoveryData.split(System.lineSeparator())) {
            String[] tokens = line.split("\t");
            String interpreterGroupId = tokens[0];
            String[] hostPort = tokens[1].split(":");
            RemoteInterpreterRunningProcess client = new RemoteInterpreterRunningProcess(interpreterSettingName, interpreterGroupId, connectTimeout, connectionPoolSize, interpreterSettingManager.getInterpreterEventServer().getHost(), interpreterSettingManager.getInterpreterEventServer().getPort(), hostPort[0], Integer.parseInt(hostPort[1]), true);
            clients.put(interpreterGroupId, client);
            LOGGER.info("Recovering Interpreter Process: " + interpreterGroupId + ", " + hostPort[0] + ":" + hostPort[1]);
        }
    }
    return clients;
}

Example 3 with RemoteInterpreterRunningProcess

use of org.apache.zeppelin.interpreter.remote.RemoteInterpreterRunningProcess in project zeppelin by apache.

the class StandardInterpreterLauncher method launchDirectly.

@Override
public InterpreterClient launchDirectly(InterpreterLaunchContext context) throws IOException {
    LOGGER.info("Launching new interpreter process of {}", context.getInterpreterSettingGroup());
    this.properties = context.getProperties();
    InterpreterOption option = context.getOption();
    InterpreterRunner runner = context.getRunner();
    String groupName = context.getInterpreterSettingGroup();
    String name = context.getInterpreterSettingName();
    int connectTimeout = getConnectTimeout();
    int connectionPoolSize = getConnectPoolSize();
    if (option.isExistingProcess()) {
        return new RemoteInterpreterRunningProcess(context.getInterpreterSettingName(), context.getInterpreterGroupId(), connectTimeout, connectionPoolSize, context.getIntpEventServerHost(), context.getIntpEventServerPort(), option.getHost(), option.getPort(), false);
    } else {
        // create new remote process
        String localRepoPath = zConf.getInterpreterLocalRepoPath() + File.separator + context.getInterpreterSettingId();
        return new ExecRemoteInterpreterProcess(context.getIntpEventServerPort(), context.getIntpEventServerHost(), zConf.getInterpreterPortRange(), zConf.getInterpreterDir() + "/" + groupName, localRepoPath, buildEnvFromProperties(context), connectTimeout, connectionPoolSize, name, context.getInterpreterGroupId(), option.isUserImpersonate(), runner != null ? runner.getPath() : zConf.getInterpreterRemoteRunnerPath());
    }
}

Example 4 with RemoteInterpreterRunningProcess

use of org.apache.zeppelin.interpreter.remote.RemoteInterpreterRunningProcess in project zeppelin by apache.

the class ClusterInterpreterLauncherTest method testConnectExistOnlineIntpProcess.

// TODO(zjffdu) disable this test because this is not a correct unit test,
// Actually the interpreter process here never start before ZEPPELIN-5300.
// @Test
public void testConnectExistOnlineIntpProcess() throws IOException {
    mockIntpProcessMeta("intpGroupId", true);
    ClusterInterpreterLauncher launcher = new ClusterInterpreterLauncher(ClusterMockTest.zconf, null);
    Properties properties = new Properties();
    properties.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getVarName(), "5000");
    InterpreterOption option = new InterpreterOption();
    option.setUserImpersonate(true);
    InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "user1", "intpGroupId", "groupId", "groupName", "name", 0, "host");
    InterpreterClient client = launcher.launch(context);
    assertTrue(client instanceof RemoteInterpreterRunningProcess);
    RemoteInterpreterRunningProcess interpreterProcess = (RemoteInterpreterRunningProcess) client;
    assertEquals("127.0.0.1", interpreterProcess.getHost());
    assertEquals("name", interpreterProcess.getInterpreterSettingName());
    assertEquals(5000, interpreterProcess.getConnectTimeout());
}