Examples with InterpreterOption org.apache.zeppelin.interpreter.InterpreterOption used on opensource projects

Example 16 with InterpreterOption

use of org.apache.zeppelin.interpreter.InterpreterOption in project zeppelin by apache.

the class ClusterInterpreterLauncherTest method testConnectExistOnlineIntpProcess.

// TODO(zjffdu) disable this test because this is not a correct unit test,
// Actually the interpreter process here never start before ZEPPELIN-5300.
// @Test
public void testConnectExistOnlineIntpProcess() throws IOException {
    mockIntpProcessMeta("intpGroupId", true);
    ClusterInterpreterLauncher launcher = new ClusterInterpreterLauncher(ClusterMockTest.zconf, null);
    Properties properties = new Properties();
    properties.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getVarName(), "5000");
    InterpreterOption option = new InterpreterOption();
    option.setUserImpersonate(true);
    InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "user1", "intpGroupId", "groupId", "groupName", "name", 0, "host");
    InterpreterClient client = launcher.launch(context);
    assertTrue(client instanceof RemoteInterpreterRunningProcess);
    RemoteInterpreterRunningProcess interpreterProcess = (RemoteInterpreterRunningProcess) client;
    assertEquals("127.0.0.1", interpreterProcess.getHost());
    assertEquals("name", interpreterProcess.getInterpreterSettingName());
    assertEquals(5000, interpreterProcess.getConnectTimeout());
}

Example 17 with InterpreterOption

use of org.apache.zeppelin.interpreter.InterpreterOption in project zeppelin by apache.

the class ClusterInterpreterLauncherTest method testConnectExistOfflineIntpProcess.

@Test
public void testConnectExistOfflineIntpProcess() throws IOException {
    mockIntpProcessMeta("intpGroupId2", false);
    ClusterInterpreterLauncher launcher = new ClusterInterpreterLauncher(ClusterMockTest.zconf, null);
    Properties properties = new Properties();
    properties.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getVarName(), "5000");
    InterpreterOption option = new InterpreterOption();
    option.setUserImpersonate(true);
    InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "user1", "intpGroupId2", "groupId", "groupName", "name", 0, "host");
    InterpreterClient client = launcher.launch(context);
    assertTrue(client instanceof ClusterInterpreterProcess);
    ClusterInterpreterProcess interpreterProcess = (ClusterInterpreterProcess) client;
    assertEquals("name", interpreterProcess.getInterpreterSettingName());
    assertEquals(".//interpreter/groupName", interpreterProcess.getInterpreterDir());
    assertEquals(".//local-repo/groupId", interpreterProcess.getLocalRepoDir());
    assertEquals(5000, interpreterProcess.getConnectTimeout());
    assertEquals(zconf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner());
    assertTrue(interpreterProcess.getEnv().size() >= 1);
    assertEquals(true, interpreterProcess.isUserImpersonated());
}

Example 18 with InterpreterOption

use of org.apache.zeppelin.interpreter.InterpreterOption in project zeppelin by apache.

the class DockerInterpreterProcessTest method testCreateIntpProcess.

@Test
public void testCreateIntpProcess() throws IOException {
    DockerInterpreterLauncher launcher = new DockerInterpreterLauncher(zconf, null);
    Properties properties = new Properties();
    properties.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getVarName(), "5000");
    InterpreterOption option = new InterpreterOption();
    InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "user1", "intpGroupId", "groupId", "groupName", "name", 0, "host");
    InterpreterClient client = launcher.launch(context);
    assertTrue(client instanceof DockerInterpreterProcess);
    DockerInterpreterProcess interpreterProcess = (DockerInterpreterProcess) client;
    assertEquals("name", interpreterProcess.getInterpreterSettingName());
    assertEquals(interpreterProcess.CONTAINER_SPARK_HOME, "/spark");
    assertEquals(interpreterProcess.uploadLocalLibToContainter, true);
    assertNotEquals(interpreterProcess.DOCKER_HOST, "http://my-docker-host:2375");
}

Example 19 with InterpreterOption

use of org.apache.zeppelin.interpreter.InterpreterOption in project zeppelin by apache.

the class SparkInterpreterLauncherTest method testYarnClusterMode_1.

@Test
public void testYarnClusterMode_1() throws IOException {
    ZeppelinConfiguration zConf = ZeppelinConfiguration.create();
    SparkInterpreterLauncher launcher = new SparkInterpreterLauncher(zConf, null);
    Properties properties = new Properties();
    properties.setProperty("SPARK_HOME", sparkHome);
    properties.setProperty("property_1", "value_1");
    properties.setProperty("spark.master", "yarn-cluster");
    properties.setProperty("spark.files", "file_1");
    properties.setProperty("spark.jars", "jar_1");
    InterpreterOption option = new InterpreterOption();
    InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "user1", "intpGroupId", "groupId", "spark", "spark", 0, "host");
    InterpreterClient client = launcher.launch(context);
    assertTrue(client instanceof ExecRemoteInterpreterProcess);
    ExecRemoteInterpreterProcess interpreterProcess = (ExecRemoteInterpreterProcess) client;
    assertEquals("spark", interpreterProcess.getInterpreterSettingName());
    assertTrue(interpreterProcess.getInterpreterDir().endsWith("/interpreter/spark"));
    assertTrue(interpreterProcess.getLocalRepoDir().endsWith("/local-repo/groupId"));
    assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner());
    assertTrue(interpreterProcess.getEnv().size() >= 3);
    assertEquals(sparkHome, interpreterProcess.getEnv().get("SPARK_HOME"));
    assertEquals("true", interpreterProcess.getEnv().get("ZEPPELIN_SPARK_YARN_CLUSTER"));
    String sparkJars = "jar_1," + zeppelinHome + "/interpreter/spark/scala-2.11/spark-scala-2.11-" + Util.getVersion() + ".jar," + zeppelinHome + "/interpreter/zeppelin-interpreter-shaded-" + Util.getVersion() + ".jar";
    String sparkrZip = sparkHome + "/R/lib/sparkr.zip#sparkr";
    String sparkFiles = "file_1," + zeppelinHome + "/conf/log4j_yarn_cluster.properties";
    assertEquals("--conf|spark.yarn.dist.archives=" + sparkrZip + "|--conf|spark.yarn.maxAppAttempts=1" + "|--conf|spark.files=" + sparkFiles + "|--conf|spark.jars=" + sparkJars + "|--conf|spark.yarn.isPython=true" + "|--conf|spark.yarn.submit.waitAppCompletion=false" + "|--conf|spark.app.name=intpGroupId" + "|--conf|spark.master=yarn-cluster", interpreterProcess.getEnv().get("ZEPPELIN_SPARK_CONF"));
}

Example 20 with InterpreterOption

use of org.apache.zeppelin.interpreter.InterpreterOption in project zeppelin by apache.

the class SparkInterpreterLauncherTest method testYarnClientMode_1.

@Test
public void testYarnClientMode_1() throws IOException {
    ZeppelinConfiguration zConf = ZeppelinConfiguration.create();
    SparkInterpreterLauncher launcher = new SparkInterpreterLauncher(zConf, null);
    Properties properties = new Properties();
    properties.setProperty("SPARK_HOME", sparkHome);
    properties.setProperty("property_1", "value_1");
    properties.setProperty("spark.master", "yarn-client");
    properties.setProperty("spark.files", "file_1");
    properties.setProperty("spark.jars", "jar_1");
    InterpreterOption option = new InterpreterOption();
    InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "user1", "intpGroupId", "groupId", "spark", "spark", 0, "host");
    InterpreterClient client = launcher.launch(context);
    assertTrue(client instanceof ExecRemoteInterpreterProcess);
    ExecRemoteInterpreterProcess interpreterProcess = (ExecRemoteInterpreterProcess) client;
    assertEquals("spark", interpreterProcess.getInterpreterSettingName());
    assertTrue(interpreterProcess.getInterpreterDir().endsWith("/interpreter/spark"));
    assertTrue(interpreterProcess.getLocalRepoDir().endsWith("/local-repo/groupId"));
    assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner());
    assertTrue(interpreterProcess.getEnv().size() >= 2);
    assertEquals(sparkHome, interpreterProcess.getEnv().get("SPARK_HOME"));
    String sparkJars = "jar_1";
    String sparkrZip = sparkHome + "/R/lib/sparkr.zip#sparkr";
    String sparkFiles = "file_1";
    assertEquals("--conf|spark.yarn.dist.archives=" + sparkrZip + "|--conf|spark.files=" + sparkFiles + "|--conf|spark.jars=" + sparkJars + "|--conf|spark.yarn.isPython=true|--conf|spark.app.name=intpGroupId|--conf|spark.master=yarn-client", interpreterProcess.getEnv().get("ZEPPELIN_SPARK_CONF"));
}