Examples with InterpreterOption org.apache.zeppelin.interpreter.InterpreterOption used on opensource projects

Example 21 with InterpreterOption

use of org.apache.zeppelin.interpreter.InterpreterOption in project zeppelin by apache.

the class SparkInterpreterLauncherTest method testYarnClusterMode_3.

@Test
public void testYarnClusterMode_3() throws IOException {
    ZeppelinConfiguration zConf = ZeppelinConfiguration.create();
    SparkInterpreterLauncher launcher = new SparkInterpreterLauncher(zConf, null);
    Properties properties = new Properties();
    properties.setProperty("SPARK_HOME", sparkHome);
    properties.setProperty("property_1", "value_1");
    properties.setProperty("spark.master", "yarn");
    properties.setProperty("spark.submit.deployMode", "cluster");
    properties.setProperty("spark.files", "{}");
    properties.setProperty("spark.jars", "jar_1");
    InterpreterOption option = new InterpreterOption();
    option.setUserImpersonate(true);
    InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "user1", "intpGroupId", "groupId", "spark", "spark", 0, "host");
    Path localRepoPath = Paths.get(zConf.getInterpreterLocalRepoPath(), context.getInterpreterSettingId());
    FileUtils.deleteDirectory(localRepoPath.toFile());
    Files.createDirectories(localRepoPath);
    InterpreterClient client = launcher.launch(context);
    assertTrue(client instanceof ExecRemoteInterpreterProcess);
    ExecRemoteInterpreterProcess interpreterProcess = (ExecRemoteInterpreterProcess) client;
    assertEquals("spark", interpreterProcess.getInterpreterSettingName());
    assertTrue(interpreterProcess.getInterpreterDir().endsWith("/interpreter/spark"));
    assertTrue(interpreterProcess.getLocalRepoDir().endsWith("/local-repo/groupId"));
    assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner());
    assertTrue(interpreterProcess.getEnv().size() >= 3);
    assertEquals(sparkHome, interpreterProcess.getEnv().get("SPARK_HOME"));
    assertEquals("true", interpreterProcess.getEnv().get("ZEPPELIN_SPARK_YARN_CLUSTER"));
    String sparkJars = "jar_1," + zeppelinHome + "/interpreter/spark/scala-2.11/spark-scala-2.11-" + Util.getVersion() + ".jar," + zeppelinHome + "/interpreter/zeppelin-interpreter-shaded-" + Util.getVersion() + ".jar";
    String sparkrZip = sparkHome + "/R/lib/sparkr.zip#sparkr";
    // escape special characters
    String sparkFiles = "{}," + zeppelinHome + "/conf/log4j_yarn_cluster.properties";
    assertEquals("--proxy-user|user1" + "|--conf|spark.yarn.dist.archives=" + sparkrZip + "|--conf|spark.yarn.isPython=true" + "|--conf|spark.app.name=intpGroupId" + "|--conf|spark.yarn.maxAppAttempts=1" + "|--conf|spark.master=yarn" + "|--conf|spark.files=" + sparkFiles + "|--conf|spark.jars=" + sparkJars + "|--conf|spark.submit.deployMode=cluster" + "|--conf|spark.yarn.submit.waitAppCompletion=false", interpreterProcess.getEnv().get("ZEPPELIN_SPARK_CONF"));
    FileUtils.deleteDirectory(localRepoPath.toFile());
}

Example 22 with InterpreterOption

use of org.apache.zeppelin.interpreter.InterpreterOption in project zeppelin by apache.

the class SparkInterpreterLauncherTest method testYarnClientMode_2.

@Test
public void testYarnClientMode_2() throws IOException {
    ZeppelinConfiguration zConf = ZeppelinConfiguration.create();
    SparkInterpreterLauncher launcher = new SparkInterpreterLauncher(zConf, null);
    Properties properties = new Properties();
    properties.setProperty("SPARK_HOME", sparkHome);
    properties.setProperty("property_1", "value_1");
    properties.setProperty("spark.master", "yarn");
    properties.setProperty("spark.submit.deployMode", "client");
    properties.setProperty("spark.files", "file_1");
    properties.setProperty("spark.jars", "jar_1");
    InterpreterOption option = new InterpreterOption();
    InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "user1", "intpGroupId", "groupId", "spark", "spark", 0, "host");
    InterpreterClient client = launcher.launch(context);
    assertTrue(client instanceof ExecRemoteInterpreterProcess);
    ExecRemoteInterpreterProcess interpreterProcess = (ExecRemoteInterpreterProcess) client;
    assertEquals("spark", interpreterProcess.getInterpreterSettingName());
    assertTrue(interpreterProcess.getInterpreterDir().endsWith("/interpreter/spark"));
    assertTrue(interpreterProcess.getLocalRepoDir().endsWith("/local-repo/groupId"));
    assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner());
    assertTrue(interpreterProcess.getEnv().size() >= 2);
    assertEquals(sparkHome, interpreterProcess.getEnv().get("SPARK_HOME"));
    String sparkJars = "jar_1";
    String sparkrZip = sparkHome + "/R/lib/sparkr.zip#sparkr";
    String sparkFiles = "file_1";
    assertEquals("--conf|spark.yarn.dist.archives=" + sparkrZip + "|--conf|spark.files=" + sparkFiles + "|--conf|spark.jars=" + sparkJars + "|--conf|spark.submit.deployMode=client" + "|--conf|spark.yarn.isPython=true|--conf|spark.app.name=intpGroupId|--conf|spark.master=yarn", interpreterProcess.getEnv().get("ZEPPELIN_SPARK_CONF"));
}

Example 23 with InterpreterOption

use of org.apache.zeppelin.interpreter.InterpreterOption in project zeppelin by apache.

the class SparkInterpreterLauncherTest method testLocalMode.

@Test
public void testLocalMode() throws IOException {
    ZeppelinConfiguration zConf = ZeppelinConfiguration.create();
    SparkInterpreterLauncher launcher = new SparkInterpreterLauncher(zConf, null);
    Properties properties = new Properties();
    properties.setProperty("SPARK_HOME", sparkHome);
    properties.setProperty("ENV_1", "");
    properties.setProperty("property_1", "value_1");
    properties.setProperty("spark.master", "local[*]");
    properties.setProperty("spark.files", "file_1");
    properties.setProperty("spark.jars", "jar_1");
    InterpreterOption option = new InterpreterOption();
    InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "user1", "intpGroupId", "groupId", "spark", "spark", 0, "host");
    InterpreterClient client = launcher.launch(context);
    assertTrue(client instanceof ExecRemoteInterpreterProcess);
    ExecRemoteInterpreterProcess interpreterProcess = (ExecRemoteInterpreterProcess) client;
    assertEquals("spark", interpreterProcess.getInterpreterSettingName());
    assertTrue(interpreterProcess.getInterpreterDir().endsWith("/interpreter/spark"));
    assertTrue(interpreterProcess.getLocalRepoDir().endsWith("/local-repo/groupId"));
    assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner());
    assertTrue(interpreterProcess.getEnv().size() >= 2);
    assertEquals(sparkHome, interpreterProcess.getEnv().get("SPARK_HOME"));
    assertFalse(interpreterProcess.getEnv().containsKey("ENV_1"));
    assertEquals("--conf|spark.files=file_1" + "|--conf|spark.jars=jar_1|--conf|spark.app.name=intpGroupId|--conf|spark.master=local[*]", interpreterProcess.getEnv().get("ZEPPELIN_SPARK_CONF"));
}

Example 24 with InterpreterOption

use of org.apache.zeppelin.interpreter.InterpreterOption in project zeppelin by apache.

the class StandardInterpreterLauncherTest method testLauncher.

@Test
public void testLauncher() throws IOException {
    ZeppelinConfiguration zConf = ZeppelinConfiguration.create();
    StandardInterpreterLauncher launcher = new StandardInterpreterLauncher(zConf, null);
    Properties properties = new Properties();
    properties.setProperty("ENV_1", "VALUE_1");
    properties.setProperty("property_1", "value_1");
    InterpreterOption option = new InterpreterOption();
    option.setUserImpersonate(true);
    InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "user1", "intpGroupId", "groupId", "groupName", "name", 0, "host");
    InterpreterClient client = launcher.launch(context);
    assertTrue(client instanceof ExecRemoteInterpreterProcess);
    ExecRemoteInterpreterProcess interpreterProcess = (ExecRemoteInterpreterProcess) client;
    assertEquals("name", interpreterProcess.getInterpreterSettingName());
    assertEquals(".//interpreter/groupName", interpreterProcess.getInterpreterDir());
    assertEquals(".//local-repo/groupId", interpreterProcess.getLocalRepoDir());
    assertEquals(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getIntValue(), interpreterProcess.getConnectTimeout());
    assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner());
    assertTrue(interpreterProcess.getEnv().size() >= 2);
    assertEquals("VALUE_1", interpreterProcess.getEnv().get("ENV_1"));
    assertTrue(interpreterProcess.getEnv().containsKey("INTERPRETER_GROUP_ID"));
    assertEquals(true, interpreterProcess.isUserImpersonated());
}

Example 25 with InterpreterOption

use of org.apache.zeppelin.interpreter.InterpreterOption in project SSM by Intel-bigdata.

the class NoteInterpreterLoaderTest method setUp.

@Before
public void setUp() throws Exception {
    tmpDir = new File(System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis());
    tmpDir.mkdirs();
    new File(tmpDir, "conf").mkdirs();
    System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), tmpDir.getAbsolutePath());
    conf = ZeppelinConfiguration.create();
    depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo");
    interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(false));
    factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
    ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
    interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, Maps.<String, Object>newHashMap()));
    interpreterInfos.add(new InterpreterInfo(MockInterpreter11.class.getName(), "mock11", false, Maps.<String, Object>newHashMap()));
    ArrayList<InterpreterInfo> interpreterInfos2 = new ArrayList<>();
    interpreterInfos2.add(new InterpreterInfo(MockInterpreter2.class.getName(), "mock2", true, Maps.<String, Object>newHashMap()));
    interpreterSettingManager.add("group1", interpreterInfos, Lists.<Dependency>newArrayList(), new InterpreterOption(), Maps.<String, InterpreterProperty>newHashMap(), "mock", null);
    interpreterSettingManager.add("group2", interpreterInfos2, Lists.<Dependency>newArrayList(), new InterpreterOption(), Maps.<String, InterpreterProperty>newHashMap(), "mock", null);
    interpreterSettingManager.createNewSetting("group1", "group1", Lists.<Dependency>newArrayList(), new InterpreterOption(), new Properties());
    interpreterSettingManager.createNewSetting("group2", "group2", Lists.<Dependency>newArrayList(), new InterpreterOption(), new Properties());
}