Examples with InterpreterOption org.apache.zeppelin.interpreter.InterpreterOption used on opensource projects

Example 1 with InterpreterOption

use of org.apache.zeppelin.interpreter.InterpreterOption in project zeppelin by apache.

the class NoteInterpreterLoaderTest method setUp.

@Before
public void setUp() throws Exception {
    tmpDir = new File(System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis());
    tmpDir.mkdirs();
    new File(tmpDir, "conf").mkdirs();
    System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), tmpDir.getAbsolutePath());
    conf = ZeppelinConfiguration.create();
    depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo");
    interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
    factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
    ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
    interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, Maps.<String, Object>newHashMap()));
    interpreterInfos.add(new InterpreterInfo(MockInterpreter11.class.getName(), "mock11", false, Maps.<String, Object>newHashMap()));
    ArrayList<InterpreterInfo> interpreterInfos2 = new ArrayList<>();
    interpreterInfos2.add(new InterpreterInfo(MockInterpreter2.class.getName(), "mock2", true, Maps.<String, Object>newHashMap()));
    interpreterSettingManager.add("group1", interpreterInfos, Lists.<Dependency>newArrayList(), new InterpreterOption(), Maps.<String, InterpreterProperty>newHashMap(), "mock", null);
    interpreterSettingManager.add("group2", interpreterInfos2, Lists.<Dependency>newArrayList(), new InterpreterOption(), Maps.<String, InterpreterProperty>newHashMap(), "mock", null);
    interpreterSettingManager.createNewSetting("group1", "group1", Lists.<Dependency>newArrayList(), new InterpreterOption(), new Properties());
    interpreterSettingManager.createNewSetting("group2", "group2", Lists.<Dependency>newArrayList(), new InterpreterOption(), new Properties());
}

Example 2 with InterpreterOption

use of org.apache.zeppelin.interpreter.InterpreterOption in project zeppelin by apache.

the class ClusterInterpreterLauncher method createClusterIntpProcess.

private RemoteInterpreterProcess createClusterIntpProcess() {
    ClusterInterpreterProcess clusterIntpProcess = null;
    try {
        InterpreterOption option = context.getOption();
        InterpreterRunner runner = context.getRunner();
        String intpSetGroupName = context.getInterpreterSettingGroup();
        String intpSetName = context.getInterpreterSettingName();
        int connectTimeout = getConnectTimeout();
        int connectionPoolSize = getConnectPoolSize();
        String localRepoPath = zConf.getInterpreterLocalRepoPath() + "/" + context.getInterpreterSettingId();
        clusterIntpProcess = new ClusterInterpreterProcess(runner != null ? runner.getPath() : zConf.getInterpreterRemoteRunnerPath(), context.getIntpEventServerPort(), context.getIntpEventServerHost(), zConf.getInterpreterPortRange(), zConf.getInterpreterDir() + "/" + intpSetGroupName, localRepoPath, buildEnvFromProperties(context), connectTimeout, connectionPoolSize, intpSetName, context.getInterpreterGroupId(), option.isUserImpersonate());
    } catch (IOException e) {
        LOGGER.error(e.getMessage(), e);
    }
    return clusterIntpProcess;
}

Example 3 with InterpreterOption

use of org.apache.zeppelin.interpreter.InterpreterOption in project zeppelin by apache.

the class SparkInterpreterLauncherTest method testConnectTimeOut.

@Test
public void testConnectTimeOut() throws IOException {
    ZeppelinConfiguration zConf = ZeppelinConfiguration.create();
    SparkInterpreterLauncher launcher = new SparkInterpreterLauncher(zConf, null);
    Properties properties = new Properties();
    properties.setProperty("SPARK_HOME", sparkHome);
    properties.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getVarName(), "10000");
    InterpreterOption option = new InterpreterOption();
    option.setUserImpersonate(true);
    InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "user1", "intpGroupId", "groupId", "groupName", "name", 0, "host");
    InterpreterClient client = launcher.launch(context);
    assertTrue(client instanceof ExecRemoteInterpreterProcess);
    ExecRemoteInterpreterProcess interpreterProcess = (ExecRemoteInterpreterProcess) client;
    assertEquals("name", interpreterProcess.getInterpreterSettingName());
    assertEquals(zeppelinHome + "/interpreter/groupName", interpreterProcess.getInterpreterDir());
    assertEquals(zeppelinHome + "/local-repo/groupId", interpreterProcess.getLocalRepoDir());
    assertEquals(10000, interpreterProcess.getConnectTimeout());
    assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner());
    assertTrue(interpreterProcess.getEnv().size() >= 2);
    assertEquals(true, interpreterProcess.isUserImpersonated());
}

Example 4 with InterpreterOption

use of org.apache.zeppelin.interpreter.InterpreterOption in project zeppelin by apache.

the class SparkInterpreterLauncherTest method testYarnClusterMode_2.

@Test
public void testYarnClusterMode_2() throws IOException {
    ZeppelinConfiguration zConf = ZeppelinConfiguration.create();
    SparkInterpreterLauncher launcher = new SparkInterpreterLauncher(zConf, null);
    Properties properties = new Properties();
    properties.setProperty("SPARK_HOME", sparkHome);
    properties.setProperty("property_1", "value_1");
    properties.setProperty("spark.master", "yarn");
    properties.setProperty("spark.submit.deployMode", "cluster");
    properties.setProperty("spark.files", "file_1");
    properties.setProperty("spark.jars", "jar_1");
    InterpreterOption option = new InterpreterOption();
    option.setUserImpersonate(true);
    InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "user1", "intpGroupId", "groupId", "spark", "spark", 0, "host");
    Path localRepoPath = Paths.get(zConf.getInterpreterLocalRepoPath(), context.getInterpreterSettingId());
    FileUtils.deleteDirectory(localRepoPath.toFile());
    Files.createDirectories(localRepoPath);
    Files.createFile(Paths.get(localRepoPath.toAbsolutePath().toString(), "test.jar"));
    InterpreterClient client = launcher.launch(context);
    assertTrue(client instanceof ExecRemoteInterpreterProcess);
    ExecRemoteInterpreterProcess interpreterProcess = (ExecRemoteInterpreterProcess) client;
    assertEquals("spark", interpreterProcess.getInterpreterSettingName());
    assertTrue(interpreterProcess.getInterpreterDir().endsWith("/interpreter/spark"));
    assertTrue(interpreterProcess.getLocalRepoDir().endsWith("/local-repo/groupId"));
    assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner());
    assertTrue(interpreterProcess.getEnv().size() >= 3);
    assertEquals(sparkHome, interpreterProcess.getEnv().get("SPARK_HOME"));
    assertEquals("true", interpreterProcess.getEnv().get("ZEPPELIN_SPARK_YARN_CLUSTER"));
    String sparkJars = "jar_1," + Paths.get(localRepoPath.toAbsolutePath().toString(), "test.jar").toString() + "," + zeppelinHome + "/interpreter/spark/scala-2.11/spark-scala-2.11-" + Util.getVersion() + ".jar," + zeppelinHome + "/interpreter/zeppelin-interpreter-shaded-" + Util.getVersion() + ".jar";
    String sparkrZip = sparkHome + "/R/lib/sparkr.zip#sparkr";
    String sparkFiles = "file_1," + zeppelinHome + "/conf/log4j_yarn_cluster.properties";
    assertEquals("--proxy-user|user1|--conf|spark.yarn.dist.archives=" + sparkrZip + "|--conf|spark.yarn.isPython=true|--conf|spark.app.name=intpGroupId" + "|--conf|spark.yarn.maxAppAttempts=1" + "|--conf|spark.master=yarn" + "|--conf|spark.files=" + sparkFiles + "|--conf|spark.jars=" + sparkJars + "|--conf|spark.submit.deployMode=cluster" + "|--conf|spark.yarn.submit.waitAppCompletion=false", interpreterProcess.getEnv().get("ZEPPELIN_SPARK_CONF"));
    Files.deleteIfExists(Paths.get(localRepoPath.toAbsolutePath().toString(), "test.jar"));
    FileUtils.deleteDirectory(localRepoPath.toFile());
}

Example 5 with InterpreterOption

use of org.apache.zeppelin.interpreter.InterpreterOption in project zeppelin by apache.

the class StandardInterpreterLauncherTest method testConnectTimeOut.

@Test
public void testConnectTimeOut() throws IOException {
    ZeppelinConfiguration zConf = ZeppelinConfiguration.create();
    StandardInterpreterLauncher launcher = new StandardInterpreterLauncher(zConf, null);
    Properties properties = new Properties();
    properties.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getVarName(), "10000");
    InterpreterOption option = new InterpreterOption();
    option.setUserImpersonate(true);
    InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "user1", "intpGroupId", "groupId", "groupName", "name", 0, "host");
    InterpreterClient client = launcher.launch(context);
    assertTrue(client instanceof ExecRemoteInterpreterProcess);
    ExecRemoteInterpreterProcess interpreterProcess = (ExecRemoteInterpreterProcess) client;
    assertEquals("name", interpreterProcess.getInterpreterSettingName());
    assertEquals(".//interpreter/groupName", interpreterProcess.getInterpreterDir());
    assertEquals(".//local-repo/groupId", interpreterProcess.getLocalRepoDir());
    assertEquals(10000, interpreterProcess.getConnectTimeout());
    assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner());
    assertTrue(interpreterProcess.getEnv().size() >= 1);
    assertTrue(interpreterProcess.getEnv().containsKey("INTERPRETER_GROUP_ID"));
    assertEquals(true, interpreterProcess.isUserImpersonated());
}