Examples with InterpreterSettingManager org.apache.zeppelin.interpreter.InterpreterSettingManager used on opensource projects

Example 11 with InterpreterSettingManager

use of org.apache.zeppelin.interpreter.InterpreterSettingManager in project zeppelin by apache.

the class VFSNotebookRepoTest method setUp.

@Before
public void setUp() throws Exception {
    String zpath = System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis();
    mainZepDir = new File(zpath);
    mainZepDir.mkdirs();
    new File(mainZepDir, "conf").mkdirs();
    String mainNotePath = zpath + "/notebook";
    mainNotebookDir = new File(mainNotePath);
    mainNotebookDir.mkdirs();
    System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), mainZepDir.getAbsolutePath());
    System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath());
    System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), "org.apache.zeppelin.notebook.repo.VFSNotebookRepo");
    conf = ZeppelinConfiguration.create();
    this.schedulerFactory = new SchedulerFactory();
    this.schedulerFactory = new SchedulerFactory();
    depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo");
    interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
    factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
    ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
    interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap<String, Object>()));
    interpreterSettingManager.add("mock1", interpreterInfos, new ArrayList<Dependency>(), new InterpreterOption(), Maps.<String, InterpreterProperty>newHashMap(), "mock1", null);
    interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList<Dependency>(), new InterpreterOption(), new Properties());
    SearchService search = mock(SearchService.class);
    notebookRepo = new VFSNotebookRepo(conf);
    notebookAuthorization = NotebookAuthorization.init(conf);
    notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, interpreterSettingManager, this, search, notebookAuthorization, null);
}

Example 12 with InterpreterSettingManager

use of org.apache.zeppelin.interpreter.InterpreterSettingManager in project zeppelin by apache.

the class ZeppelinServer method setupClusterManagerServer.

private static void setupClusterManagerServer(ServiceLocator serviceLocator) {
    if (conf.isClusterMode()) {
        LOG.info("Cluster mode is enabled, starting ClusterManagerServer");
        ClusterManagerServer clusterManagerServer = ClusterManagerServer.getInstance(conf);
        NotebookServer notebookServer = serviceLocator.getService(NotebookServer.class);
        clusterManagerServer.addClusterEventListeners(ClusterManagerServer.CLUSTER_NOTE_EVENT_TOPIC, notebookServer);
        AuthorizationService authorizationService = serviceLocator.getService(AuthorizationService.class);
        clusterManagerServer.addClusterEventListeners(ClusterManagerServer.CLUSTER_AUTH_EVENT_TOPIC, authorizationService);
        InterpreterSettingManager interpreterSettingManager = serviceLocator.getService(InterpreterSettingManager.class);
        clusterManagerServer.addClusterEventListeners(ClusterManagerServer.CLUSTER_INTP_SETTING_EVENT_TOPIC, interpreterSettingManager);
        // This allows the ClusterInterpreterLauncher to listen for cluster events.
        try {
            InterpreterSettingManager intpSettingManager = sharedServiceLocator.getService(InterpreterSettingManager.class);
            RecoveryStorage recoveryStorage = ReflectionUtils.createClazzInstance(conf.getRecoveryStorageClass(), new Class[] { ZeppelinConfiguration.class, InterpreterSettingManager.class }, new Object[] { conf, intpSettingManager });
            recoveryStorage.init();
            PluginManager.get().loadInterpreterLauncher(InterpreterSetting.CLUSTER_INTERPRETER_LAUNCHER_NAME, recoveryStorage);
        } catch (IOException e) {
            LOG.error(e.getMessage(), e);
        }
        clusterManagerServer.start();
    } else {
        LOG.info("Cluster mode is disabled");
    }
}

Example 13 with InterpreterSettingManager

use of org.apache.zeppelin.interpreter.InterpreterSettingManager in project zeppelin by apache.

the class NotebookServiceTest method setUp.

@Before
public void setUp() throws Exception {
    notebookDir = Files.createTempDirectory("notebookDir").toAbsolutePath().toFile();
    System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath());
    ZeppelinConfiguration zeppelinConfiguration = ZeppelinConfiguration.create();
    NotebookRepo notebookRepo = new VFSNotebookRepo();
    notebookRepo.init(zeppelinConfiguration);
    InterpreterSettingManager mockInterpreterSettingManager = mock(InterpreterSettingManager.class);
    InterpreterFactory mockInterpreterFactory = mock(InterpreterFactory.class);
    Interpreter mockInterpreter = mock(Interpreter.class);
    when(mockInterpreterFactory.getInterpreter(any(), any())).thenReturn(mockInterpreter);
    when(mockInterpreter.interpret(eq("invalid_code"), any())).thenReturn(new InterpreterResult(Code.ERROR, "failed"));
    when(mockInterpreter.interpret(eq("1+1"), any())).thenReturn(new InterpreterResult(Code.SUCCESS, "succeed"));
    doCallRealMethod().when(mockInterpreter).getScheduler();
    when(mockInterpreter.getFormType()).thenReturn(FormType.NATIVE);
    ManagedInterpreterGroup mockInterpreterGroup = mock(ManagedInterpreterGroup.class);
    when(mockInterpreter.getInterpreterGroup()).thenReturn(mockInterpreterGroup);
    InterpreterSetting mockInterpreterSetting = mock(InterpreterSetting.class);
    when(mockInterpreterSetting.isUserAuthorized(any())).thenReturn(true);
    when(mockInterpreterGroup.getInterpreterSetting()).thenReturn(mockInterpreterSetting);
    when(mockInterpreterSetting.getStatus()).thenReturn(InterpreterSetting.Status.READY);
    Credentials credentials = new Credentials();
    NoteManager noteManager = new NoteManager(notebookRepo, zeppelinConfiguration);
    AuthorizationService authorizationService = new AuthorizationService(noteManager, zeppelinConfiguration);
    notebook = new Notebook(zeppelinConfiguration, authorizationService, notebookRepo, noteManager, mockInterpreterFactory, mockInterpreterSettingManager, credentials, null);
    searchService = new LuceneSearch(zeppelinConfiguration, notebook);
    QuartzSchedulerService schedulerService = new QuartzSchedulerService(zeppelinConfiguration, notebook);
    schedulerService.waitForFinishInit();
    notebookService = new NotebookService(notebook, authorizationService, zeppelinConfiguration, schedulerService);
    String interpreterName = "test";
    when(mockInterpreterSetting.getName()).thenReturn(interpreterName);
    when(mockInterpreterSettingManager.getDefaultInterpreterSetting()).thenReturn(mockInterpreterSetting);
}

Example 14 with InterpreterSettingManager

use of org.apache.zeppelin.interpreter.InterpreterSettingManager in project zeppelin by apache.

the class MiniZeppelin method start.

public void start(Class clazz) throws IOException {
    zeppelinHome = new File("..");
    System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), zeppelinHome.getAbsolutePath());
    confDir = new File(zeppelinHome, "conf_" + clazz.getSimpleName());
    notebookDir = new File(zeppelinHome, "notebook_" + clazz.getSimpleName());
    confDir.mkdirs();
    notebookDir.mkdirs();
    LOGGER.info("ZEPPELIN_HOME: " + zeppelinHome.getAbsolutePath());
    FileUtils.copyFile(new File(zeppelinHome, "conf/log4j.properties"), new File(confDir, "log4j.properties"));
    FileUtils.copyFile(new File(zeppelinHome, "conf/log4j2.properties"), new File(confDir, "log4j2.properties"));
    FileUtils.copyFile(new File(zeppelinHome, "conf/log4j_yarn_cluster.properties"), new File(confDir, "log4j_yarn_cluster.properties"));
    System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_CONF_DIR.getVarName(), confDir.getAbsolutePath());
    System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath());
    System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getVarName(), "120000");
    conf = ZeppelinConfiguration.create();
    interpreterSettingManager = new InterpreterSettingManager(conf, mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class));
    interpreterFactory = new InterpreterFactory(interpreterSettingManager);
}

Example 15 with InterpreterSettingManager

use of org.apache.zeppelin.interpreter.InterpreterSettingManager in project zeppelin by apache.

the class NotebookRestApiTest method testRunNoteNonBlocking_Isolated.

@Test
public void testRunNoteNonBlocking_Isolated() throws IOException, InterruptedException {
    LOG.info("Running testRunNoteNonBlocking_Isolated");
    String note1Id = null;
    try {
        InterpreterSettingManager interpreterSettingManager = TestUtils.getInstance(InterpreterSettingManager.class);
        InterpreterSetting interpreterSetting = interpreterSettingManager.getInterpreterSettingByName("python");
        int pythonProcessNum = interpreterSetting.getAllInterpreterGroups().size();
        note1Id = TestUtils.getInstance(Notebook.class).createNote("note1", anonymous);
        // 2 paragraphs
        // P1:
        // %python
        // from __future__ import print_function
        // import time
        // time.sleep(1)
        // user='abc'
        // P2:
        // %python
        // print(user)
        // 
        TestUtils.getInstance(Notebook.class).processNote(note1Id, note1 -> {
            Paragraph p1 = note1.addNewParagraph(AuthenticationInfo.ANONYMOUS);
            Paragraph p2 = note1.addNewParagraph(AuthenticationInfo.ANONYMOUS);
            p1.setText("%python from __future__ import print_function\nimport time\ntime.sleep(1)\nuser='abc'");
            p2.setText("%python print(user)");
            return null;
        });
        CloseableHttpResponse post = httpPost("/notebook/job/" + note1Id + "?blocking=false&isolated=true", "");
        assertThat(post, isAllowed());
        Map<String, Object> resp = gson.fromJson(EntityUtils.toString(post.getEntity(), StandardCharsets.UTF_8), new TypeToken<Map<String, Object>>() {
        }.getType());
        assertEquals("OK", resp.get("status"));
        post.close();
        // wait for all the paragraphs are done
        boolean isRunning = TestUtils.getInstance(Notebook.class).processNote(note1Id, Note::isRunning);
        while (isRunning) {
            Thread.sleep(1000);
            isRunning = TestUtils.getInstance(Notebook.class).processNote(note1Id, Note::isRunning);
        }
        TestUtils.getInstance(Notebook.class).processNote(note1Id, note1 -> {
            Paragraph p1 = note1.getParagraph(0);
            Paragraph p2 = note1.getParagraph(1);
            assertEquals(Job.Status.FINISHED, p1.getStatus());
            assertEquals(Job.Status.FINISHED, p2.getStatus());
            assertEquals("abc\n", p2.getReturn().message().get(0).getData());
            return null;
        });
        // no new python process is created because it is isolated mode.
        assertEquals(pythonProcessNum, interpreterSetting.getAllInterpreterGroups().size());
    } finally {
        // cleanup
        if (null != note1Id) {
            TestUtils.getInstance(Notebook.class).removeNote(note1Id, anonymous);
        }
    }
}