Examples with InterpreterSettingManager org.apache.zeppelin.interpreter.InterpreterSettingManager used on opensource projects

Example 1 with InterpreterSettingManager

use of org.apache.zeppelin.interpreter.InterpreterSettingManager in project zeppelin by apache.

the class NoteInterpreterLoaderTest method setUp.

@Before
public void setUp() throws Exception {
    tmpDir = new File(System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis());
    tmpDir.mkdirs();
    new File(tmpDir, "conf").mkdirs();
    System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), tmpDir.getAbsolutePath());
    conf = ZeppelinConfiguration.create();
    depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo");
    interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
    factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
    ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
    interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, Maps.<String, Object>newHashMap()));
    interpreterInfos.add(new InterpreterInfo(MockInterpreter11.class.getName(), "mock11", false, Maps.<String, Object>newHashMap()));
    ArrayList<InterpreterInfo> interpreterInfos2 = new ArrayList<>();
    interpreterInfos2.add(new InterpreterInfo(MockInterpreter2.class.getName(), "mock2", true, Maps.<String, Object>newHashMap()));
    interpreterSettingManager.add("group1", interpreterInfos, Lists.<Dependency>newArrayList(), new InterpreterOption(), Maps.<String, InterpreterProperty>newHashMap(), "mock", null);
    interpreterSettingManager.add("group2", interpreterInfos2, Lists.<Dependency>newArrayList(), new InterpreterOption(), Maps.<String, InterpreterProperty>newHashMap(), "mock", null);
    interpreterSettingManager.createNewSetting("group1", "group1", Lists.<Dependency>newArrayList(), new InterpreterOption(), new Properties());
    interpreterSettingManager.createNewSetting("group2", "group2", Lists.<Dependency>newArrayList(), new InterpreterOption(), new Properties());
}

Example 2 with InterpreterSettingManager

use of org.apache.zeppelin.interpreter.InterpreterSettingManager in project zeppelin by apache.

the class NotebookRestApiTest method testRunNoteBlocking_Isolated.

@Test
public void testRunNoteBlocking_Isolated() throws IOException {
    LOG.info("Running testRunNoteBlocking_Isolated");
    String note1Id = null;
    try {
        InterpreterSettingManager interpreterSettingManager = TestUtils.getInstance(InterpreterSettingManager.class);
        InterpreterSetting interpreterSetting = interpreterSettingManager.getInterpreterSettingByName("python");
        int pythonProcessNum = interpreterSetting.getAllInterpreterGroups().size();
        note1Id = TestUtils.getInstance(Notebook.class).createNote("note1", anonymous);
        // 2 paragraphs
        // P1:
        // %python
        // from __future__ import print_function
        // import time
        // time.sleep(1)
        // user='abc'
        // P2:
        // %python
        // print(user)
        // 
        TestUtils.getInstance(Notebook.class).processNote(note1Id, note1 -> {
            Paragraph p1 = note1.addNewParagraph(AuthenticationInfo.ANONYMOUS);
            Paragraph p2 = note1.addNewParagraph(AuthenticationInfo.ANONYMOUS);
            p1.setText("%python from __future__ import print_function\nimport time\ntime.sleep(1)\nuser='abc'");
            p2.setText("%python print(user)");
            return null;
        });
        CloseableHttpResponse post = httpPost("/notebook/job/" + note1Id + "?blocking=true&isolated=true", "");
        assertThat(post, isAllowed());
        Map<String, Object> resp = gson.fromJson(EntityUtils.toString(post.getEntity(), StandardCharsets.UTF_8), new TypeToken<Map<String, Object>>() {
        }.getType());
        assertEquals("OK", resp.get("status"));
        post.close();
        TestUtils.getInstance(Notebook.class).processNote(note1Id, note1 -> {
            Paragraph p1 = note1.getParagraph(0);
            Paragraph p2 = note1.getParagraph(1);
            assertEquals(Job.Status.FINISHED, p1.getStatus());
            assertEquals(Job.Status.FINISHED, p2.getStatus());
            assertEquals("abc\n", p2.getReturn().message().get(0).getData());
            return null;
        });
        // no new python process is created because it is isolated mode.
        assertEquals(pythonProcessNum, interpreterSetting.getAllInterpreterGroups().size());
    } finally {
        // cleanup
        if (null != note1Id) {
            TestUtils.getInstance(Notebook.class).removeNote(note1Id, anonymous);
        }
    }
}

Example 3 with InterpreterSettingManager

use of org.apache.zeppelin.interpreter.InterpreterSettingManager in project zeppelin by apache.

the class RecoveryTest method testRecovery_Finished_Paragraph_python.

@Test
public void testRecovery_Finished_Paragraph_python() throws Exception {
    LOG.info("Test testRecovery_Finished_Paragraph_python");
    String note1Id = null;
    try {
        InterpreterSettingManager interpreterSettingManager = TestUtils.getInstance(InterpreterSettingManager.class);
        InterpreterSetting interpreterSetting = interpreterSettingManager.getInterpreterSettingByName("python");
        interpreterSetting.setProperty("zeppelin.python.useIPython", "false");
        interpreterSetting.setProperty("zeppelin.interpreter.result.cache", "100");
        note1Id = TestUtils.getInstance(Notebook.class).createNote("note4", AuthenticationInfo.ANONYMOUS);
        // run  paragraph async, print 'hello' after 10 seconds
        Paragraph p1 = TestUtils.getInstance(Notebook.class).processNote(note1Id, note1 -> {
            return note1.addNewParagraph(AuthenticationInfo.ANONYMOUS);
        });
        p1.setText("%python import time\n" + "for i in range(1, 10):\n" + "    time.sleep(1)\n" + "    print(i)");
        CloseableHttpResponse post = httpPost("/notebook/job/" + note1Id + "/" + p1.getId(), "");
        assertThat(post, isAllowed());
        post.close();
        // wait until paragraph is running
        while (p1.getStatus() != Job.Status.RUNNING) {
            Thread.sleep(1000);
        }
        // shutdown zeppelin and restart it
        shutDown();
        // sleep 15 seconds to make sure the paragraph is finished
        Thread.sleep(15 * 1000);
        startUp(RecoveryTest.class.getSimpleName(), false);
        // sleep 10 seconds to make sure recovering is finished
        Thread.sleep(10 * 1000);
        assertEquals(Job.Status.FINISHED, p1.getStatus());
        assertEquals("1\n" + "2\n" + "3\n" + "4\n" + "5\n" + "6\n" + "7\n" + "8\n" + "9\n", p1.getReturn().message().get(0).getData());
    } catch (Exception e) {
        LOG.error(e.toString(), e);
        throw e;
    } finally {
        if (null != note1Id) {
            TestUtils.getInstance(Notebook.class).removeNote(note1Id, anonymous);
        }
    }
}

Example 4 with InterpreterSettingManager

use of org.apache.zeppelin.interpreter.InterpreterSettingManager in project zeppelin by apache.

the class ZeppelinSparkClusterTest method scalaOutputTest.

@Test
public void scalaOutputTest() throws IOException, InterruptedException {
    assumeTrue("Hadoop version mismatch, skip test", isHadoopVersionMatch());
    String noteId = null;
    try {
        // create new note
        noteId = TestUtils.getInstance(Notebook.class).createNote("note1", anonymous);
        TestUtils.getInstance(Notebook.class).processNote(noteId, note -> {
            Paragraph p = note.addNewParagraph(anonymous);
            p.setText("%spark import java.util.Date\n" + "import java.net.URL\n" + "println(\"hello\")\n");
            note.run(p.getId(), true);
            assertEquals(Status.FINISHED, p.getStatus());
            assertEquals("hello\n" + "import java.util.Date\n" + "import java.net.URL\n", p.getReturn().message().get(0).getData());
            // check spark weburl in zeppelin-server side
            InterpreterSettingManager interpreterSettingManager = TestUtils.getInstance(InterpreterSettingManager.class);
            InterpreterSetting sparkInterpreterSetting = interpreterSettingManager.getByName("spark");
            assertEquals(1, sparkInterpreterSetting.getAllInterpreterGroups().size());
            assertNotNull(sparkInterpreterSetting.getAllInterpreterGroups().get(0).getWebUrl());
            p.setText("%spark invalid_code");
            note.run(p.getId(), true);
            assertEquals(Status.ERROR, p.getStatus());
            assertTrue(p.getReturn().message().get(0).getData().contains("error: "));
            // test local properties
            p.setText("%spark(p1=v1,p2=v2) print(z.getInterpreterContext().getLocalProperties().size())");
            note.run(p.getId(), true);
            assertEquals(Status.FINISHED, p.getStatus());
            assertEquals("2", p.getReturn().message().get(0).getData());
            // test code completion
            List<InterpreterCompletion> completions = note.completion(p.getId(), "sc.", 2, AuthenticationInfo.ANONYMOUS);
            assertTrue(completions.size() > 0);
            // test cancel
            p.setText("%spark sc.range(1,10).map(e=>{Thread.sleep(1000); e}).collect()");
            note.run(p.getId(), false);
            waitForRunning(p);
            p.abort();
            waitForFinish(p);
            assertEquals(Status.ABORT, p.getStatus());
            return null;
        });
    } finally {
        if (null != noteId) {
            TestUtils.getInstance(Notebook.class).removeNote(noteId, anonymous);
        }
    }
}

Example 5 with InterpreterSettingManager

use of org.apache.zeppelin.interpreter.InterpreterSettingManager in project zeppelin by apache.

the class StopInterpreter method main.

public static void main(String[] args) throws IOException {
    ZeppelinConfiguration zConf = ZeppelinConfiguration.create();
    InterpreterSettingManager interpreterSettingManager = new InterpreterSettingManager(zConf, null, null, null);
    RecoveryStorage recoveryStorage = ReflectionUtils.createClazzInstance(zConf.getRecoveryStorageClass(), new Class[] { ZeppelinConfiguration.class, InterpreterSettingManager.class }, new Object[] { zConf, interpreterSettingManager });
    LOGGER.info("Using RecoveryStorage: {}", recoveryStorage.getClass().getName());
    Map<String, InterpreterClient> restoredClients = recoveryStorage.restore();
    if (restoredClients != null) {
        for (InterpreterClient client : restoredClients.values()) {
            LOGGER.info("Stop Interpreter Process: {}:{}", client.getHost(), client.getPort());
            client.stop();
        }
    }
}