use of org.apache.zeppelin.interpreter.InterpreterSettingManager in project zeppelin by apache.
the class VFSNotebookRepoTest method setUp.
@Before
public void setUp() throws Exception {
String zpath = System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis();
mainZepDir = new File(zpath);
mainZepDir.mkdirs();
new File(mainZepDir, "conf").mkdirs();
String mainNotePath = zpath + "/notebook";
mainNotebookDir = new File(mainNotePath);
mainNotebookDir.mkdirs();
System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), mainZepDir.getAbsolutePath());
System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath());
System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), "org.apache.zeppelin.notebook.repo.VFSNotebookRepo");
conf = ZeppelinConfiguration.create();
this.schedulerFactory = new SchedulerFactory();
this.schedulerFactory = new SchedulerFactory();
depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo");
interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap<String, Object>()));
interpreterSettingManager.add("mock1", interpreterInfos, new ArrayList<Dependency>(), new InterpreterOption(), Maps.<String, InterpreterProperty>newHashMap(), "mock1", null);
interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList<Dependency>(), new InterpreterOption(), new Properties());
SearchService search = mock(SearchService.class);
notebookRepo = new VFSNotebookRepo(conf);
notebookAuthorization = NotebookAuthorization.init(conf);
notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, interpreterSettingManager, this, search, notebookAuthorization, null);
}
use of org.apache.zeppelin.interpreter.InterpreterSettingManager in project zeppelin by apache.
the class ZeppelinServer method setupClusterManagerServer.
private static void setupClusterManagerServer(ServiceLocator serviceLocator) {
if (conf.isClusterMode()) {
LOG.info("Cluster mode is enabled, starting ClusterManagerServer");
ClusterManagerServer clusterManagerServer = ClusterManagerServer.getInstance(conf);
NotebookServer notebookServer = serviceLocator.getService(NotebookServer.class);
clusterManagerServer.addClusterEventListeners(ClusterManagerServer.CLUSTER_NOTE_EVENT_TOPIC, notebookServer);
AuthorizationService authorizationService = serviceLocator.getService(AuthorizationService.class);
clusterManagerServer.addClusterEventListeners(ClusterManagerServer.CLUSTER_AUTH_EVENT_TOPIC, authorizationService);
InterpreterSettingManager interpreterSettingManager = serviceLocator.getService(InterpreterSettingManager.class);
clusterManagerServer.addClusterEventListeners(ClusterManagerServer.CLUSTER_INTP_SETTING_EVENT_TOPIC, interpreterSettingManager);
// This allows the ClusterInterpreterLauncher to listen for cluster events.
try {
InterpreterSettingManager intpSettingManager = sharedServiceLocator.getService(InterpreterSettingManager.class);
RecoveryStorage recoveryStorage = ReflectionUtils.createClazzInstance(conf.getRecoveryStorageClass(), new Class[] { ZeppelinConfiguration.class, InterpreterSettingManager.class }, new Object[] { conf, intpSettingManager });
recoveryStorage.init();
PluginManager.get().loadInterpreterLauncher(InterpreterSetting.CLUSTER_INTERPRETER_LAUNCHER_NAME, recoveryStorage);
} catch (IOException e) {
LOG.error(e.getMessage(), e);
}
clusterManagerServer.start();
} else {
LOG.info("Cluster mode is disabled");
}
}
use of org.apache.zeppelin.interpreter.InterpreterSettingManager in project zeppelin by apache.
the class NotebookServiceTest method setUp.
@Before
public void setUp() throws Exception {
notebookDir = Files.createTempDirectory("notebookDir").toAbsolutePath().toFile();
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath());
ZeppelinConfiguration zeppelinConfiguration = ZeppelinConfiguration.create();
NotebookRepo notebookRepo = new VFSNotebookRepo();
notebookRepo.init(zeppelinConfiguration);
InterpreterSettingManager mockInterpreterSettingManager = mock(InterpreterSettingManager.class);
InterpreterFactory mockInterpreterFactory = mock(InterpreterFactory.class);
Interpreter mockInterpreter = mock(Interpreter.class);
when(mockInterpreterFactory.getInterpreter(any(), any())).thenReturn(mockInterpreter);
when(mockInterpreter.interpret(eq("invalid_code"), any())).thenReturn(new InterpreterResult(Code.ERROR, "failed"));
when(mockInterpreter.interpret(eq("1+1"), any())).thenReturn(new InterpreterResult(Code.SUCCESS, "succeed"));
doCallRealMethod().when(mockInterpreter).getScheduler();
when(mockInterpreter.getFormType()).thenReturn(FormType.NATIVE);
ManagedInterpreterGroup mockInterpreterGroup = mock(ManagedInterpreterGroup.class);
when(mockInterpreter.getInterpreterGroup()).thenReturn(mockInterpreterGroup);
InterpreterSetting mockInterpreterSetting = mock(InterpreterSetting.class);
when(mockInterpreterSetting.isUserAuthorized(any())).thenReturn(true);
when(mockInterpreterGroup.getInterpreterSetting()).thenReturn(mockInterpreterSetting);
when(mockInterpreterSetting.getStatus()).thenReturn(InterpreterSetting.Status.READY);
Credentials credentials = new Credentials();
NoteManager noteManager = new NoteManager(notebookRepo, zeppelinConfiguration);
AuthorizationService authorizationService = new AuthorizationService(noteManager, zeppelinConfiguration);
notebook = new Notebook(zeppelinConfiguration, authorizationService, notebookRepo, noteManager, mockInterpreterFactory, mockInterpreterSettingManager, credentials, null);
searchService = new LuceneSearch(zeppelinConfiguration, notebook);
QuartzSchedulerService schedulerService = new QuartzSchedulerService(zeppelinConfiguration, notebook);
schedulerService.waitForFinishInit();
notebookService = new NotebookService(notebook, authorizationService, zeppelinConfiguration, schedulerService);
String interpreterName = "test";
when(mockInterpreterSetting.getName()).thenReturn(interpreterName);
when(mockInterpreterSettingManager.getDefaultInterpreterSetting()).thenReturn(mockInterpreterSetting);
}
use of org.apache.zeppelin.interpreter.InterpreterSettingManager in project zeppelin by apache.
the class MiniZeppelin method start.
public void start(Class clazz) throws IOException {
zeppelinHome = new File("..");
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), zeppelinHome.getAbsolutePath());
confDir = new File(zeppelinHome, "conf_" + clazz.getSimpleName());
notebookDir = new File(zeppelinHome, "notebook_" + clazz.getSimpleName());
confDir.mkdirs();
notebookDir.mkdirs();
LOGGER.info("ZEPPELIN_HOME: " + zeppelinHome.getAbsolutePath());
FileUtils.copyFile(new File(zeppelinHome, "conf/log4j.properties"), new File(confDir, "log4j.properties"));
FileUtils.copyFile(new File(zeppelinHome, "conf/log4j2.properties"), new File(confDir, "log4j2.properties"));
FileUtils.copyFile(new File(zeppelinHome, "conf/log4j_yarn_cluster.properties"), new File(confDir, "log4j_yarn_cluster.properties"));
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_CONF_DIR.getVarName(), confDir.getAbsolutePath());
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath());
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getVarName(), "120000");
conf = ZeppelinConfiguration.create();
interpreterSettingManager = new InterpreterSettingManager(conf, mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class));
interpreterFactory = new InterpreterFactory(interpreterSettingManager);
}
use of org.apache.zeppelin.interpreter.InterpreterSettingManager in project zeppelin by apache.
the class NotebookRestApiTest method testRunNoteNonBlocking_Isolated.
@Test
public void testRunNoteNonBlocking_Isolated() throws IOException, InterruptedException {
LOG.info("Running testRunNoteNonBlocking_Isolated");
String note1Id = null;
try {
InterpreterSettingManager interpreterSettingManager = TestUtils.getInstance(InterpreterSettingManager.class);
InterpreterSetting interpreterSetting = interpreterSettingManager.getInterpreterSettingByName("python");
int pythonProcessNum = interpreterSetting.getAllInterpreterGroups().size();
note1Id = TestUtils.getInstance(Notebook.class).createNote("note1", anonymous);
// 2 paragraphs
// P1:
// %python
// from __future__ import print_function
// import time
// time.sleep(1)
// user='abc'
// P2:
// %python
// print(user)
//
TestUtils.getInstance(Notebook.class).processNote(note1Id, note1 -> {
Paragraph p1 = note1.addNewParagraph(AuthenticationInfo.ANONYMOUS);
Paragraph p2 = note1.addNewParagraph(AuthenticationInfo.ANONYMOUS);
p1.setText("%python from __future__ import print_function\nimport time\ntime.sleep(1)\nuser='abc'");
p2.setText("%python print(user)");
return null;
});
CloseableHttpResponse post = httpPost("/notebook/job/" + note1Id + "?blocking=false&isolated=true", "");
assertThat(post, isAllowed());
Map<String, Object> resp = gson.fromJson(EntityUtils.toString(post.getEntity(), StandardCharsets.UTF_8), new TypeToken<Map<String, Object>>() {
}.getType());
assertEquals("OK", resp.get("status"));
post.close();
// wait for all the paragraphs are done
boolean isRunning = TestUtils.getInstance(Notebook.class).processNote(note1Id, Note::isRunning);
while (isRunning) {
Thread.sleep(1000);
isRunning = TestUtils.getInstance(Notebook.class).processNote(note1Id, Note::isRunning);
}
TestUtils.getInstance(Notebook.class).processNote(note1Id, note1 -> {
Paragraph p1 = note1.getParagraph(0);
Paragraph p2 = note1.getParagraph(1);
assertEquals(Job.Status.FINISHED, p1.getStatus());
assertEquals(Job.Status.FINISHED, p2.getStatus());
assertEquals("abc\n", p2.getReturn().message().get(0).getData());
return null;
});
// no new python process is created because it is isolated mode.
assertEquals(pythonProcessNum, interpreterSetting.getAllInterpreterGroups().size());
} finally {
// cleanup
if (null != note1Id) {
TestUtils.getInstance(Notebook.class).removeNote(note1Id, anonymous);
}
}
}
Aggregations