use of org.apache.zeppelin.resource.LocalResourcePool in project zeppelin by apache.
the class RemoteInterpreterTest method testInterpreterGroupResetDuringProcessRunning.
@Test
public void testInterpreterGroupResetDuringProcessRunning() throws InterruptedException {
Properties p = new Properties();
intpGroup.put("note", new LinkedList<Interpreter>());
final RemoteInterpreter intpA = createMockInterpreterA(p);
intpGroup.get("note").add(intpA);
intpA.setInterpreterGroup(intpGroup);
intpA.open();
Job jobA = new Job("jobA", null) {
private Object r;
@Override
public Object getReturn() {
return r;
}
@Override
public void setResult(Object results) {
this.r = results;
}
@Override
public int progress() {
return 0;
}
@Override
public Map<String, Object> info() {
return null;
}
@Override
protected Object jobRun() throws Throwable {
return intpA.interpret("2000", new InterpreterContext("note", "jobA", null, "title", "text", new AuthenticationInfo(), new HashMap<String, Object>(), new GUI(), new AngularObjectRegistry(intpGroup.getId(), null), new LocalResourcePool("pool1"), new LinkedList<InterpreterContextRunner>(), null));
}
@Override
protected boolean jobAbort() {
return false;
}
};
intpA.getScheduler().submit(jobA);
// wait for job started
while (intpA.getScheduler().getJobsRunning().size() == 0) {
Thread.sleep(100);
}
// restart interpreter
RemoteInterpreterProcess processA = intpA.getInterpreterProcess();
intpA.close();
InterpreterGroup newInterpreterGroup = new InterpreterGroup(intpA.getInterpreterGroup().getId());
newInterpreterGroup.put("note", new LinkedList<Interpreter>());
intpA.setInterpreterGroup(newInterpreterGroup);
intpA.open();
RemoteInterpreterProcess processB = intpA.getInterpreterProcess();
assertNotSame(processA.hashCode(), processB.hashCode());
}
use of org.apache.zeppelin.resource.LocalResourcePool in project zeppelin by apache.
the class RemoteInterpreterTest method testRunParallel.
@Test
public void testRunParallel() throws InterruptedException {
Properties p = new Properties();
p.put("parallel", "true");
intpGroup.put("note", new LinkedList<Interpreter>());
final RemoteInterpreter intpA = createMockInterpreterA(p);
intpGroup.get("note").add(intpA);
intpA.setInterpreterGroup(intpGroup);
intpA.open();
int concurrency = 4;
final int timeToSleep = 1000;
final List<InterpreterResultMessage> results = new LinkedList<>();
long start = System.currentTimeMillis();
Scheduler scheduler = intpA.getScheduler();
for (int i = 0; i < concurrency; i++) {
final String jobId = Integer.toString(i);
scheduler.submit(new Job(jobId, Integer.toString(i), null, 300) {
private Object r;
@Override
public Object getReturn() {
return r;
}
@Override
public void setResult(Object results) {
this.r = results;
}
@Override
public int progress() {
return 0;
}
@Override
public Map<String, Object> info() {
return null;
}
@Override
protected Object jobRun() throws Throwable {
String stmt = Integer.toString(timeToSleep);
InterpreterResult ret = intpA.interpret(stmt, new InterpreterContext("note", jobId, null, "title", "text", new AuthenticationInfo(), new HashMap<String, Object>(), new GUI(), new AngularObjectRegistry(intpGroup.getId(), null), new LocalResourcePool("pool1"), new LinkedList<InterpreterContextRunner>(), null));
synchronized (results) {
results.addAll(ret.message());
results.notify();
}
return stmt;
}
@Override
protected boolean jobAbort() {
return false;
}
});
}
// wait for job finished
synchronized (results) {
while (results.size() != concurrency) {
results.wait(300);
}
}
long end = System.currentTimeMillis();
assertTrue(end - start < timeToSleep * concurrency);
intpA.close();
}
use of org.apache.zeppelin.resource.LocalResourcePool in project zeppelin by apache.
the class RemoteInterpreterTest method testRemoteInterperterErrorStatus.
@Test
public void testRemoteInterperterErrorStatus() throws TTransportException, IOException {
Properties p = new Properties();
RemoteInterpreter intpA = createMockInterpreterA(p);
intpGroup.put("note", new LinkedList<Interpreter>());
intpGroup.get("note").add(intpA);
intpA.setInterpreterGroup(intpGroup);
intpA.open();
InterpreterResult ret = intpA.interpret("non numeric value", new InterpreterContext("noteId", "id", null, "title", "text", new AuthenticationInfo(), new HashMap<String, Object>(), new GUI(), new AngularObjectRegistry(intpGroup.getId(), null), new LocalResourcePool("pool1"), new LinkedList<InterpreterContextRunner>(), null));
assertEquals(Code.ERROR, ret.code());
}
use of org.apache.zeppelin.resource.LocalResourcePool in project zeppelin by apache.
the class RemoteSchedulerTest method test.
@Test
public void test() throws Exception {
Properties p = new Properties();
final InterpreterGroup intpGroup = new InterpreterGroup();
Map<String, String> env = new HashMap<>();
env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
final RemoteInterpreter intpA = new RemoteInterpreter(p, "note", MockInterpreterA.class.getName(), new File(INTERPRETER_SCRIPT).getAbsolutePath(), "fake", "fakeRepo", env, 10 * 1000, this, null, "anonymous", false);
intpGroup.put("note", new LinkedList<Interpreter>());
intpGroup.get("note").add(intpA);
intpA.setInterpreterGroup(intpGroup);
intpA.open();
Scheduler scheduler = schedulerSvc.createOrGetRemoteScheduler("test", "note", intpA.getInterpreterProcess(), 10);
Job job = new Job("jobId", "jobName", null, 200) {
Object results;
@Override
public Object getReturn() {
return results;
}
@Override
public int progress() {
return 0;
}
@Override
public Map<String, Object> info() {
return null;
}
@Override
protected Object jobRun() throws Throwable {
intpA.interpret("1000", new InterpreterContext("note", "jobId", null, "title", "text", new AuthenticationInfo(), new HashMap<String, Object>(), new GUI(), new AngularObjectRegistry(intpGroup.getId(), null), new LocalResourcePool("pool1"), new LinkedList<InterpreterContextRunner>(), null));
return "1000";
}
@Override
protected boolean jobAbort() {
return false;
}
@Override
public void setResult(Object results) {
this.results = results;
}
};
scheduler.submit(job);
int cycles = 0;
while (!job.isRunning() && cycles < MAX_WAIT_CYCLES) {
Thread.sleep(TICK_WAIT);
cycles++;
}
assertTrue(job.isRunning());
Thread.sleep(5 * TICK_WAIT);
assertEquals(0, scheduler.getJobsWaiting().size());
assertEquals(1, scheduler.getJobsRunning().size());
cycles = 0;
while (!job.isTerminated() && cycles < MAX_WAIT_CYCLES) {
Thread.sleep(TICK_WAIT);
cycles++;
}
assertTrue(job.isTerminated());
assertEquals(0, scheduler.getJobsWaiting().size());
assertEquals(0, scheduler.getJobsRunning().size());
intpA.close();
schedulerSvc.removeScheduler("test");
}
use of org.apache.zeppelin.resource.LocalResourcePool in project zeppelin by apache.
the class NotebookTest method testResourceRemovealOnParagraphNoteRemove.
@Test
public void testResourceRemovealOnParagraphNoteRemove() throws IOException {
Note note = notebook.createNote(anonymous);
interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
for (InterpreterGroup intpGroup : InterpreterGroup.getAll()) {
intpGroup.setResourcePool(new LocalResourcePool(intpGroup.getId()));
}
Paragraph p1 = note.addParagraph(AuthenticationInfo.ANONYMOUS);
p1.setText("hello");
Paragraph p2 = note.addParagraph(AuthenticationInfo.ANONYMOUS);
p2.setText("%mock2 world");
note.runAll();
while (p1.isTerminated() == false || p1.getResult() == null) Thread.yield();
while (p2.isTerminated() == false || p2.getResult() == null) Thread.yield();
assertEquals(2, ResourcePoolUtils.getAllResources().size());
// remove a paragraph
note.removeParagraph(anonymous.getUser(), p1.getId());
assertEquals(1, ResourcePoolUtils.getAllResources().size());
// remove note
notebook.removeNote(note.getId(), anonymous);
assertEquals(0, ResourcePoolUtils.getAllResources().size());
}
Aggregations