use of org.apache.zeppelin.resource.LocalResourcePool in project zeppelin by apache.
the class SparkInterpreterTest method testParagraphUrls.
@Test
public void testParagraphUrls() {
String paraId = "test_para_job_url";
InterpreterContext intpCtx = new InterpreterContext("note", paraId, null, "title", "text", new AuthenticationInfo(), new HashMap<String, Object>(), new GUI(), new AngularObjectRegistry(intpGroup.getId(), null), new LocalResourcePool("id"), new LinkedList<InterpreterContextRunner>(), new InterpreterOutput(null));
repl.interpret("sc.parallelize(1 to 10).map(x => {x}).collect", intpCtx);
Map<String, String> paraInfos = paraIdToInfosMap.get(intpCtx.getParagraphId());
String jobUrl = null;
if (paraInfos != null) {
jobUrl = paraInfos.get("jobUrl");
}
String sparkUIUrl = repl.getSparkUIUrl();
assertNotNull(jobUrl);
assertTrue(jobUrl.startsWith(sparkUIUrl + "/jobs/job?id="));
}
use of org.apache.zeppelin.resource.LocalResourcePool in project zeppelin by apache.
the class RemoteInterpreterTest method testRemoteInterperterCall.
@Test
public void testRemoteInterperterCall() throws TTransportException, IOException {
Properties p = new Properties();
intpGroup.put("note", new LinkedList<Interpreter>());
RemoteInterpreter intpA = createMockInterpreterA(p);
intpGroup.get("note").add(intpA);
intpA.setInterpreterGroup(intpGroup);
RemoteInterpreter intpB = createMockInterpreterB(p);
intpGroup.get("note").add(intpB);
intpB.setInterpreterGroup(intpGroup);
RemoteInterpreterProcess process = intpA.getInterpreterProcess();
process.equals(intpB.getInterpreterProcess());
assertFalse(process.isRunning());
assertEquals(0, process.getNumIdleClient());
assertEquals(0, process.referenceCount());
// initializa all interpreters in the same group
intpA.open();
assertTrue(process.isRunning());
assertEquals(1, process.getNumIdleClient());
assertEquals(2, process.referenceCount());
intpA.interpret("1", new InterpreterContext("note", "id", null, "title", "text", new AuthenticationInfo(), new HashMap<String, Object>(), new GUI(), new AngularObjectRegistry(intpGroup.getId(), null), new LocalResourcePool("pool1"), new LinkedList<InterpreterContextRunner>(), null));
intpB.open();
assertEquals(2, process.referenceCount());
intpA.close();
assertEquals(1, process.referenceCount());
intpB.close();
assertEquals(0, process.referenceCount());
assertFalse(process.isRunning());
}
use of org.apache.zeppelin.resource.LocalResourcePool in project zeppelin by apache.
the class RemoteInterpreterTest method testInterpreterGroupResetDuringProcessRunning.
@Test
public void testInterpreterGroupResetDuringProcessRunning() throws InterruptedException {
Properties p = new Properties();
intpGroup.put("note", new LinkedList<Interpreter>());
final RemoteInterpreter intpA = createMockInterpreterA(p);
intpGroup.get("note").add(intpA);
intpA.setInterpreterGroup(intpGroup);
intpA.open();
Job jobA = new Job("jobA", null) {
private Object r;
@Override
public Object getReturn() {
return r;
}
@Override
public void setResult(Object results) {
this.r = results;
}
@Override
public int progress() {
return 0;
}
@Override
public Map<String, Object> info() {
return null;
}
@Override
protected Object jobRun() throws Throwable {
return intpA.interpret("2000", new InterpreterContext("note", "jobA", null, "title", "text", new AuthenticationInfo(), new HashMap<String, Object>(), new GUI(), new AngularObjectRegistry(intpGroup.getId(), null), new LocalResourcePool("pool1"), new LinkedList<InterpreterContextRunner>(), null));
}
@Override
protected boolean jobAbort() {
return false;
}
};
intpA.getScheduler().submit(jobA);
// wait for job started
while (intpA.getScheduler().getJobsRunning().size() == 0) {
Thread.sleep(100);
}
// restart interpreter
RemoteInterpreterProcess processA = intpA.getInterpreterProcess();
intpA.close();
InterpreterGroup newInterpreterGroup = new InterpreterGroup(intpA.getInterpreterGroup().getId());
newInterpreterGroup.put("note", new LinkedList<Interpreter>());
intpA.setInterpreterGroup(newInterpreterGroup);
intpA.open();
RemoteInterpreterProcess processB = intpA.getInterpreterProcess();
assertNotSame(processA.hashCode(), processB.hashCode());
}
use of org.apache.zeppelin.resource.LocalResourcePool in project zeppelin by apache.
the class RemoteInterpreterTest method testRunParallel.
@Test
public void testRunParallel() throws InterruptedException {
Properties p = new Properties();
p.put("parallel", "true");
intpGroup.put("note", new LinkedList<Interpreter>());
final RemoteInterpreter intpA = createMockInterpreterA(p);
intpGroup.get("note").add(intpA);
intpA.setInterpreterGroup(intpGroup);
intpA.open();
int concurrency = 4;
final int timeToSleep = 1000;
final List<InterpreterResultMessage> results = new LinkedList<>();
long start = System.currentTimeMillis();
Scheduler scheduler = intpA.getScheduler();
for (int i = 0; i < concurrency; i++) {
final String jobId = Integer.toString(i);
scheduler.submit(new Job(jobId, Integer.toString(i), null, 300) {
private Object r;
@Override
public Object getReturn() {
return r;
}
@Override
public void setResult(Object results) {
this.r = results;
}
@Override
public int progress() {
return 0;
}
@Override
public Map<String, Object> info() {
return null;
}
@Override
protected Object jobRun() throws Throwable {
String stmt = Integer.toString(timeToSleep);
InterpreterResult ret = intpA.interpret(stmt, new InterpreterContext("note", jobId, null, "title", "text", new AuthenticationInfo(), new HashMap<String, Object>(), new GUI(), new AngularObjectRegistry(intpGroup.getId(), null), new LocalResourcePool("pool1"), new LinkedList<InterpreterContextRunner>(), null));
synchronized (results) {
results.addAll(ret.message());
results.notify();
}
return stmt;
}
@Override
protected boolean jobAbort() {
return false;
}
});
}
// wait for job finished
synchronized (results) {
while (results.size() != concurrency) {
results.wait(300);
}
}
long end = System.currentTimeMillis();
assertTrue(end - start < timeToSleep * concurrency);
intpA.close();
}
use of org.apache.zeppelin.resource.LocalResourcePool in project zeppelin by apache.
the class SparkSqlInterpreterTest method testSingleRowResult.
@Test
public void testSingleRowResult() throws InterpreterException, IOException {
sparkInterpreter.interpret("case class P(age:Int)", context);
sparkInterpreter.interpret("val gr = sc.parallelize(Seq(P(1),P(2),P(3),P(4),P(5),P(6),P(7),P(8),P(9),P(10)))", context);
sparkInterpreter.interpret("gr.toDF.registerTempTable(\"gr\")", context);
context = InterpreterContext.builder().setNoteId("noteId").setParagraphId("paragraphId").setParagraphTitle("title").setAngularObjectRegistry(new AngularObjectRegistry(intpGroup.getId(), null)).setResourcePool(new LocalResourcePool("id")).setInterpreterOut(new InterpreterOutput()).setIntpEventClient(mock(RemoteInterpreterEventClient.class)).build();
context.getLocalProperties().put("template", "Total count: <h1>{0}</h1>, Total age: <h1>{1}</h1>");
InterpreterResult ret = sqlInterpreter.interpret("select count(1), sum(age) from gr", context);
context.getLocalProperties().remove("template");
assertEquals(InterpreterResult.Code.SUCCESS, ret.code());
assertEquals(Type.HTML, context.out.toInterpreterResultMessage().get(0).getType());
assertEquals("Total count: <h1>10</h1>, Total age: <h1>55</h1>", context.out.toInterpreterResultMessage().get(0).getData());
}
Aggregations