Examples with Scheduler org.apache.zeppelin.scheduler.Scheduler used on opensource projects

Example 6 with Scheduler

use of org.apache.zeppelin.scheduler.Scheduler in project zeppelin by apache.

the class RemoteInterpreterServer method interpret.

@Override
public RemoteInterpreterResult interpret(String noteId, String className, String st, RemoteInterpreterContext interpreterContext) throws TException {
    if (logger.isDebugEnabled()) {
        logger.debug("st:\n{}", st);
    }
    Interpreter intp = getInterpreter(noteId, className);
    InterpreterContext context = convert(interpreterContext);
    context.setClassName(intp.getClassName());
    Scheduler scheduler = intp.getScheduler();
    InterpretJobListener jobListener = new InterpretJobListener();
    InterpretJob job = new InterpretJob(interpreterContext.getParagraphId(), "remoteInterpretJob_" + System.currentTimeMillis(), jobListener, JobProgressPoller.DEFAULT_INTERVAL_MSEC, intp, st, context);
    scheduler.submit(job);
    while (!job.isTerminated()) {
        synchronized (jobListener) {
            try {
                jobListener.wait(1000);
            } catch (InterruptedException e) {
                logger.info("Exception in RemoteInterpreterServer while interpret, jobListener.wait", e);
            }
        }
    }
    InterpreterResult result;
    if (job.getStatus() == Status.ERROR) {
        result = new InterpreterResult(Code.ERROR, Job.getStack(job.getException()));
    } else {
        result = (InterpreterResult) job.getReturn();
        // in case of job abort in PENDING status, result can be null
        if (result == null) {
            result = new InterpreterResult(Code.KEEP_PREVIOUS_RESULT);
        }
    }
    return convert(result, context.getConfig(), context.getGui());
}

Example 7 with Scheduler

use of org.apache.zeppelin.scheduler.Scheduler in project zeppelin by apache.

the class Paragraph method jobAbort.

@Override
protected boolean jobAbort() {
    Interpreter repl = getRepl(getRequiredReplName());
    if (repl == null) {
        // when interpreters are already destroyed
        return true;
    }
    Scheduler scheduler = repl.getScheduler();
    if (scheduler == null) {
        return true;
    }
    Job job = scheduler.removeFromWaitingQueue(getId());
    if (job != null) {
        job.setStatus(Status.ABORT);
    } else {
        repl.cancel(getInterpreterContextWithoutRunner(null));
    }
    return true;
}

Example 8 with Scheduler

use of org.apache.zeppelin.scheduler.Scheduler in project zeppelin by apache.

the class FIFOSchedulerTest method testRun.

public void testRun() throws InterruptedException {
    Scheduler s = schedulerSvc.createOrGetFIFOScheduler("test");
    assertEquals(0, s.getJobsRunning().size());
    assertEquals(0, s.getJobsWaiting().size());
    Job job1 = new SleepingJob("job1", null, 500);
    Job job2 = new SleepingJob("job2", null, 500);
    s.submit(job1);
    s.submit(job2);
    Thread.sleep(200);
    assertEquals(Status.RUNNING, job1.getStatus());
    assertEquals(Status.PENDING, job2.getStatus());
    assertEquals(1, s.getJobsRunning().size());
    assertEquals(1, s.getJobsWaiting().size());
    Thread.sleep(500);
    assertEquals(Status.FINISHED, job1.getStatus());
    assertEquals(Status.RUNNING, job2.getStatus());
    assertTrue((500 < (Long) job1.getReturn()));
    assertEquals(1, s.getJobsRunning().size());
    assertEquals(0, s.getJobsWaiting().size());
}

Example 9 with Scheduler

use of org.apache.zeppelin.scheduler.Scheduler in project zeppelin by apache.

the class ParallelSchedulerTest method testRun.

public void testRun() throws InterruptedException {
    Scheduler s = schedulerSvc.createOrGetParallelScheduler("test", 2);
    assertEquals(0, s.getJobsRunning().size());
    assertEquals(0, s.getJobsWaiting().size());
    Job job1 = new SleepingJob("job1", null, 500);
    Job job2 = new SleepingJob("job2", null, 500);
    Job job3 = new SleepingJob("job3", null, 500);
    s.submit(job1);
    s.submit(job2);
    s.submit(job3);
    Thread.sleep(200);
    assertEquals(Status.RUNNING, job1.getStatus());
    assertEquals(Status.RUNNING, job2.getStatus());
    assertEquals(Status.PENDING, job3.getStatus());
    assertEquals(2, s.getJobsRunning().size());
    assertEquals(1, s.getJobsWaiting().size());
    Thread.sleep(500);
    assertEquals(Status.FINISHED, job1.getStatus());
    assertEquals(Status.FINISHED, job2.getStatus());
    assertEquals(Status.RUNNING, job3.getStatus());
    assertEquals(1, s.getJobsRunning().size());
    assertEquals(0, s.getJobsWaiting().size());
}

Example 10 with Scheduler

use of org.apache.zeppelin.scheduler.Scheduler in project zeppelin by apache.

the class RemoteInterpreterTest method testRunOrderPreserved.

@Test
public void testRunOrderPreserved() throws InterruptedException {
    Properties p = new Properties();
    intpGroup.put("note", new LinkedList<Interpreter>());
    final RemoteInterpreter intpA = createMockInterpreterA(p);
    intpGroup.get("note").add(intpA);
    intpA.setInterpreterGroup(intpGroup);
    intpA.open();
    int concurrency = 3;
    final List<InterpreterResultMessage> results = new LinkedList<>();
    Scheduler scheduler = intpA.getScheduler();
    for (int i = 0; i < concurrency; i++) {
        final String jobId = Integer.toString(i);
        scheduler.submit(new Job(jobId, Integer.toString(i), null, 200) {

            private Object r;

            @Override
            public Object getReturn() {
                return r;
            }

            @Override
            public void setResult(Object results) {
                this.r = results;
            }

            @Override
            public int progress() {
                return 0;
            }

            @Override
            public Map<String, Object> info() {
                return null;
            }

            @Override
            protected Object jobRun() throws Throwable {
                InterpreterResult ret = intpA.interpret(getJobName(), new InterpreterContext("note", jobId, null, "title", "text", new AuthenticationInfo(), new HashMap<String, Object>(), new GUI(), new AngularObjectRegistry(intpGroup.getId(), null), new LocalResourcePool("pool1"), new LinkedList<InterpreterContextRunner>(), null));
                synchronized (results) {
                    results.addAll(ret.message());
                    results.notify();
                }
                return null;
            }

            @Override
            protected boolean jobAbort() {
                return false;
            }
        });
    }
    // wait for job finished
    synchronized (results) {
        while (results.size() != concurrency) {
            results.wait(300);
        }
    }
    int i = 0;
    for (InterpreterResultMessage result : results) {
        assertEquals(Integer.toString(i++), result.getData());
    }
    assertEquals(concurrency, i);
    intpA.close();
}