Examples with Scheduler org.apache.zeppelin.scheduler.Scheduler used on opensource projects

Example 1 with Scheduler

use of org.apache.zeppelin.scheduler.Scheduler in project zeppelin by apache.

the class JDBCInterpreterTest method concurrentSettingTest.

@Test
public void concurrentSettingTest() {
    Properties properties = new Properties();
    properties.setProperty("zeppelin.jdbc.concurrent.use", "true");
    properties.setProperty("zeppelin.jdbc.concurrent.max_connection", "10");
    JDBCInterpreter jdbcInterpreter = new JDBCInterpreter(properties);
    assertTrue(jdbcInterpreter.isConcurrentExecution());
    assertEquals(10, jdbcInterpreter.getMaxConcurrentConnection());
    Scheduler scheduler = jdbcInterpreter.getScheduler();
    assertTrue(scheduler instanceof ParallelScheduler);
    properties.clear();
    properties.setProperty("zeppelin.jdbc.concurrent.use", "false");
    jdbcInterpreter = new JDBCInterpreter(properties);
    assertFalse(jdbcInterpreter.isConcurrentExecution());
    scheduler = jdbcInterpreter.getScheduler();
    assertTrue(scheduler instanceof FIFOScheduler);
}

Example 2 with Scheduler

use of org.apache.zeppelin.scheduler.Scheduler in project zeppelin by apache.

the class InterpreterGroup method close.

private void close(final Map<String, InterpreterGroup> interpreterGroupRef, final String processKey, final String sessionKey, final Collection<Interpreter> intpToClose) {
    if (intpToClose == null) {
        return;
    }
    Thread t = new Thread() {

        public void run() {
            for (Interpreter interpreter : intpToClose) {
                Scheduler scheduler = interpreter.getScheduler();
                interpreter.close();
                if (null != scheduler) {
                    SchedulerFactory.singleton().removeScheduler(scheduler.getName());
                }
            }
            if (remoteInterpreterProcess != null) {
                // remoteInterpreterProcess.dereference();
                if (remoteInterpreterProcess.referenceCount() <= 0) {
                    remoteInterpreterProcess = null;
                    allInterpreterGroups.remove(id);
                }
            }
            // interpreters are removed. OMG. It's too dirty!!
            if (null != interpreterGroupRef && null != processKey && null != sessionKey) {
                InterpreterGroup interpreterGroup = interpreterGroupRef.get(processKey);
                if (1 == interpreterGroup.size() && interpreterGroup.containsKey(sessionKey)) {
                    interpreterGroupRef.remove(processKey);
                } else {
                    interpreterGroup.remove(sessionKey);
                }
            }
        }
    };
    t.start();
    try {
        t.join();
    } catch (InterruptedException e) {
        LOGGER.error("Can't close interpreter: {}", getId(), e);
    }
}

Example 3 with Scheduler

use of org.apache.zeppelin.scheduler.Scheduler in project zeppelin by apache.

the class RemoteInterpreterTest method testRunParallel.

@Test
public void testRunParallel() throws InterruptedException {
    Properties p = new Properties();
    p.put("parallel", "true");
    intpGroup.put("note", new LinkedList<Interpreter>());
    final RemoteInterpreter intpA = createMockInterpreterA(p);
    intpGroup.get("note").add(intpA);
    intpA.setInterpreterGroup(intpGroup);
    intpA.open();
    int concurrency = 4;
    final int timeToSleep = 1000;
    final List<InterpreterResultMessage> results = new LinkedList<>();
    long start = System.currentTimeMillis();
    Scheduler scheduler = intpA.getScheduler();
    for (int i = 0; i < concurrency; i++) {
        final String jobId = Integer.toString(i);
        scheduler.submit(new Job(jobId, Integer.toString(i), null, 300) {

            private Object r;

            @Override
            public Object getReturn() {
                return r;
            }

            @Override
            public void setResult(Object results) {
                this.r = results;
            }

            @Override
            public int progress() {
                return 0;
            }

            @Override
            public Map<String, Object> info() {
                return null;
            }

            @Override
            protected Object jobRun() throws Throwable {
                String stmt = Integer.toString(timeToSleep);
                InterpreterResult ret = intpA.interpret(stmt, new InterpreterContext("note", jobId, null, "title", "text", new AuthenticationInfo(), new HashMap<String, Object>(), new GUI(), new AngularObjectRegistry(intpGroup.getId(), null), new LocalResourcePool("pool1"), new LinkedList<InterpreterContextRunner>(), null));
                synchronized (results) {
                    results.addAll(ret.message());
                    results.notify();
                }
                return stmt;
            }

            @Override
            protected boolean jobAbort() {
                return false;
            }
        });
    }
    // wait for job finished
    synchronized (results) {
        while (results.size() != concurrency) {
            results.wait(300);
        }
    }
    long end = System.currentTimeMillis();
    assertTrue(end - start < timeToSleep * concurrency);
    intpA.close();
}

Example 4 with Scheduler

use of org.apache.zeppelin.scheduler.Scheduler in project zeppelin by apache.

the class FIFOSchedulerTest method testAbort.

public void testAbort() throws InterruptedException {
    Scheduler s = schedulerSvc.createOrGetFIFOScheduler("test");
    assertEquals(0, s.getJobsRunning().size());
    assertEquals(0, s.getJobsWaiting().size());
    Job job1 = new SleepingJob("job1", null, 500);
    Job job2 = new SleepingJob("job2", null, 500);
    s.submit(job1);
    s.submit(job2);
    Thread.sleep(200);
    job1.abort();
    job2.abort();
    Thread.sleep(200);
    assertEquals(Status.ABORT, job1.getStatus());
    assertEquals(Status.ABORT, job2.getStatus());
    assertTrue((500 > (Long) job1.getReturn()));
    assertEquals(null, job2.getReturn());
}

Example 5 with Scheduler

use of org.apache.zeppelin.scheduler.Scheduler in project zeppelin by apache.

the class FIFOSchedulerTest method testRemoveFromWaitingQueue.

public void testRemoveFromWaitingQueue() throws InterruptedException {
    Scheduler s = schedulerSvc.createOrGetFIFOScheduler("test");
    assertEquals(0, s.getJobsRunning().size());
    assertEquals(0, s.getJobsWaiting().size());
    Job job1 = new SleepingJob("job1", null, 500);
    Job job2 = new SleepingJob("job2", null, 500);
    s.submit(job1);
    s.submit(job2);
    Thread.sleep(200);
    job1.abort();
    job2.abort();
    Thread.sleep(200);
    assertEquals(Status.ABORT, job1.getStatus());
    assertEquals(Status.ABORT, job2.getStatus());
    assertTrue((500 > (Long) job1.getReturn()));
    assertEquals(null, job2.getReturn());
}