Examples with LocalResourcePool org.apache.zeppelin.resource.LocalResourcePool used on opensource projects

Example 21 with LocalResourcePool

use of org.apache.zeppelin.resource.LocalResourcePool in project zeppelin by apache.

the class RemoteInterpreterTest method testEnvronmentAndPropertySet.

@Test
public void testEnvronmentAndPropertySet() {
    Properties p = new Properties();
    p.setProperty("MY_ENV1", "env value 1");
    p.setProperty("my.property.1", "property value 1");
    RemoteInterpreter intp = new RemoteInterpreter(p, "note", MockInterpreterEnv.class.getName(), new File(INTERPRETER_SCRIPT).getAbsolutePath(), "fake", "fakeRepo", env, 10 * 1000, null, null, "anonymous", false);
    intpGroup.put("note", new LinkedList<Interpreter>());
    intpGroup.get("note").add(intp);
    intp.setInterpreterGroup(intpGroup);
    intp.open();
    InterpreterContext context = new InterpreterContext("noteId", "id", null, "title", "text", new AuthenticationInfo(), new HashMap<String, Object>(), new GUI(), new AngularObjectRegistry(intpGroup.getId(), null), new LocalResourcePool("pool1"), new LinkedList<InterpreterContextRunner>(), null);
    assertEquals("env value 1", intp.interpret("getEnv MY_ENV1", context).message().get(0).getData());
    assertEquals(Code.ERROR, intp.interpret("getProperty MY_ENV1", context).code());
    assertEquals(Code.ERROR, intp.interpret("getEnv my.property.1", context).code());
    assertEquals("property value 1", intp.interpret("getProperty my.property.1", context).message().get(0).getData());
    intp.close();
}

Example 22 with LocalResourcePool

use of org.apache.zeppelin.resource.LocalResourcePool in project zeppelin by apache.

the class RemoteInterpreterTest method testRemoteSchedulerSharingSubmit.

@Test
public void testRemoteSchedulerSharingSubmit() throws TTransportException, IOException, InterruptedException {
    Properties p = new Properties();
    intpGroup.put("note", new LinkedList<Interpreter>());
    final RemoteInterpreter intpA = createMockInterpreterA(p);
    intpGroup.get("note").add(intpA);
    intpA.setInterpreterGroup(intpGroup);
    final RemoteInterpreter intpB = createMockInterpreterB(p);
    intpGroup.get("note").add(intpB);
    intpB.setInterpreterGroup(intpGroup);
    intpA.open();
    intpB.open();
    long start = System.currentTimeMillis();
    Job jobA = new Job("jobA", null) {

        private Object r;

        @Override
        public Object getReturn() {
            return r;
        }

        @Override
        public void setResult(Object results) {
            this.r = results;
        }

        @Override
        public int progress() {
            return 0;
        }

        @Override
        public Map<String, Object> info() {
            return null;
        }

        @Override
        protected Object jobRun() throws Throwable {
            return intpA.interpret("500", new InterpreterContext("note", "jobA", null, "title", "text", new AuthenticationInfo(), new HashMap<String, Object>(), new GUI(), new AngularObjectRegistry(intpGroup.getId(), null), new LocalResourcePool("pool1"), new LinkedList<InterpreterContextRunner>(), null));
        }

        @Override
        protected boolean jobAbort() {
            return false;
        }
    };
    intpA.getScheduler().submit(jobA);
    Job jobB = new Job("jobB", null) {

        private Object r;

        @Override
        public Object getReturn() {
            return r;
        }

        @Override
        public void setResult(Object results) {
            this.r = results;
        }

        @Override
        public int progress() {
            return 0;
        }

        @Override
        public Map<String, Object> info() {
            return null;
        }

        @Override
        protected Object jobRun() throws Throwable {
            return intpB.interpret("500", new InterpreterContext("note", "jobB", null, "title", "text", new AuthenticationInfo(), new HashMap<String, Object>(), new GUI(), new AngularObjectRegistry(intpGroup.getId(), null), new LocalResourcePool("pool1"), new LinkedList<InterpreterContextRunner>(), null));
        }

        @Override
        protected boolean jobAbort() {
            return false;
        }
    };
    intpB.getScheduler().submit(jobB);
    // wait until both job finished
    while (jobA.getStatus() != Status.FINISHED || jobB.getStatus() != Status.FINISHED) {
        Thread.sleep(100);
    }
    long end = System.currentTimeMillis();
    assertTrue(end - start >= 1000);
    assertEquals("1000", ((InterpreterResult) jobB.getReturn()).message().get(0).getData());
    intpA.close();
    intpB.close();
}

Example 23 with LocalResourcePool

use of org.apache.zeppelin.resource.LocalResourcePool in project zeppelin by apache.

the class ZeppCtxtVariableTest method setUp.

@Before
public void setUp() throws Exception {
    resourcePool = new LocalResourcePool("ZeppelinContextVariableInterpolationTest");
    resourcePool.put("PI", "3.1415");
}

Example 24 with LocalResourcePool

use of org.apache.zeppelin.resource.LocalResourcePool in project zeppelin by apache.

the class NotebookTest method testResourceRemovealOnParagraphNoteRemove.

@Test
public void testResourceRemovealOnParagraphNoteRemove() throws Exception {
    String noteId = notebook.createNote("note1", anonymous);
    notebook.processNote(noteId, note -> {
        Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
        p1.setText("%mock1 hello");
        Paragraph p2 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
        p2.setText("%mock2 world");
        for (InterpreterGroup intpGroup : interpreterSettingManager.getAllInterpreterGroup()) {
            intpGroup.setResourcePool(new LocalResourcePool(intpGroup.getId()));
        }
        try {
            note.runAll(anonymous, true, false, new HashMap<>());
        } catch (Exception e) {
            fail();
        }
        assertEquals(2, interpreterSettingManager.getAllResources().size());
        // remove a paragraph
        note.removeParagraph(anonymous.getUser(), p1.getId());
        assertEquals(1, interpreterSettingManager.getAllResources().size());
        return null;
    });
    // remove note
    notebook.removeNote(noteId, anonymous);
    assertEquals(0, interpreterSettingManager.getAllResources().size());
}

Example 25 with LocalResourcePool

use of org.apache.zeppelin.resource.LocalResourcePool in project zeppelin by apache.

the class RemoteSchedulerTest method testAbortOnPending.

@Test
public void testAbortOnPending() throws Exception {
    final RemoteInterpreter intpA = (RemoteInterpreter) interpreterSetting.getInterpreter("user1", note1Id, "mock");
    intpA.open();
    Scheduler scheduler = intpA.getScheduler();
    Job<Object> job1 = new Job<Object>("jobId1", "jobName1", null) {

        Object results;

        InterpreterContext context = InterpreterContext.builder().setNoteId("noteId").setParagraphId("jobId1").setResourcePool(new LocalResourcePool("pool1")).build();

        @Override
        public Object getReturn() {
            return results;
        }

        @Override
        public int progress() {
            return 0;
        }

        @Override
        public Map<String, Object> info() {
            return null;
        }

        @Override
        protected Object jobRun() throws Throwable {
            intpA.interpret("1000", context);
            return "1000";
        }

        @Override
        protected boolean jobAbort() {
            if (isRunning()) {
                try {
                    intpA.cancel(context);
                } catch (InterpreterException e) {
                    e.printStackTrace();
                }
            }
            return true;
        }

        @Override
        public void setResult(Object results) {
            this.results = results;
        }
    };
    Job<Object> job2 = new Job<Object>("jobId2", "jobName2", null) {

        public Object results;

        InterpreterContext context = InterpreterContext.builder().setNoteId("noteId").setParagraphId("jobId2").setResourcePool(new LocalResourcePool("pool1")).build();

        @Override
        public Object getReturn() {
            return results;
        }

        @Override
        public int progress() {
            return 0;
        }

        @Override
        public Map<String, Object> info() {
            return null;
        }

        @Override
        protected Object jobRun() throws Throwable {
            intpA.interpret("1000", context);
            return "1000";
        }

        @Override
        protected boolean jobAbort() {
            if (isRunning()) {
                try {
                    intpA.cancel(context);
                } catch (InterpreterException e) {
                    e.printStackTrace();
                }
            }
            return true;
        }

        @Override
        public void setResult(Object results) {
            this.results = results;
        }
    };
    job2.setResult("result2");
    scheduler.submit(job1);
    scheduler.submit(job2);
    int cycles = 0;
    while (!job1.isRunning() && cycles < MAX_WAIT_CYCLES) {
        Thread.sleep(TICK_WAIT);
        cycles++;
    }
    assertTrue(job1.isRunning());
    assertEquals(Status.PENDING, job2.getStatus());
    job2.abort();
    cycles = 0;
    while (!job1.isTerminated() && cycles < MAX_WAIT_CYCLES) {
        Thread.sleep(TICK_WAIT);
        cycles++;
    }
    assertNotNull(job1.getDateFinished());
    assertTrue(job1.isTerminated());
    assertNull(job2.getDateFinished());
    assertTrue(job2.isTerminated());
    assertEquals("result2", job2.getReturn());
    intpA.close();
    schedulerSvc.removeScheduler("test");
}