Examples with LocalResourcePool org.apache.zeppelin.resource.LocalResourcePool used on opensource projects

Example 26 with LocalResourcePool

use of org.apache.zeppelin.resource.LocalResourcePool in project zeppelin by apache.

the class KotlinSparkInterpreterTest method setUp.

@BeforeClass
public static void setUp() throws Exception {
    intpGroup = new InterpreterGroup();
    context = InterpreterContext.builder().setNoteId("noteId").setParagraphId("paragraphId").setParagraphTitle("title").setAngularObjectRegistry(new AngularObjectRegistry(intpGroup.getId(), null)).setResourcePool(new LocalResourcePool("id")).setInterpreterOut(new InterpreterOutput()).setIntpEventClient(mock(RemoteInterpreterEventClient.class)).build();
    context.out = new InterpreterOutput(new InterpreterOutputListener() {

        @Override
        public void onUpdateAll(InterpreterOutput out) {
        }

        @Override
        public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) {
            try {
                output = out.toInterpreterResultMessage().getData();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }

        @Override
        public void onUpdate(int index, InterpreterResultMessageOutput out) {
        }
    });
    InterpreterContext.set(context);
    intpGroup.put("note", new LinkedList<Interpreter>());
    Properties properties = getSparkTestProperties(tmpDir);
    repl = new SparkInterpreter(properties);
    repl.setInterpreterGroup(intpGroup);
    intpGroup.get("note").add(repl);
    repl.open();
    repl.interpret("sc", context);
    interpreter = new KotlinSparkInterpreter(properties);
    interpreter.setInterpreterGroup(intpGroup);
    intpGroup.get("note").add(interpreter);
    try {
        interpreter.open();
        sparkSupported = true;
    } catch (UnsupportedClassVersionError e) {
        sparkSupported = false;
    }
}

Example 27 with LocalResourcePool

use of org.apache.zeppelin.resource.LocalResourcePool in project zeppelin by apache.

the class IPythonKernelTest method setUp.

@Before
public void setUp() throws InterpreterException {
    Properties properties = new Properties();
    interpreter = new LazyOpenInterpreter(new JupyterInterpreter(properties));
    intpGroup = new InterpreterGroup();
    resourcePool = new LocalResourcePool("local");
    intpGroup.setResourcePool(resourcePool);
    intpGroup.put("session_1", new ArrayList<>());
    intpGroup.get("session_1").add(interpreter);
    interpreter.setInterpreterGroup(intpGroup);
    interpreter.open();
}

Example 28 with LocalResourcePool

use of org.apache.zeppelin.resource.LocalResourcePool in project SSM by Intel-bigdata.

the class RemoteInterpreterTest method testInterpreterGroupResetDuringProcessRunning.

@Test
public void testInterpreterGroupResetDuringProcessRunning() throws InterruptedException {
    Properties p = new Properties();
    intpGroup.put("note", new LinkedList<Interpreter>());
    final RemoteInterpreter intpA = createMockInterpreterA(p);
    intpGroup.get("note").add(intpA);
    intpA.setInterpreterGroup(intpGroup);
    intpA.open();
    Job jobA = new Job("jobA", null) {

        private Object r;

        @Override
        public Object getReturn() {
            return r;
        }

        @Override
        public void setResult(Object results) {
            this.r = results;
        }

        @Override
        public int progress() {
            return 0;
        }

        @Override
        public Map<String, Object> info() {
            return null;
        }

        @Override
        protected Object jobRun() throws Throwable {
            return intpA.interpret("2000", new InterpreterContext("note", "jobA", null, "title", "text", new AuthenticationInfo(), new HashMap<String, Object>(), new GUI(), new AngularObjectRegistry(intpGroup.getId(), null), new LocalResourcePool("pool1"), new LinkedList<InterpreterContextRunner>(), null));
        }

        @Override
        protected boolean jobAbort() {
            return false;
        }
    };
    intpA.getScheduler().submit(jobA);
    // wait for job started
    while (intpA.getScheduler().getJobsRunning().size() == 0) {
        Thread.sleep(100);
    }
    // restart interpreter
    RemoteInterpreterProcess processA = intpA.getInterpreterProcess();
    intpA.close();
    InterpreterGroup newInterpreterGroup = new InterpreterGroup(intpA.getInterpreterGroup().getId());
    newInterpreterGroup.put("note", new LinkedList<Interpreter>());
    intpA.setInterpreterGroup(newInterpreterGroup);
    intpA.open();
    RemoteInterpreterProcess processB = intpA.getInterpreterProcess();
    assertNotSame(processA.hashCode(), processB.hashCode());
}

Example 29 with LocalResourcePool

use of org.apache.zeppelin.resource.LocalResourcePool in project SSM by Intel-bigdata.

the class RemoteInterpreterTest method testRunOrderPreserved.

@Test
public void testRunOrderPreserved() throws InterruptedException {
    Properties p = new Properties();
    intpGroup.put("note", new LinkedList<Interpreter>());
    final RemoteInterpreter intpA = createMockInterpreterA(p);
    intpGroup.get("note").add(intpA);
    intpA.setInterpreterGroup(intpGroup);
    intpA.open();
    int concurrency = 3;
    final List<InterpreterResultMessage> results = new LinkedList<>();
    Scheduler scheduler = intpA.getScheduler();
    for (int i = 0; i < concurrency; i++) {
        final String jobId = Integer.toString(i);
        scheduler.submit(new Job(jobId, Integer.toString(i), null, 200) {

            private Object r;

            @Override
            public Object getReturn() {
                return r;
            }

            @Override
            public void setResult(Object results) {
                this.r = results;
            }

            @Override
            public int progress() {
                return 0;
            }

            @Override
            public Map<String, Object> info() {
                return null;
            }

            @Override
            protected Object jobRun() throws Throwable {
                InterpreterResult ret = intpA.interpret(getJobName(), new InterpreterContext("note", jobId, null, "title", "text", new AuthenticationInfo(), new HashMap<String, Object>(), new GUI(), new AngularObjectRegistry(intpGroup.getId(), null), new LocalResourcePool("pool1"), new LinkedList<InterpreterContextRunner>(), null));
                synchronized (results) {
                    results.addAll(ret.message());
                    results.notify();
                }
                return null;
            }

            @Override
            protected boolean jobAbort() {
                return false;
            }
        });
    }
    // wait for job finished
    synchronized (results) {
        while (results.size() != concurrency) {
            results.wait(300);
        }
    }
    int i = 0;
    for (InterpreterResultMessage result : results) {
        assertEquals(Integer.toString(i++), result.getData());
    }
    assertEquals(concurrency, i);
    intpA.close();
}

Example 30 with LocalResourcePool

use of org.apache.zeppelin.resource.LocalResourcePool in project SSM by Intel-bigdata.

the class RemoteInterpreterTest method testEnvronmentAndPropertySet.

@Test
public void testEnvronmentAndPropertySet() {
    Properties p = new Properties();
    p.setProperty("MY_ENV1", "env value 1");
    p.setProperty("my.property.1", "property value 1");
    RemoteInterpreter intp = new RemoteInterpreter(p, "note", MockInterpreterEnv.class.getName(), new File(INTERPRETER_SCRIPT).getAbsolutePath(), "fake", "fakeRepo", env, 10 * 1000, null, null, "anonymous", false);
    intpGroup.put("note", new LinkedList<Interpreter>());
    intpGroup.get("note").add(intp);
    intp.setInterpreterGroup(intpGroup);
    intp.open();
    InterpreterContext context = new InterpreterContext("noteId", "id", null, "title", "text", new AuthenticationInfo(), new HashMap<String, Object>(), new GUI(), new AngularObjectRegistry(intpGroup.getId(), null), new LocalResourcePool("pool1"), new LinkedList<InterpreterContextRunner>(), null);
    assertEquals("env value 1", intp.interpret("getEnv MY_ENV1", context).message().get(0).getData());
    assertEquals(Code.ERROR, intp.interpret("getProperty MY_ENV1", context).code());
    assertEquals(Code.ERROR, intp.interpret("getEnv my.property.1", context).code());
    assertEquals("property value 1", intp.interpret("getProperty my.property.1", context).message().get(0).getData());
    intp.close();
}