Examples with Interpreter org.apache.zeppelin.interpreter.Interpreter used on opensource projects

Example 51 with Interpreter

use of org.apache.zeppelin.interpreter.Interpreter in project zeppelin by apache.

the class LivyInterpreterIT method testSparkInterpreter.

@Test
public void testSparkInterpreter() throws InterpreterException {
    if (!checkPreCondition()) {
        return;
    }
    InterpreterGroup interpreterGroup = new InterpreterGroup("group_1");
    interpreterGroup.put("session_1", new ArrayList<Interpreter>());
    LivySparkInterpreter sparkInterpreter = new LivySparkInterpreter(properties);
    sparkInterpreter.setInterpreterGroup(interpreterGroup);
    interpreterGroup.get("session_1").add(sparkInterpreter);
    AuthenticationInfo authInfo = new AuthenticationInfo("user1");
    MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
    InterpreterOutput output = new InterpreterOutput(outputListener);
    InterpreterContext context = InterpreterContext.builder().setNoteId("noteId").setParagraphId("paragraphId").setAuthenticationInfo(authInfo).setInterpreterOut(output).build();
    sparkInterpreter.open();
    LivySparkSQLInterpreter sqlInterpreter = new LivySparkSQLInterpreter(properties);
    interpreterGroup.get("session_1").add(sqlInterpreter);
    sqlInterpreter.setInterpreterGroup(interpreterGroup);
    sqlInterpreter.open();
    try {
        // detect spark version
        InterpreterResult result = sparkInterpreter.interpret("sc.version", context);
        assertEquals(result.toString(), InterpreterResult.Code.SUCCESS, result.code());
        assertEquals(1, result.message().size());
        boolean isSpark2 = isSpark2(sparkInterpreter, context);
        testRDD(sparkInterpreter, isSpark2);
        testDataFrame(sparkInterpreter, sqlInterpreter, isSpark2);
    } finally {
        sparkInterpreter.close();
        sqlInterpreter.close();
    }
}

Example 52 with Interpreter

use of org.apache.zeppelin.interpreter.Interpreter in project zeppelin by apache.

the class RemoteInterpreterServer method createInterpreter.

@Override
public void createInterpreter(String interpreterGroupId, String sessionId, String className, Map<String, String> properties, String userName) throws InterpreterRPCException, TException {
    try {
        if (interpreterGroup == null) {
            interpreterGroup = new InterpreterGroup(interpreterGroupId);
            angularObjectRegistry = new AngularObjectRegistry(interpreterGroup.getId(), intpEventClient);
            hookRegistry = new InterpreterHookRegistry();
            resourcePool = new DistributedResourcePool(interpreterGroup.getId(), intpEventClient);
            interpreterGroup.setInterpreterHookRegistry(hookRegistry);
            interpreterGroup.setAngularObjectRegistry(angularObjectRegistry);
            interpreterGroup.setResourcePool(resourcePool);
            intpEventClient.setIntpGroupId(interpreterGroupId);
            String localRepoPath = properties.get("zeppelin.interpreter.localRepo");
            if (properties.containsKey("zeppelin.interpreter.output.limit")) {
                InterpreterOutput.LIMIT = Integer.parseInt(properties.get("zeppelin.interpreter.output.limit"));
            }
            depLoader = new DependencyResolver(localRepoPath);
            appLoader = new ApplicationLoader(resourcePool, depLoader);
            resultCacheInSeconds = Integer.parseInt(properties.getOrDefault("zeppelin.interpreter.result.cache", "0"));
        }
        Class<Interpreter> replClass = (Class<Interpreter>) Object.class.forName(className);
        Properties p = new Properties();
        p.putAll(properties);
        setSystemProperty(p);
        Constructor<Interpreter> constructor = replClass.getConstructor(new Class[] { Properties.class });
        Interpreter interpreter = constructor.newInstance(p);
        interpreter.setClassloaderUrls(new URL[] {});
        interpreter.setInterpreterGroup(interpreterGroup);
        interpreter.setUserName(userName);
        interpreterGroup.addInterpreterToSession(new LazyOpenInterpreter(interpreter), sessionId);
        this.isForceShutdown = Boolean.parseBoolean(properties.getOrDefault("zeppelin.interpreter.forceShutdown", "true"));
        LOGGER.info("Instantiate interpreter {}, isForceShutdown: {}", className, isForceShutdown);
    } catch (Exception e) {
        LOGGER.error(e.getMessage(), e);
        throw new InterpreterRPCException("Fail to create interpreter, cause: " + e.toString());
    }
}

Example 53 with Interpreter

use of org.apache.zeppelin.interpreter.Interpreter in project zeppelin by apache.

the class NotebookServiceTest method setUp.

@Before
public void setUp() throws Exception {
    notebookDir = Files.createTempDirectory("notebookDir").toAbsolutePath().toFile();
    System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath());
    ZeppelinConfiguration zeppelinConfiguration = ZeppelinConfiguration.create();
    NotebookRepo notebookRepo = new VFSNotebookRepo();
    notebookRepo.init(zeppelinConfiguration);
    InterpreterSettingManager mockInterpreterSettingManager = mock(InterpreterSettingManager.class);
    InterpreterFactory mockInterpreterFactory = mock(InterpreterFactory.class);
    Interpreter mockInterpreter = mock(Interpreter.class);
    when(mockInterpreterFactory.getInterpreter(any(), any())).thenReturn(mockInterpreter);
    when(mockInterpreter.interpret(eq("invalid_code"), any())).thenReturn(new InterpreterResult(Code.ERROR, "failed"));
    when(mockInterpreter.interpret(eq("1+1"), any())).thenReturn(new InterpreterResult(Code.SUCCESS, "succeed"));
    doCallRealMethod().when(mockInterpreter).getScheduler();
    when(mockInterpreter.getFormType()).thenReturn(FormType.NATIVE);
    ManagedInterpreterGroup mockInterpreterGroup = mock(ManagedInterpreterGroup.class);
    when(mockInterpreter.getInterpreterGroup()).thenReturn(mockInterpreterGroup);
    InterpreterSetting mockInterpreterSetting = mock(InterpreterSetting.class);
    when(mockInterpreterSetting.isUserAuthorized(any())).thenReturn(true);
    when(mockInterpreterGroup.getInterpreterSetting()).thenReturn(mockInterpreterSetting);
    when(mockInterpreterSetting.getStatus()).thenReturn(InterpreterSetting.Status.READY);
    Credentials credentials = new Credentials();
    NoteManager noteManager = new NoteManager(notebookRepo, zeppelinConfiguration);
    AuthorizationService authorizationService = new AuthorizationService(noteManager, zeppelinConfiguration);
    notebook = new Notebook(zeppelinConfiguration, authorizationService, notebookRepo, noteManager, mockInterpreterFactory, mockInterpreterSettingManager, credentials, null);
    searchService = new LuceneSearch(zeppelinConfiguration, notebook);
    QuartzSchedulerService schedulerService = new QuartzSchedulerService(zeppelinConfiguration, notebook);
    schedulerService.waitForFinishInit();
    notebookService = new NotebookService(notebook, authorizationService, zeppelinConfiguration, schedulerService);
    String interpreterName = "test";
    when(mockInterpreterSetting.getName()).thenReturn(interpreterName);
    when(mockInterpreterSettingManager.getDefaultInterpreterSetting()).thenReturn(mockInterpreterSetting);
}

Example 54 with Interpreter

use of org.apache.zeppelin.interpreter.Interpreter in project zeppelin by apache.

the class Paragraph method recover.

public void recover() {
    try {
        LOGGER.info("Recovering paragraph: {}", getId());
        this.interpreter = getBindedInterpreter();
        InterpreterSetting interpreterSetting = ((ManagedInterpreterGroup) interpreter.getInterpreterGroup()).getInterpreterSetting();
        Map<String, Object> config = interpreterSetting.getConfig(interpreter.getClassName());
        mergeConfig(config);
        if (shouldSkipRunParagraph()) {
            LOGGER.info("Skip to run blank paragraph. {}", getId());
            setStatus(Job.Status.FINISHED);
            return;
        }
        setStatus(Status.READY);
        localProperties.put("isRecover", "true");
        for (List<Interpreter> sessions : this.interpreter.getInterpreterGroup().values()) {
            for (Interpreter intp : sessions) {
                // exclude ConfInterpreter
                if (intp instanceof RemoteInterpreter) {
                    ((RemoteInterpreter) intp).setOpened(true);
                }
            }
        }
        if (getConfig().get("enabled") == null || (Boolean) getConfig().get("enabled")) {
            setAuthenticationInfo(getAuthenticationInfo());
            interpreter.getScheduler().submit(this);
        }
    } catch (InterpreterNotFoundException e) {
        InterpreterResult intpResult = new InterpreterResult(InterpreterResult.Code.ERROR, String.format("Interpreter %s not found", this.intpText));
        setReturn(intpResult, e);
        setStatus(Job.Status.ERROR);
    } catch (Throwable e) {
        InterpreterResult intpResult = new InterpreterResult(InterpreterResult.Code.ERROR, "Unexpected exception: " + ExceptionUtils.getStackTrace(e));
        setReturn(intpResult, e);
        setStatus(Job.Status.ERROR);
    }
}

Example 55 with Interpreter

use of org.apache.zeppelin.interpreter.Interpreter in project zeppelin by apache.

the class Note method getBindedInterpreterSettings.

public List<InterpreterSetting> getBindedInterpreterSettings(List<String> userAndRoles) {
    // use LinkedHashSet because order matters, the first one represent the default interpreter setting.
    Set<InterpreterSetting> settings = new LinkedHashSet<>();
    // add the default interpreter group
    InterpreterSetting defaultIntpSetting = interpreterSettingManager.getByName(getDefaultInterpreterGroup());
    if (defaultIntpSetting != null) {
        settings.add(defaultIntpSetting);
    }
    // add the interpreter setting with the same group of default interpreter group
    if (defaultIntpSetting != null) {
        for (InterpreterSetting intpSetting : interpreterSettingManager.get()) {
            if (intpSetting.getGroup().equals(defaultIntpSetting.getGroup())) {
                if (intpSetting.isUserAuthorized(userAndRoles)) {
                    settings.add(intpSetting);
                }
            }
        }
    }
    // add interpreter group used by each paragraph
    for (Paragraph p : getParagraphs()) {
        try {
            Interpreter intp = p.getBindedInterpreter();
            InterpreterSetting interpreterSetting = ((ManagedInterpreterGroup) intp.getInterpreterGroup()).getInterpreterSetting();
            if (interpreterSetting.isUserAuthorized(userAndRoles)) {
                settings.add(interpreterSetting);
            }
        } catch (InterpreterNotFoundException e) {
        // ignore this
        }
    }
    return new ArrayList<>(settings);
}