Examples with Interpreter org.apache.zeppelin.interpreter.Interpreter used on opensource projects

Example 36 with Interpreter

use of org.apache.zeppelin.interpreter.Interpreter in project zeppelin by apache.

the class YarnInterpreterLauncherIntegrationTest method testLaunchShellInYarn.

@Test
public void testLaunchShellInYarn() throws YarnException, InterpreterException, InterruptedException {
    InterpreterSetting shellInterpreterSetting = interpreterSettingManager.getInterpreterSettingByName("sh");
    shellInterpreterSetting.setProperty("zeppelin.interpreter.launcher", "yarn");
    shellInterpreterSetting.setProperty("HADOOP_CONF_DIR", hadoopCluster.getConfigPath());
    Interpreter shellInterpreter = interpreterFactory.getInterpreter("sh", new ExecutionContext("user1", "note1", "sh"));
    InterpreterContext context = new InterpreterContext.Builder().setNoteId("note1").setParagraphId("paragraph_1").build();
    InterpreterResult interpreterResult = shellInterpreter.interpret("pwd", context);
    assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
    assertTrue(interpreterResult.toString(), interpreterResult.message().get(0).getData().contains("/usercache/"));
    Thread.sleep(1000);
    // 1 yarn application launched
    GetApplicationsRequest request = GetApplicationsRequest.newInstance(EnumSet.of(YarnApplicationState.RUNNING));
    GetApplicationsResponse response = hadoopCluster.getYarnCluster().getResourceManager().getClientRMService().getApplications(request);
    assertEquals(1, response.getApplicationList().size());
    interpreterSettingManager.close();
}

Example 37 with Interpreter

use of org.apache.zeppelin.interpreter.Interpreter in project zeppelin by apache.

the class YarnInterpreterLauncherIntegrationTest method testJdbcPython_YarnLauncher.

@Test
public void testJdbcPython_YarnLauncher() throws InterpreterException, YarnException, InterruptedException {
    InterpreterSetting jdbcInterpreterSetting = interpreterSettingManager.getInterpreterSettingByName("jdbc");
    jdbcInterpreterSetting.setProperty("default.driver", "com.mysql.jdbc.Driver");
    jdbcInterpreterSetting.setProperty("default.url", "jdbc:mysql://localhost:3306/");
    jdbcInterpreterSetting.setProperty("default.user", "root");
    jdbcInterpreterSetting.setProperty("zeppelin.interpreter.launcher", "yarn");
    jdbcInterpreterSetting.setProperty("zeppelin.interpreter.yarn.resource.memory", "512");
    jdbcInterpreterSetting.setProperty("HADOOP_CONF_DIR", hadoopCluster.getConfigPath());
    Dependency dependency = new Dependency("mysql:mysql-connector-java:5.1.46");
    jdbcInterpreterSetting.setDependencies(Arrays.asList(dependency));
    interpreterSettingManager.restart(jdbcInterpreterSetting.getId());
    jdbcInterpreterSetting.waitForReady(60 * 1000);
    InterpreterSetting pythonInterpreterSetting = interpreterSettingManager.getInterpreterSettingByName("python");
    pythonInterpreterSetting.setProperty("zeppelin.interpreter.launcher", "yarn");
    pythonInterpreterSetting.setProperty("zeppelin.interpreter.yarn.resource.memory", "512");
    pythonInterpreterSetting.setProperty("HADOOP_CONF_DIR", hadoopCluster.getConfigPath());
    Interpreter jdbcInterpreter = interpreterFactory.getInterpreter("jdbc", new ExecutionContext("user1", "note1", "test"));
    assertNotNull("JdbcInterpreter is null", jdbcInterpreter);
    InterpreterContext context = new InterpreterContext.Builder().setNoteId("note1").setParagraphId("paragraph_1").setAuthenticationInfo(AuthenticationInfo.ANONYMOUS).build();
    InterpreterResult interpreterResult = jdbcInterpreter.interpret("show databases;", context);
    assertEquals(interpreterResult.toString(), InterpreterResult.Code.SUCCESS, interpreterResult.code());
    context.getLocalProperties().put("saveAs", "table_1");
    interpreterResult = jdbcInterpreter.interpret("SELECT 1 as c1, 2 as c2;", context);
    assertEquals(interpreterResult.toString(), InterpreterResult.Code.SUCCESS, interpreterResult.code());
    assertEquals(1, interpreterResult.message().size());
    assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(0).getType());
    assertEquals("c1\tc2\n1\t2\n", interpreterResult.message().get(0).getData());
    // read table_1 from python interpreter
    Interpreter pythonInterpreter = interpreterFactory.getInterpreter("python", new ExecutionContext("user1", "note1", "test"));
    assertNotNull("PythonInterpreter is null", pythonInterpreter);
    context = new InterpreterContext.Builder().setNoteId("note1").setParagraphId("paragraph_1").setAuthenticationInfo(AuthenticationInfo.ANONYMOUS).build();
    interpreterResult = pythonInterpreter.interpret("df=z.getAsDataFrame('table_1')\nz.show(df)", context);
    assertEquals(interpreterResult.toString(), InterpreterResult.Code.SUCCESS, interpreterResult.code());
    assertEquals(1, interpreterResult.message().size());
    assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(0).getType());
    assertEquals("c1\tc2\n1\t2\n", interpreterResult.message().get(0).getData());
    // 2 yarn application launched
    GetApplicationsRequest request = GetApplicationsRequest.newInstance(EnumSet.of(YarnApplicationState.RUNNING));
    GetApplicationsResponse response = hadoopCluster.getYarnCluster().getResourceManager().getClientRMService().getApplications(request);
    assertEquals(2, response.getApplicationList().size());
    interpreterSettingManager.close();
    // sleep for 5 seconds to make sure yarn apps are finished
    Thread.sleep(5 * 1000);
    request = GetApplicationsRequest.newInstance(EnumSet.of(YarnApplicationState.RUNNING));
    response = hadoopCluster.getYarnCluster().getResourceManager().getClientRMService().getApplications(request);
    assertEquals(0, response.getApplicationList().size());
}

Example 38 with Interpreter

use of org.apache.zeppelin.interpreter.Interpreter in project zeppelin by apache.

the class Note method runAllSync.

/**
 * Run all the paragraphs in sync(blocking) way.
 *
 * @param authInfo
 * @param isolated
 */
private void runAllSync(AuthenticationInfo authInfo, boolean isolated, Map<String, Object> params) throws Exception {
    try {
        for (Paragraph p : getParagraphs()) {
            if (!p.isEnabled()) {
                continue;
            }
            p.setAuthenticationInfo(authInfo);
            Map<String, Object> originalParams = p.settings.getParams();
            try {
                if (params != null && !params.isEmpty()) {
                    p.settings.setParams(params);
                }
                Interpreter interpreter = p.getBindedInterpreter();
                if (interpreter != null) {
                    // set interpreter property to execution.mode to be note
                    // so that it could use the correct scheduler. see ZEPPELIN-4832
                    interpreter.setProperty(".execution.mode", "note");
                    interpreter.setProperty(".noteId", id);
                }
                // Must run each paragraph in blocking way.
                if (!run(p.getId(), true)) {
                    LOGGER.warn("Skip running the remain notes because paragraph {} fails", p.getId());
                    return;
                }
            } catch (InterpreterNotFoundException e) {
            // ignore, because the following run method will fail if interpreter not found.
            } finally {
                // reset params to the original value
                p.settings.setParams(originalParams);
            }
        }
    } catch (Exception e) {
        throw e;
    } finally {
        if (isolated) {
            LOGGER.info("Releasing interpreters used by this note: {}", id);
            for (InterpreterSetting setting : getUsedInterpreterSettings()) {
                setting.closeInterpreters(getExecutionContext());
                for (Paragraph p : paragraphs) {
                    p.setInterpreter(null);
                }
            }
        }
    }
}

Example 39 with Interpreter

use of org.apache.zeppelin.interpreter.Interpreter in project zeppelin by apache.

the class Notebook method loadNoteFromRepo.

@SuppressWarnings("rawtypes")
public Note loadNoteFromRepo(String id, AuthenticationInfo subject) {
    Note note = null;
    try {
        // note can be safely returned here, because it's just broadcasted in json later on
        note = processNote(id, n -> n);
    } catch (IOException e) {
        LOGGER.error("Fail to get note: {}", id, e);
        return null;
    }
    if (note == null) {
        return null;
    }
    // Manually inject ALL dependencies, as DI constructor was NOT used
    note.setCredentials(this.credentials);
    note.setInterpreterFactory(replFactory);
    note.setInterpreterSettingManager(interpreterSettingManager);
    note.setParagraphJobListener(this.paragraphJobListener);
    note.setCronSupported(getConf());
    if (note.getDefaultInterpreterGroup() == null) {
        note.setDefaultInterpreterGroup(conf.getString(ConfVars.ZEPPELIN_INTERPRETER_GROUP_DEFAULT));
    }
    Map<String, SnapshotAngularObject> angularObjectSnapshot = new HashMap<>();
    // restore angular object --------------
    Date lastUpdatedDate = new Date(0);
    for (Paragraph p : note.getParagraphs()) {
        p.setNote(note);
        if (p.getDateFinished() != null && lastUpdatedDate.before(p.getDateFinished())) {
            lastUpdatedDate = p.getDateFinished();
        }
    }
    Map<String, List<AngularObject>> savedObjects = note.getAngularObjects();
    if (savedObjects != null) {
        for (Entry<String, List<AngularObject>> intpGroupNameEntry : savedObjects.entrySet()) {
            String intpGroupName = intpGroupNameEntry.getKey();
            List<AngularObject> objectList = intpGroupNameEntry.getValue();
            for (AngularObject object : objectList) {
                SnapshotAngularObject snapshot = angularObjectSnapshot.get(object.getName());
                if (snapshot == null || snapshot.getLastUpdate().before(lastUpdatedDate)) {
                    angularObjectSnapshot.put(object.getName(), new SnapshotAngularObject(intpGroupName, object, lastUpdatedDate));
                }
            }
        }
    }
    note.setNoteEventListeners(this.noteEventListeners);
    for (Entry<String, SnapshotAngularObject> angularObjectSnapshotEntry : angularObjectSnapshot.entrySet()) {
        String name = angularObjectSnapshotEntry.getKey();
        SnapshotAngularObject snapshot = angularObjectSnapshotEntry.getValue();
        List<InterpreterSetting> settings = interpreterSettingManager.get();
        for (InterpreterSetting setting : settings) {
            InterpreterGroup intpGroup = setting.getInterpreterGroup(note.getExecutionContext());
            if (intpGroup != null && intpGroup.getId().equals(snapshot.getIntpGroupId())) {
                AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
                String noteId = snapshot.getAngularObject().getNoteId();
                String paragraphId = snapshot.getAngularObject().getParagraphId();
                // at this point, remote interpreter process is not created.
                // so does not make sense add it to the remote.
                // 
                // therefore instead of addAndNotifyRemoteProcess(), need to use add()
                // that results add angularObject only in ZeppelinServer side not remoteProcessSide
                registry.add(name, snapshot.getAngularObject().get(), noteId, paragraphId);
            }
        }
    }
    return note;
}

Example 40 with Interpreter

use of org.apache.zeppelin.interpreter.Interpreter in project zeppelin by apache.

the class Notebook method getBindedInterpreterSettings.

public List<InterpreterSetting> getBindedInterpreterSettings(String noteId) throws IOException {
    List<InterpreterSetting> interpreterSettings = new ArrayList<>();
    processNote(noteId, note -> {
        if (note != null) {
            Set<InterpreterSetting> settings = new HashSet<>();
            for (Paragraph p : note.getParagraphs()) {
                try {
                    Interpreter intp = p.getBindedInterpreter();
                    settings.add(((ManagedInterpreterGroup) intp.getInterpreterGroup()).getInterpreterSetting());
                } catch (InterpreterNotFoundException e) {
                // ignore this
                }
            }
            // add the default interpreter group
            InterpreterSetting defaultIntpSetting = interpreterSettingManager.getByName(note.getDefaultInterpreterGroup());
            if (defaultIntpSetting != null) {
                settings.add(defaultIntpSetting);
            }
            interpreterSettings.addAll(settings);
        }
        return null;
    });
    return interpreterSettings;
}