Examples with ManagedInterpreterGroup org.apache.zeppelin.interpreter.ManagedInterpreterGroup used on opensource projects

Example 1 with ManagedInterpreterGroup

use of org.apache.zeppelin.interpreter.ManagedInterpreterGroup in project zeppelin by apache.

the class SessionManagerService method getSessionInfo.

/**
 * Get the sessionInfo.
 * It method will also update its state if there's an associated interpreter process.
 *
 * @param sessionId
 * @return
 * @throws Exception
 */
public SessionInfo getSessionInfo(String sessionId) throws Exception {
    SessionInfo sessionInfo = sessions.get(sessionId);
    if (sessionInfo == null) {
        LOGGER.warn("No such session: " + sessionId);
        return null;
    }
    ManagedInterpreterGroup interpreterGroup = this.interpreterSettingManager.getInterpreterGroupById(sessionId);
    if (interpreterGroup != null) {
        RemoteInterpreterProcess remoteInterpreterProcess = interpreterGroup.getRemoteInterpreterProcess();
        if (remoteInterpreterProcess == null) {
            sessionInfo.setState(SessionState.READY.name());
        } else {
            sessionInfo.setStartTime(remoteInterpreterProcess.getStartTime());
            sessionInfo.setWeburl(interpreterGroup.getWebUrl());
            if (remoteInterpreterProcess.isRunning()) {
                sessionInfo.setState(SessionState.RUNNING.name());
            } else {
                // e.g. InterpreterProcess is terminated for whatever unexpected reason.
                if (SessionState.RUNNING.name().equalsIgnoreCase(sessionInfo.getState())) {
                    sessionInfo.setState(SessionState.STOPPED.name());
                }
            }
        }
    } else {
        if (SessionState.RUNNING.name().equalsIgnoreCase(sessionInfo.getState())) {
            // if it is running before, but interpreterGroup is null now, that means the session is stopped.
            // e.g. InterpreterProcess is killed if it exceed idle timeout threshold.
            sessionInfo.setState(SessionState.STOPPED.name());
        }
    }
    return sessionInfo;
}

Example 2 with ManagedInterpreterGroup

use of org.apache.zeppelin.interpreter.ManagedInterpreterGroup in project zeppelin by apache.

the class RemoteInterpreter method getOrCreateInterpreterProcess.

public synchronized RemoteInterpreterProcess getOrCreateInterpreterProcess() throws IOException {
    if (this.interpreterProcess != null) {
        return this.interpreterProcess;
    }
    ManagedInterpreterGroup intpGroup = getInterpreterGroup();
    this.interpreterProcess = intpGroup.getOrCreateInterpreterProcess(getUserName(), properties);
    return interpreterProcess;
}

Example 3 with ManagedInterpreterGroup

use of org.apache.zeppelin.interpreter.ManagedInterpreterGroup in project zeppelin by apache.

the class ParagraphTest method credentialReplacement.

// (TODO zjffdu) temporary disable it.
// https://github.com/apache/zeppelin/pull/3416
@Ignore
@Test
public void credentialReplacement() throws Throwable {
    Note mockNote = mock(Note.class);
    Credentials creds = mock(Credentials.class);
    when(mockNote.getCredentials()).thenReturn(creds);
    Paragraph spyParagraph = spy(new Paragraph("para_1", mockNote, null));
    UserCredentials uc = mock(UserCredentials.class);
    when(creds.getUserCredentials(anyString())).thenReturn(uc);
    UsernamePassword up = new UsernamePassword("user", "pwd");
    when(uc.getUsernamePassword("ent")).thenReturn(up);
    Interpreter mockInterpreter = mock(Interpreter.class);
    spyParagraph.setInterpreter(mockInterpreter);
    doReturn(mockInterpreter).when(spyParagraph).getBindedInterpreter();
    ManagedInterpreterGroup mockInterpreterGroup = mock(ManagedInterpreterGroup.class);
    when(mockInterpreter.getInterpreterGroup()).thenReturn(mockInterpreterGroup);
    when(mockInterpreterGroup.getId()).thenReturn("mock_id_1");
    when(mockInterpreterGroup.getAngularObjectRegistry()).thenReturn(mock(AngularObjectRegistry.class));
    when(mockInterpreterGroup.getResourcePool()).thenReturn(mock(ResourcePool.class));
    when(mockInterpreter.getFormType()).thenReturn(FormType.NONE);
    ParagraphJobListener mockJobListener = mock(ParagraphJobListener.class);
    doReturn(mockJobListener).when(spyParagraph).getListener();
    InterpreterResult mockInterpreterResult = mock(InterpreterResult.class);
    when(mockInterpreter.interpret(anyString(), Mockito.<InterpreterContext>any())).thenReturn(mockInterpreterResult);
    when(mockInterpreterResult.code()).thenReturn(Code.SUCCESS);
    AuthenticationInfo user1 = new AuthenticationInfo("user1");
    spyParagraph.setAuthenticationInfo(user1);
    spyParagraph.setText("val x = \"usr={user.ent}&pass={password.ent}\"");
    // Credentials should only be injected when it is enabled for an interpreter or when specified in a local property
    when(mockInterpreter.getProperty(Constants.INJECT_CREDENTIALS, "false")).thenReturn("false");
    spyParagraph.jobRun();
    verify(mockInterpreter).interpret(eq("val x = \"usr={user.ent}&pass={password.ent}\""), any(InterpreterContext.class));
    when(mockInterpreter.getProperty(Constants.INJECT_CREDENTIALS, "false")).thenReturn("true");
    mockInterpreter.setProperty(Constants.INJECT_CREDENTIALS, "true");
    spyParagraph.jobRun();
    verify(mockInterpreter).interpret(eq("val x = \"usr=user&pass=pwd\""), any(InterpreterContext.class));
    // Check if local property override works
    when(mockInterpreter.getProperty(Constants.INJECT_CREDENTIALS, "false")).thenReturn("true");
    spyParagraph.getLocalProperties().put(Constants.INJECT_CREDENTIALS, "true");
    spyParagraph.jobRun();
    verify(mockInterpreter).interpret(eq("val x = \"usr=user&pass=pwd\""), any(InterpreterContext.class));
}

Example 4 with ManagedInterpreterGroup

use of org.apache.zeppelin.interpreter.ManagedInterpreterGroup in project zeppelin by apache.

the class Paragraph method execute.

/**
 * Return true only when paragraph run successfully with state of FINISHED.
 * @param blocking
 * @return
 */
public boolean execute(String interpreterGroupId, boolean blocking) {
    try {
        this.interpreterGroupId = interpreterGroupId;
        this.interpreter = getBindedInterpreter();
        InterpreterSetting interpreterSetting = ((ManagedInterpreterGroup) interpreter.getInterpreterGroup()).getInterpreterSetting();
        Map<String, Object> config = interpreterSetting.getConfig(interpreter.getClassName());
        mergeConfig(config);
        // clear output
        setResult(null);
        cleanOutputBuffer();
        cleanRuntimeInfos();
        setStatus(Status.PENDING);
        if (shouldSkipRunParagraph()) {
            LOGGER.info("Skip to run blank paragraph. {}", getId());
            setStatus(Job.Status.FINISHED);
            return true;
        }
        if (isEnabled()) {
            setAuthenticationInfo(getAuthenticationInfo());
            interpreter.getScheduler().submit(this);
        } else {
            LOGGER.info("Skip disabled paragraph. {}", getId());
            setStatus(Job.Status.FINISHED);
            return true;
        }
        if (blocking) {
            while (!getStatus().isCompleted()) {
                try {
                    Thread.sleep(100);
                } catch (InterruptedException e) {
                    throw new RuntimeException(e);
                }
            }
            return getStatus() == Status.FINISHED;
        } else {
            return true;
        }
    } catch (InterpreterNotFoundException e) {
        InterpreterResult intpResult = new InterpreterResult(InterpreterResult.Code.ERROR, String.format("Interpreter %s not found", this.intpText));
        setReturn(intpResult, e);
        setStatus(Job.Status.ERROR);
        return false;
    } catch (Throwable e) {
        InterpreterResult intpResult = new InterpreterResult(InterpreterResult.Code.ERROR, "Unexpected exception: " + ExceptionUtils.getStackTrace(e));
        setReturn(intpResult, e);
        setStatus(Job.Status.ERROR);
        return false;
    }
}

Example 5 with ManagedInterpreterGroup

use of org.apache.zeppelin.interpreter.ManagedInterpreterGroup in project zeppelin by apache.

the class Paragraph method jobRun.

@Override
protected InterpreterResult jobRun() throws Throwable {
    try {
        if (localProperties.getOrDefault("isRecover", "false").equals("false")) {
            this.runtimeInfos.clear();
        }
        this.interpreter = getBindedInterpreter();
        if (this.interpreter == null) {
            LOGGER.error("Can not find interpreter name " + intpText);
            throw new RuntimeException("Can not find interpreter for " + intpText);
        }
        LOGGER.info("Run paragraph [paragraph_id: {}, interpreter: {}, note_id: {}, user: {}]", getId(), this.interpreter.getClassName(), note.getId(), subject.getUser());
        InterpreterSetting interpreterSetting = ((ManagedInterpreterGroup) interpreter.getInterpreterGroup()).getInterpreterSetting();
        if (interpreterSetting.getStatus() != InterpreterSetting.Status.READY) {
            String message = String.format("Interpreter Setting '%s' is not ready, its status is %s", interpreterSetting.getName(), interpreterSetting.getStatus());
            LOGGER.error(message);
            throw new RuntimeException(message);
        }
        if (this.user != null) {
            if (subject != null && !interpreterSetting.isUserAuthorized(subject.getUsersAndRoles())) {
                String msg = String.format("%s has no permission for %s", subject.getUser(), intpText);
                LOGGER.error(msg);
                return new InterpreterResult(Code.ERROR, msg);
            }
        }
        for (Paragraph p : userParagraphMap.values()) {
            p.setText(getText());
        }
        // inject form
        String script = this.scriptText;
        String form = localProperties.getOrDefault("form", interpreter.getFormType().name());
        if (form.equalsIgnoreCase("simple")) {
            // inputs will be built from script body
            LinkedHashMap<String, Input> inputs = Input.extractSimpleQueryForm(script, false);
            LinkedHashMap<String, Input> noteInputs = Input.extractSimpleQueryForm(script, true);
            final AngularObjectRegistry angularRegistry = interpreter.getInterpreterGroup().getAngularObjectRegistry();
            String scriptBody = extractVariablesFromAngularRegistry(script, inputs, angularRegistry);
            settings.setForms(inputs);
            if (!noteInputs.isEmpty()) {
                if (!note.getNoteForms().isEmpty()) {
                    Map<String, Input> currentNoteForms = note.getNoteForms();
                    for (String s : noteInputs.keySet()) {
                        if (!currentNoteForms.containsKey(s)) {
                            currentNoteForms.put(s, noteInputs.get(s));
                        }
                    }
                } else {
                    note.setNoteForms(noteInputs);
                }
            }
            script = Input.getSimpleQuery(note.getNoteParams(), scriptBody, true);
            script = Input.getSimpleQuery(settings.getParams(), script, false);
        } else {
            settings.clear();
        }
        LOGGER.debug("RUN : " + script);
        try {
            InterpreterContext context = getInterpreterContext();
            InterpreterContext.set(context);
            // Inject credentials
            String injectPropStr = interpreter.getProperty(Constants.INJECT_CREDENTIALS, "false");
            injectPropStr = context.getStringLocalProperty(Constants.INJECT_CREDENTIALS, injectPropStr);
            boolean shouldInjectCredentials = Boolean.parseBoolean(injectPropStr);
            InterpreterResult ret = null;
            if (shouldInjectCredentials) {
                UserCredentials creds = context.getAuthenticationInfo().getUserCredentials();
                CredentialInjector credinjector = new CredentialInjector(creds);
                String code = credinjector.replaceCredentials(script);
                ret = interpreter.interpret(code, context);
                ret = credinjector.hidePasswords(ret);
            } else {
                ret = interpreter.interpret(script, context);
            }
            if (interpreter.getFormType() == FormType.NATIVE) {
                note.setNoteParams(context.getNoteGui().getParams());
                note.setNoteForms(context.getNoteGui().getForms());
            }
            if (Code.KEEP_PREVIOUS_RESULT == ret.code()) {
                return getReturn();
            }
            Paragraph p = getUserParagraph(getUser());
            if (null != p) {
                p.setResult(ret);
                p.settings.setParams(settings.getParams());
            }
            return ret;
        } finally {
            InterpreterContext.remove();
        }
    } catch (Exception e) {
        return new InterpreterResult(Code.ERROR, ExceptionUtils.getStackTrace(e));
    } finally {
        localProperties.remove("isRecover");
    }
}