Examples with ApexPythonInterpreterException org.apache.apex.malhar.python.base.ApexPythonInterpreterException used on opensource projects

Example 1 with ApexPythonInterpreterException

use of org.apache.apex.malhar.python.base.ApexPythonInterpreterException in project apex-malhar by apache.

the class InterpreterThread method loadMandatoryJVMLibraries.

/**
 * Loads the JEP dynamic library for the JVM to use the JNI bridge into the interpreter
 * @throws ApexPythonInterpreterException if the library could not be loaded or located
 */
private void loadMandatoryJVMLibraries() throws ApexPythonInterpreterException {
    LOG.info("Java library path being used for Interpreted ID " + threadID + " " + System.getProperty("java.library.path"));
    try {
        System.loadLibrary(JEP_LIBRARY_NAME);
    } catch (Exception e) {
        throw new ApexPythonInterpreterException(e);
    }
    LOG.info("JEP library loaded successfully");
}

Example 2 with ApexPythonInterpreterException

use of org.apache.apex.malhar.python.base.ApexPythonInterpreterException in project apex-malhar by apache.

the class InterpreterThread method processCommand.

/**
 * Responsible for polling the request queue and formatting the request payload to make it compatible to the
 *  individual processing logic of the functionalities provided by the interpreter API methods.
 * @param <T> Java templating signature enforcement
 * @throws ApexPythonInterpreterException if an unrecognized command is issued.
 * @throws InterruptedException if interrupted while trying to wait for a request from request queue
 */
private <T> void processCommand() throws ApexPythonInterpreterException, InterruptedException {
    PythonRequestResponse requestResponseHandle = requestQueue.poll(timeOutToPollFromRequestQueue, timeUnitsToPollFromRequestQueue);
    if (requestResponseHandle != null) {
        LOG.debug("Processing command " + requestResponseHandle.getPythonInterpreterRequest().getCommandType());
        busyFlag = true;
        if (errorEncountered) {
            LOG.debug("Error state detected from a previous command. Resetting state to  the previous" + " state of the error");
            try {
                JEP_INSTANCE.eval(null);
                errorEncountered = false;
            } catch (JepException e) {
                LOG.error("Error while trying to clear the state of the interpreter due to previous command" + " " + e.getMessage(), e);
            }
        }
        PythonInterpreterRequest<T> request = requestResponseHandle.getPythonInterpreterRequest();
        PythonInterpreterResponse<T> response = requestResponseHandle.getPythonInterpreterResponse();
        Map<String, Boolean> commandStatus = new HashMap<>(1);
        switch(request.getCommandType()) {
            case EVAL_COMMAND:
                EvalCommandRequestPayload evalPayload = request.getEvalCommandRequestPayload();
                T responseVal = eval(evalPayload.getEvalCommand(), evalPayload.getVariableNameToExtractInEvalCall(), evalPayload.getParamsForEvalCommand(), evalPayload.isDeleteVariableAfterEvalCall(), request.getExpectedReturnType());
                response.setResponse(responseVal);
                if (responseVal != null) {
                    commandStatus.put(evalPayload.getEvalCommand(), Boolean.TRUE);
                } else {
                    commandStatus.put(evalPayload.getEvalCommand(), Boolean.FALSE);
                }
                response.setCommandStatus(commandStatus);
                break;
            case SCRIPT_COMMAND:
                ScriptExecutionRequestPayload scriptPayload = request.getScriptExecutionRequestPayload();
                if (executeScript(scriptPayload.getScriptName())) {
                    commandStatus.put(scriptPayload.getScriptName(), Boolean.TRUE);
                } else {
                    commandStatus.put(scriptPayload.getScriptName(), Boolean.FALSE);
                }
                response.setCommandStatus(commandStatus);
                break;
            case METHOD_INVOCATION_COMMAND:
                MethodCallRequestPayload requestpayload = request.getMethodCallRequest();
                response.setResponse(executeMethodCall(requestpayload.getNameOfMethod(), requestpayload.getArgs(), request.getExpectedReturnType()));
                if (response.getResponse() == null) {
                    commandStatus.put(requestpayload.getNameOfMethod(), Boolean.FALSE);
                } else {
                    commandStatus.put(requestpayload.getNameOfMethod(), Boolean.TRUE);
                }
                response.setCommandStatus(commandStatus);
                break;
            case GENERIC_COMMANDS:
                response.setCommandStatus(runCommands(request.getGenericCommandsRequestPayload().getGenericCommands()));
                break;
            default:
                throw new ApexPythonInterpreterException(new Exception("Unspecified Interpreter command"));
        }
        requestResponseHandle.setRequestCompletionTime(System.currentTimeMillis());
        responseQueue.put(requestResponseHandle);
        LOG.debug("Submitted the response and executed " + response.getCommandStatus().size() + " instances of command");
    }
    busyFlag = false;
}

Example 3 with ApexPythonInterpreterException

use of org.apache.apex.malhar.python.base.ApexPythonInterpreterException in project apex-malhar by apache.

the class InterpreterThread method startInterpreter.

/**
 * Starts the interpreter by loading the shared libraries
 * @throws ApexPythonInterpreterException if the interpreter could not be started
 */
public void startInterpreter() throws ApexPythonInterpreterException {
    Thread.currentThread().setName(threadID);
    // To allow for time aware calls
    Thread.currentThread().setPriority(Thread.MAX_PRIORITY);
    loadMandatoryJVMLibraries();
    JepConfig config = new JepConfig().setRedirectOutputStreams(true).setInteractive(false).setClassLoader(Thread.currentThread().getContextClassLoader());
    if (initConfigs.containsKey(PYTHON_INCLUDE_PATHS)) {
        List<String> includePaths = (List<String>) initConfigs.get(PYTHON_INCLUDE_PATHS);
        if (includePaths != null) {
            LOG.info("Adding include path for the in-memory interpreter instance");
            for (String anIncludePath : includePaths) {
                config.addIncludePaths(anIncludePath);
            }
        }
    }
    if (initConfigs.containsKey(PYTHON_SHARED_LIBS)) {
        Set<String> sharedLibs = (Set<String>) initConfigs.get(PYTHON_SHARED_LIBS);
        if (sharedLibs != null) {
            config.setSharedModules(sharedLibs);
            LOG.info("Loaded " + sharedLibs.size() + " shared libraries as config");
        }
    } else {
        LOG.info(" No shared libraries loaded");
    }
    if (initConfigs.containsKey(IDLE_INTERPRETER_SPIN_POLICY)) {
        spinPolicy = SpinPolicy.valueOf((String) initConfigs.get(IDLE_INTERPRETER_SPIN_POLICY));
        LOG.debug("Configuring spin policy to be " + spinPolicy);
    }
    if (initConfigs.containsKey(SLEEP_TIME_MS_IN_CASE_OF_NO_REQUESTS)) {
        sleepTimeMsInCaseOfNoRequests = (Long) initConfigs.get(SLEEP_TIME_MS_IN_CASE_OF_NO_REQUESTS);
        LOG.debug("Configuring sleep time for no requests situation to be " + sleepTimeMsInCaseOfNoRequests);
    }
    try {
        LOG.info("Launching the in-memory interpreter");
        JEP_INSTANCE = new Jep(config);
    } catch (JepException e) {
        // Purposefully logging as this will help in startup issues being captured inline
        LOG.error(e.getMessage(), e);
        throw new ApexPythonInterpreterException(e);
    }
}

Example 4 with ApexPythonInterpreterException

use of org.apache.apex.malhar.python.base.ApexPythonInterpreterException in project apex-malhar by apache.

the class InterpreterThread method stopInterpreter.

/**
 * Stops the interpreter as requested from the operator/main thread
 * @throws ApexPythonInterpreterException if not able to stop the
 */
public void stopInterpreter() throws ApexPythonInterpreterException {
    isStopped = true;
    LOG.info("Attempting to close the interpreter thread");
    try {
        JEP_INSTANCE.close();
    } catch (Exception e) {
        LOG.error("Error while stopping the interpreter thread ", e);
        throw new ApexPythonInterpreterException(e);
    }
    LOG.info("Interpreter closed");
}

Example 5 with ApexPythonInterpreterException

use of org.apache.apex.malhar.python.base.ApexPythonInterpreterException in project apex-malhar by apache.

the class JepPythonEngine method executeMethodCall.

/**
 *  See {@link ApexPythonEngine#executeMethodCall(WorkerExecutionMode, long, long, PythonInterpreterRequest)} for more
 *  details. Note that if the worker execution mode {@link WorkerExecutionMode} is BROADCAST, then the time SLA
 *  set is the total time for all workers i.e. each worker is given a ( total time / N ) where N is the current
 *   number of worker threads
 *
 * @param executionMode If the method call needs to be invoked on all workers or any single worker
 * @param windowId used to select the worker from the worker pool.Can be any long if an operator is not using this.
 * @param requestId used to select the worker from the worker pool. Can be any long if an operator is not using this.
 * @param req Represents the request to be processed.
 * @param <T>
 * @return
 * @throws ApexPythonInterpreterException
 */
@Override
public <T> Map<String, PythonRequestResponse<T>> executeMethodCall(WorkerExecutionMode executionMode, long windowId, long requestId, PythonInterpreterRequest<T> req) throws ApexPythonInterpreterException {
    checkNotNullConditions(req);
    checkNotNull(req.getMethodCallRequest(), "Method call info not set");
    checkNotNull(req.getMethodCallRequest().getNameOfMethod(), "Method name not set");
    Map<String, PythonRequestResponse<T>> returnStatus = new HashMap<>();
    req.setCommandType(PythonCommandType.METHOD_INVOCATION_COMMAND);
    PythonRequestResponse lastSuccessfullySubmittedRequest = null;
    try {
        if (executionMode.equals(WorkerExecutionMode.BROADCAST)) {
            long timeOutPerWorker = TimeUnit.NANOSECONDS.convert(req.getTimeout(), req.getTimeUnit()) / numWorkerThreads;
            if (timeOutPerWorker == 0) {
                timeOutPerWorker = 1;
            }
            req.setTimeout(timeOutPerWorker);
            req.setTimeUnit(TimeUnit.NANOSECONDS);
            for (InterpreterWrapper wrapper : workers) {
                lastSuccessfullySubmittedRequest = wrapper.executeMethodCall(windowId, requestId, req);
                if (lastSuccessfullySubmittedRequest != null) {
                    returnStatus.put(wrapper.getInterpreterId(), lastSuccessfullySubmittedRequest);
                }
            }
            if (returnStatus.size() > 0) {
                commandHistory.add(lastSuccessfullySubmittedRequest);
            }
        } else {
            InterpreterWrapper currentThread = null;
            if (executionMode.equals(WorkerExecutionMode.ANY)) {
                currentThread = selectWorkerForCurrentCall(requestId);
            }
            if (executionMode.equals(WorkerExecutionMode.STICKY)) {
                currentThread = workers.get(req.hashCode() % numWorkerThreads);
                LOG.debug(" Choosing sticky worker " + currentThread.getInterpreterId());
            }
            if (currentThread != null) {
                lastSuccessfullySubmittedRequest = currentThread.executeMethodCall(windowId, requestId, req);
                if (lastSuccessfullySubmittedRequest != null) {
                    returnStatus.put(currentThread.getInterpreterId(), lastSuccessfullySubmittedRequest);
                } else {
                    throw new ApexPythonInterpreterException("No free interpreter threads available." + " Consider increasing workers and relaunch");
                }
            }
        }
    } catch (InterruptedException e) {
        throw new ApexPythonInterpreterException(e);
    }
    return returnStatus;
}