Examples with PythonRequestResponse org.apache.apex.malhar.python.base.requestresponse.PythonRequestResponse used on opensource projects

Example 6 with PythonRequestResponse

use of org.apache.apex.malhar.python.base.requestresponse.PythonRequestResponse in project apex-malhar by apache.

the class ThreadStarvationBasedPartitioner method buildTargetPartitions.

/**
 * Calculates the partitions that are required based on the starvations encountered for each checkpoint state. The
 *  new instance is fed with the command history of the original operator ( if any ) so that the new instance can
 *   be in the same state of the original operator when it starts processing the new tuples.
 * @param partitions The current set of partitions
 * @param context The partitioning context
 * @return The new set of partitioned instances keeping the old ones in tact and rebuilding only new ones if needed.
 */
@Override
protected List<Partition<BasePythonExecutionOperator>> buildTargetPartitions(Collection<Partition<BasePythonExecutionOperator>> partitions, PartitioningContext context) {
    List<Partition<BasePythonExecutionOperator>> returnList = new ArrayList<>();
    if (partitions != null) {
        returnList.addAll(partitions);
        for (Partition<BasePythonExecutionOperator> aCurrentPartition : partitions) {
            BasePythonExecutionOperator anOperator = aCurrentPartition.getPartitionedInstance();
            long starvedCount = anOperator.getNumStarvedReturns();
            long requestsForCheckpointWindow = anOperator.getNumberOfRequestsProcessedPerCheckpoint();
            if (requestsForCheckpointWindow != 0) {
                // when the operator is starting for the first time
                float starvationPercent = 100 - (((requestsForCheckpointWindow - starvedCount) / requestsForCheckpointWindow) * 100);
                if (starvationPercent > anOperator.getStarvationPercentBeforeSpawningNewInstance()) {
                    LOG.info("Creating a new instance of the python operator as starvation % is " + starvationPercent);
                    Partition<BasePythonExecutionOperator> newInstance = clonePartition();
                    List<PythonRequestResponse> commandHistory = new ArrayList<>();
                    commandHistory.addAll(anOperator.getAccumulatedCommandHistory());
                    newInstance.getPartitionedInstance().setAccumulatedCommandHistory(commandHistory);
                    returnList.add(newInstance);
                }
            }
        }
    }
    return returnList;
}

Example 7 with PythonRequestResponse

use of org.apache.apex.malhar.python.base.requestresponse.PythonRequestResponse in project apex-malhar by apache.

the class JepPythonEngine method runCommands.

/**
 * See {@link ApexPythonEngine#runCommands(WorkerExecutionMode, long, long, PythonInterpreterRequest)} for more
 *  details. Note that if the worker execution mode {@link WorkerExecutionMode} is BROADCAST, then the time SLA
 *  set is the total time for all workers i.e. each worker is given a ( total time / N ) where N is the current
 *   number of worker threads
 * @param executionMode Whether these commands need to be run on all worker nodes or any of the worker node
 * @param windowId used to select the worker from the worker pool.Can be any long if an operator is not using this.
 * @param requestId used to select the worker from the worker pool. Can be any long if an operator is not using this.
 * @param request Represents the request to be processed.
 * @return A map containing the command as key and boolean representing success or failure as the value.
 * @throws ApexPythonInterpreterException
 */
@Override
public Map<String, PythonRequestResponse<Void>> runCommands(WorkerExecutionMode executionMode, long windowId, long requestId, PythonInterpreterRequest<Void> request) throws ApexPythonInterpreterException {
    checkNotNullConditions(request);
    checkNotNull(request.getGenericCommandsRequestPayload(), "Run commands payload not set");
    checkNotNull(request.getGenericCommandsRequestPayload().getGenericCommands(), "Commands that need to be run not set");
    Map<String, PythonRequestResponse<Void>> returnStatus = new HashMap<>();
    PythonRequestResponse lastSuccessfullySubmittedRequest = null;
    try {
        if (executionMode.equals(WorkerExecutionMode.BROADCAST)) {
            LOG.debug("Executing run commands on all of the interpreter worker threads");
            long timeOutPerWorker = TimeUnit.NANOSECONDS.convert(request.getTimeout(), request.getTimeUnit()) / numWorkerThreads;
            LOG.debug("Allocating " + timeOutPerWorker + " nanoseconds for each of the worker threads");
            if (timeOutPerWorker == 0) {
                timeOutPerWorker = 1;
            }
            request.setTimeout(timeOutPerWorker);
            request.setTimeUnit(TimeUnit.NANOSECONDS);
            for (InterpreterWrapper wrapper : workers) {
                lastSuccessfullySubmittedRequest = wrapper.runCommands(windowId, requestId, request);
                if (lastSuccessfullySubmittedRequest != null) {
                    returnStatus.put(wrapper.getInterpreterId(), lastSuccessfullySubmittedRequest);
                }
            }
            if (returnStatus.size() > 0) {
                commandHistory.add(lastSuccessfullySubmittedRequest);
            }
        } else {
            InterpreterWrapper currentThread = null;
            if (executionMode.equals(WorkerExecutionMode.ANY)) {
                LOG.debug("Executing run commands on a single interpreter worker thread");
                currentThread = selectWorkerForCurrentCall(requestId);
            }
            if (executionMode.equals(WorkerExecutionMode.STICKY)) {
                currentThread = workers.get(request.hashCode() % numWorkerThreads);
                LOG.debug(" Choosing sticky worker " + currentThread.getInterpreterId());
            }
            if (currentThread != null) {
                lastSuccessfullySubmittedRequest = currentThread.runCommands(windowId, requestId, request);
                if (lastSuccessfullySubmittedRequest != null) {
                    returnStatus.put(currentThread.getInterpreterId(), lastSuccessfullySubmittedRequest);
                }
            } else {
                throw new ApexPythonInterpreterException("No free interpreter threads available." + " Consider increasing workers and relaunch");
            }
        }
    } catch (InterruptedException e) {
        throw new ApexPythonInterpreterException(e);
    }
    return returnStatus;
}

Example 8 with PythonRequestResponse

use of org.apache.apex.malhar.python.base.requestresponse.PythonRequestResponse in project apex-malhar by apache.

the class BasePythonExecutionOperator method emitStragglers.

/**
 * Used to emit the stragglers into the Stragglers port. Invoked either when a new tuple arrives or when Idle time
 *  is detected on this operator.
 */
private void emitStragglers() {
    LOG.debug("Emitting stragglers");
    List<PythonRequestResponse> stragglerResponse = new ArrayList<>();
    getApexPythonEngine().getDelayedResponseQueue().drainTo(stragglerResponse);
    for (PythonRequestResponse aReqResponse : stragglerResponse) {
        responseCounterForThisWindow += 1;
        stragglersPort.emit(aReqResponse);
    }
}

Example 9 with PythonRequestResponse

use of org.apache.apex.malhar.python.base.requestresponse.PythonRequestResponse in project apex-malhar by apache.

the class InterpreterWrapper method executeMethodCall.

/**
 * Implements the time based SLA over the interpreters run commands implementation. See
 *  {@link InterpreterThread#executeMethodCall(String, List, Class)}
 * @param windowId The window ID as provided by the Apex operator. Used for selecting a worker from the worker pool.
 * @param requestId The request ID as provided by the Apex operator. Used for selecting a worker from the worker pool.
 * @param request The payload of the request
 * @return A response object with the results of the execution. Null if the request could not be processed on time.
 * @throws InterruptedException if interrupted while processing the wait for request or writing to delayed response
 *  queue
 */
public <T> PythonRequestResponse<T> executeMethodCall(long windowId, long requestId, PythonInterpreterRequest<T> request) throws InterruptedException {
    request.setCommandType(PythonCommandType.METHOD_INVOCATION_COMMAND);
    PythonRequestResponse requestResponse = buildRequestRespObject(request, windowId, requestId);
    return processRequest(requestResponse, request);
}

Example 10 with PythonRequestResponse

use of org.apache.apex.malhar.python.base.requestresponse.PythonRequestResponse in project apex-malhar by apache.

the class InterpreterWrapper method runCommands.

/**
 * Implements the time based SLA over the interpreters run commands implementation. See
 *  {@link InterpreterThread#runCommands(List)}
 * @param windowId The window ID as provided by the Apex operator. Used for selecting a worker from the worker pool.
 * @param requestId The request ID as provided by the Apex operator. Used for selecting a worker from the worker pool.
 * @param request The payload of the request
 * @return A response object with the results of the execution. Null if the request could not be processed on time.
 * @throws InterruptedException if interrupted while processing the wait for request or writing to delayed response
 *  queue
 */
public PythonRequestResponse<Void> runCommands(long windowId, long requestId, PythonInterpreterRequest<Void> request) throws InterruptedException {
    request.setCommandType(PythonCommandType.GENERIC_COMMANDS);
    PythonRequestResponse requestResponse = buildRequestRespObject(request, windowId, requestId);
    return processRequest(requestResponse, request);
}