Examples with BasePythonExecutionOperator org.apache.apex.malhar.python.base.BasePythonExecutionOperator used on opensource projects

Example 1 with BasePythonExecutionOperator

use of org.apache.apex.malhar.python.base.BasePythonExecutionOperator in project apex-malhar by apache.

the class ThreadStarvationBasedPartitioner method buildTargetPartitions.

/**
 * Calculates the partitions that are required based on the starvations encountered for each checkpoint state. The
 *  new instance is fed with the command history of the original operator ( if any ) so that the new instance can
 *   be in the same state of the original operator when it starts processing the new tuples.
 * @param partitions The current set of partitions
 * @param context The partitioning context
 * @return The new set of partitioned instances keeping the old ones in tact and rebuilding only new ones if needed.
 */
@Override
protected List<Partition<BasePythonExecutionOperator>> buildTargetPartitions(Collection<Partition<BasePythonExecutionOperator>> partitions, PartitioningContext context) {
    List<Partition<BasePythonExecutionOperator>> returnList = new ArrayList<>();
    if (partitions != null) {
        returnList.addAll(partitions);
        for (Partition<BasePythonExecutionOperator> aCurrentPartition : partitions) {
            BasePythonExecutionOperator anOperator = aCurrentPartition.getPartitionedInstance();
            long starvedCount = anOperator.getNumStarvedReturns();
            long requestsForCheckpointWindow = anOperator.getNumberOfRequestsProcessedPerCheckpoint();
            if (requestsForCheckpointWindow != 0) {
                // when the operator is starting for the first time
                float starvationPercent = 100 - (((requestsForCheckpointWindow - starvedCount) / requestsForCheckpointWindow) * 100);
                if (starvationPercent > anOperator.getStarvationPercentBeforeSpawningNewInstance()) {
                    LOG.info("Creating a new instance of the python operator as starvation % is " + starvationPercent);
                    Partition<BasePythonExecutionOperator> newInstance = clonePartition();
                    List<PythonRequestResponse> commandHistory = new ArrayList<>();
                    commandHistory.addAll(anOperator.getAccumulatedCommandHistory());
                    newInstance.getPartitionedInstance().setAccumulatedCommandHistory(commandHistory);
                    returnList.add(newInstance);
                }
            }
        }
    }
    return returnList;
}