Examples with Shuffle edu.iu.dsc.tws.comms.shuffle.Shuffle used on opensource projects

Example 6 with Shuffle

use of edu.iu.dsc.tws.comms.shuffle.Shuffle in project twister2 by DSC-SPIDAL.

the class DKGatherBatchFinalReceiver method finishProgress.

/**
 * Performs the final steps of the progress method in the receiver. Since this is the disk based
 * gather the finish method will signal the Shuffle that the provided target is completed.
 *
 * @param needsFurtherProgress current state of needsFurtherProgress value
 * @param target the target(which is a source in this instance) from which the messages are sent
 * @return true if further progress is needed or false otherwise
 */
protected boolean finishProgress(boolean needsFurtherProgress, int target) {
    batchDone.put(target, true);
    Shuffle sortedMerger = sortedMergers.get(target);
    sortedMerger.switchToReading();
    Iterator<Object> itr = sortedMerger.readIterator();
    bulkReceiver.receive(target, itr);
    return needsFurtherProgress;
}

Example 7 with Shuffle

use of edu.iu.dsc.tws.comms.shuffle.Shuffle in project twister2 by DSC-SPIDAL.

the class DPartitionBatchFinalReceiver method progress.

@Override
public boolean progress() {
    if (lock.tryLock()) {
        try {
            boolean needFurtherProgress = false;
            for (int i = 0; i < targetsArray.length; i++) {
                int target = targetsArray[i];
                Shuffle sorts = sortedMergers.get(target);
                sorts.run();
                ReceiverState state = targetStates.get(target);
                if (state != ReceiverState.SYNCED) {
                    needFurtherProgress = true;
                }
            }
            if (!needFurtherProgress) {
                return needFurtherProgress;
            }
            for (int i = 0; i < finishedTargets.size(); i++) {
                int target = finishedTargets.get(i);
                if (!finishedTargetsCompleted.contains(target) && partition.isDelegateComplete()) {
                    finishTarget(target);
                    targetStates.put(target, ReceiverState.SYNCED);
                    onSyncEvent(target, null);
                    finishedTargetsCompleted.add(target);
                }
            }
        } finally {
            lock.unlock();
        }
    }
    complete = finishedTargetsCompleted.size() == targets.size();
    return !complete;
}

Example 8 with Shuffle

use of edu.iu.dsc.tws.comms.shuffle.Shuffle in project twister2 by DSC-SPIDAL.

the class DKGatherBatchFinalReceiver method init.

@Override
public void init(Config cfg, DataFlowOperation op, Map<Integer, List<Integer>> expectedIds) {
    super.init(cfg, op, expectedIds);
    long maxBytesInMemory = CommunicationContext.getShuffleMaxBytesInMemory(cfg);
    long maxRecordsInMemory = CommunicationContext.getShuffleMaxRecordsInMemory(cfg);
    long maxBytesToFile = CommunicationContext.getShuffleFileSize(cfg);
    int parallelIOAllowance = CommunicationContext.getParallelIOAllowance(cfg);
    for (Integer target : expectedIds.keySet()) {
        Shuffle sortedMerger;
        if (sorted) {
            sortedMerger = new FSKeyedSortedMerger2(maxBytesInMemory, maxBytesToFile, shuffleDirectory, getOperationName(target), dataFlowOperation.getKeyType(), dataFlowOperation.getDataType(), comparator, target, this.groupByKey, parallelIOAllowance);
        } else {
            sortedMerger = new FSKeyedMerger(maxBytesInMemory, maxRecordsInMemory, shuffleDirectory, getOperationName(target), dataFlowOperation.getKeyType(), dataFlowOperation.getDataType());
        }
        sortedMergers.put(target, sortedMerger);
    }
    this.bulkReceiver.init(cfg, expectedIds.keySet());
}