Examples with FSMerger edu.iu.dsc.tws.comms.shuffle.FSMerger used on opensource projects

Example 1 with FSMerger

use of edu.iu.dsc.tws.comms.shuffle.FSMerger in project twister2 by DSC-SPIDAL.

the class DGatherBatchFinalReceiver method progress.

/**
 * Method used to communicationProgress work
 */
@SuppressWarnings("unchecked")
public boolean progress() {
    boolean needsFurtherProgress = false;
    for (int t : messages.keySet()) {
        FSMerger fsMerger = sortedMergers.get(t);
        if (batchDone.get(t)) {
            continue;
        }
        boolean allFinished = true;
        // now check weather we have the messages for this source
        Map<Integer, Queue<Object>> map = messages.get(t);
        Map<Integer, Boolean> finishedForTarget = finished.get(t);
        boolean found = true;
        boolean moreThanOne = false;
        for (Map.Entry<Integer, Queue<Object>> e : map.entrySet()) {
            if (e.getValue().size() == 0 && !finishedForTarget.get(e.getKey())) {
                found = false;
            } else {
                moreThanOne = true;
            }
            if (!finishedForTarget.get(e.getKey())) {
                allFinished = false;
            }
        }
        // if we have queues with 0 and more than zero we need further progress
        if (!found && moreThanOne) {
            needsFurtherProgress = true;
        }
        if (found) {
            List<Object> out = new AggregatedObjects<>();
            for (Map.Entry<Integer, Queue<Object>> e : map.entrySet()) {
                Queue<Object> valueList = e.getValue();
                if (valueList.size() > 0) {
                    Object value = valueList.poll();
                    if (value instanceof List) {
                        out.addAll((List) value);
                    } else {
                        out.add(value);
                    }
                    allFinished = false;
                }
            }
            for (Object o : out) {
                byte[] d = gather.getDataType().getDataPacker().packToByteArray(o);
                fsMerger.add(d, d.length);
            }
        } else {
            allFinished = false;
        }
        if (allFinished) {
            batchDone.put(t, true);
            fsMerger.switchToReading();
            bulkReceiver.receive(t, fsMerger.readIterator());
        }
    }
    if (!needsFurtherProgress) {
        complete = true;
    }
    return needsFurtherProgress;
}

Example 2 with FSMerger

use of edu.iu.dsc.tws.comms.shuffle.FSMerger in project twister2 by DSC-SPIDAL.

the class DPartitionBatchFinalReceiver method initMergers.

/**
 * Initialize the mergers, this happens after each refresh
 */
private void initMergers(long maxBytesInMemory, long maxRecordsInMemory, long maxFileSize, int parallelIOAllowance) {
    for (Integer target : expIds.keySet()) {
        String shuffleDirectory = this.shuffleDirectories.get(partition.getLogicalPlan().getIndexOfTaskInNode(target) % this.shuffleDirectories.size());
        Shuffle sortedMerger;
        if (partition.getKeyType() == null) {
            sortedMerger = new FSMerger(maxBytesInMemory, maxRecordsInMemory, shuffleDirectory, DFWIOUtils.getOperationName(target, partition, refresh), partition.getDataType());
        } else {
            if (comparator != null) {
                sortedMerger = new FSKeyedSortedMerger2(maxBytesInMemory, maxFileSize, shuffleDirectory, DFWIOUtils.getOperationName(target, partition, refresh), partition.getKeyType(), partition.getDataType(), comparator, target, groupByKey, parallelIOAllowance);
            } else {
                sortedMerger = new FSKeyedMerger(maxBytesInMemory, maxRecordsInMemory, shuffleDirectory, DFWIOUtils.getOperationName(target, partition, refresh), partition.getKeyType(), partition.getDataType());
            }
        }
        sortedMergers.put(target, sortedMerger);
        finishedSources.put(target, new HashSet<>());
    }
}

Example 3 with FSMerger

use of edu.iu.dsc.tws.comms.shuffle.FSMerger in project twister2 by DSC-SPIDAL.

the class DGatherBatchFinalReceiver method init.

@Override
public void init(Config cfg, DataFlowOperation op, Map<Integer, List<Integer>> expectedIds) {
    long maxBytesInMemory = CommunicationContext.getShuffleMaxBytesInMemory(cfg);
    long maxRecordsInMemory = CommunicationContext.getShuffleMaxRecordsInMemory(cfg);
    gather = op;
    sendPendingMax = CommunicationContext.sendPendingMax(cfg);
    for (Map.Entry<Integer, List<Integer>> e : expectedIds.entrySet()) {
        Map<Integer, Queue<Object>> messagesPerTask = new HashMap<>();
        Map<Integer, Boolean> finishedPerTask = new HashMap<>();
        for (int i : e.getValue()) {
            messagesPerTask.put(i, new ArrayBlockingQueue<>(sendPendingMax));
            finishedPerTask.put(i, false);
        }
        messages.put(e.getKey(), messagesPerTask);
        finished.put(e.getKey(), finishedPerTask);
        finalMessages.put(e.getKey(), new ArrayList<>());
        batchDone.put(e.getKey(), false);
        FSMerger merger = new FSMerger(maxBytesInMemory, maxRecordsInMemory, shuffleDirectory, getOperationName(e.getKey()), gather.getDataType());
        sortedMergers.put(e.getKey(), merger);
    }
    this.bulkReceiver.init(cfg, expectedIds.keySet());
}