Examples with Shuffle edu.iu.dsc.tws.comms.shuffle.Shuffle used on opensource projects

Example 1 with Shuffle

use of edu.iu.dsc.tws.comms.shuffle.Shuffle in project twister2 by DSC-SPIDAL.

the class DPartitionBatchFinalReceiver method onMessage.

@Override
@SuppressWarnings("unchecked")
public boolean onMessage(int source, int path, int target, int flags, Object object) {
    if (lock.tryLock()) {
        try {
            Shuffle sortedMerger = sortedMergers.get(target);
            if (sortedMerger == null) {
                throw new RuntimeException("Un-expected target id: " + target);
            }
            if (targetStates.get(target) == ReceiverState.INIT) {
                targetStates.put(target, ReceiverState.RECEIVING);
            }
            if ((flags & MessageFlags.SYNC_EMPTY) == MessageFlags.SYNC_EMPTY) {
                Set<Integer> finished = finishedSources.get(target);
                if (finished.contains(source)) {
                    LOG.log(Level.FINE, String.format("%d Duplicate finish from source id %d -> %d", this.thisWorker, source, target));
                } else {
                    finished.add(source);
                }
                if (finished.size() == partition.getSources().size()) {
                    if (!finishedTargets.contains(target)) {
                        finishedTargets.add(target);
                    }
                    targetStates.put(target, ReceiverState.ALL_SYNCS_RECEIVED);
                }
                return true;
            }
            if (targetStates.get(target) == ReceiverState.ALL_SYNCS_RECEIVED || targetStates.get(target) == ReceiverState.SYNCED) {
                return false;
            }
            // add the object to the map
            if (keyed) {
                List<Tuple> tuples = (List<Tuple>) object;
                for (Tuple kc : tuples) {
                    Object data = kc.getValue();
                    byte[] d;
                    if (partition.getReceiveDataType() != MessageTypes.BYTE_ARRAY || !(data instanceof byte[]) || ((flags & MessageFlags.ORIGIN_SENDER) == MessageFlags.ORIGIN_SENDER && partition.getDataType() == MessageTypes.OBJECT)) {
                        // 3rd case handles, when user use Object data type, but send a byte[]
                        d = partition.getDataType().getDataPacker().packToByteArray(data);
                        kc.setValue(d);
                    }
                    sortedMerger.add(kc);
                }
            } else {
                List<Object> contents = (List<Object>) object;
                for (Object kc : contents) {
                    byte[] d;
                    if (partition.getReceiveDataType() != MessageTypes.BYTE_ARRAY || !(kc instanceof byte[]) || ((flags & MessageFlags.ORIGIN_SENDER) == MessageFlags.ORIGIN_SENDER && partition.getDataType() == MessageTypes.OBJECT)) {
                        d = partition.getDataType().getDataPacker().packToByteArray(kc);
                    } else {
                        d = (byte[]) kc;
                    }
                    sortedMerger.add(d, d.length);
                }
            }
            return true;
        } finally {
            lock.unlock();
        }
    }
    return false;
}

Example 2 with Shuffle

use of edu.iu.dsc.tws.comms.shuffle.Shuffle in project twister2 by DSC-SPIDAL.

the class DPartitionBatchFinalReceiver method initMergers.

/**
 * Initialize the mergers, this happens after each refresh
 */
private void initMergers(long maxBytesInMemory, long maxRecordsInMemory, long maxFileSize, int parallelIOAllowance) {
    for (Integer target : expIds.keySet()) {
        String shuffleDirectory = this.shuffleDirectories.get(partition.getLogicalPlan().getIndexOfTaskInNode(target) % this.shuffleDirectories.size());
        Shuffle sortedMerger;
        if (partition.getKeyType() == null) {
            sortedMerger = new FSMerger(maxBytesInMemory, maxRecordsInMemory, shuffleDirectory, DFWIOUtils.getOperationName(target, partition, refresh), partition.getDataType());
        } else {
            if (comparator != null) {
                sortedMerger = new FSKeyedSortedMerger2(maxBytesInMemory, maxFileSize, shuffleDirectory, DFWIOUtils.getOperationName(target, partition, refresh), partition.getKeyType(), partition.getDataType(), comparator, target, groupByKey, parallelIOAllowance);
            } else {
                sortedMerger = new FSKeyedMerger(maxBytesInMemory, maxRecordsInMemory, shuffleDirectory, DFWIOUtils.getOperationName(target, partition, refresh), partition.getKeyType(), partition.getDataType());
            }
        }
        sortedMergers.put(target, sortedMerger);
        finishedSources.put(target, new HashSet<>());
    }
}

Example 3 with Shuffle

use of edu.iu.dsc.tws.comms.shuffle.Shuffle in project twister2 by DSC-SPIDAL.

the class DPartitionBatchFinalReceiver method finishTarget.

private void finishTarget(int target) {
    Shuffle sortedMerger = sortedMergers.get(target);
    sortedMerger.switchToReading();
    Iterator<Object> itr = sortedMerger.readIterator();
    bulkReceiver.receive(target, itr);
}

Example 4 with Shuffle

use of edu.iu.dsc.tws.comms.shuffle.Shuffle in project twister2 by DSC-SPIDAL.

the class DPartitionBatchFinalReceiver method clean.

@Override
public void clean() {
    for (Shuffle s : sortedMergers.values()) {
        s.clean();
    }
    finishedTargetsCompleted.clear();
    finishedTargets.clear();
    finishedSources.forEach((k, v) -> v.clear());
    for (int taraget : targetStates.keySet()) {
        targetStates.put(taraget, ReceiverState.INIT);
    }
    complete = false;
    refresh++;
}

Example 5 with Shuffle

use of edu.iu.dsc.tws.comms.shuffle.Shuffle in project twister2 by DSC-SPIDAL.

the class DKGatherBatchFinalReceiver method sendToTarget.

/**
 * Called from the progress method to perform the communication calls to send the queued messages
 * Since this is the disk based gather this method will save the values to disk
 *
 * @param needsFurtherProgress current state of needsFurtherProgress value
 * @param sourcesFinished specifies if the sources have completed
 * @param target the target(which is a source in this instance) from which the messages are sent
 * @param targetSendQueue the data structure that contains all the message data
 * @return true if further progress is needed or false otherwise
 */
@Override
protected boolean sendToTarget(boolean needsFurtherProgress, boolean sourcesFinished, int target, Queue<Object> targetSendQueue) {
    Shuffle sortedMerger = sortedMergers.get(target);
    while (!targetSendQueue.isEmpty()) {
        Tuple kc = (Tuple) targetSendQueue.poll();
        Object data = kc.getValue();
        byte[] d = dataFlowOperation.getDataType().getDataPacker().packToByteArray(data);
        sortedMerger.add(kc.getKey(), d, d.length);
    }
    // write to disk if overflows
    sortedMerger.run();
    return needsFurtherProgress;
}