Examples with AggregatedObjects edu.iu.dsc.tws.comms.dfw.io.AggregatedObjects used on opensource projects

Example 1 with AggregatedObjects

use of edu.iu.dsc.tws.comms.dfw.io.AggregatedObjects in project twister2 by DSC-SPIDAL.

the class MToNChain method receiveSendInternally.

@Override
public boolean receiveSendInternally(int source, int target, int path, int flags, Object message) {
    if (finishedReceiving) {
        competedReceives++;
        return true;
    }
    if ((flags & MessageFlags.SYNC_EMPTY) == MessageFlags.SYNC_EMPTY) {
        int worker = sourcesToWorkers.get(source);
        IntArrayList sourcesOfWorker = workerToSources.get(worker);
        for (int src : sourcesOfWorker) {
            boolean recv = finalReceiver.onMessage(src, 0, target, flags, message);
            if (recv) {
                addSync(src, target);
            } else {
                throw new RuntimeException("Sync should be accepted");
            }
        }
        competedReceives++;
        return true;
    }
    if (message instanceof AggregatedObjects) {
        if (((AggregatedObjects) message).size() > 0) {
            boolean recv = finalReceiver.onMessage(source, 0, target, flags, message);
            if (recv) {
                competedReceives++;
            }
            return recv;
        } else {
            // we received an empty value, lets increase but not call final receiver
            competedReceives++;
            return true;
        }
    } else if (message == null) {
        competedReceives++;
        return true;
    } else {
        throw new RuntimeException("we can only receive Aggregator objects");
    }
}

Example 2 with AggregatedObjects

use of edu.iu.dsc.tws.comms.dfw.io.AggregatedObjects in project twister2 by DSC-SPIDAL.

the class MToNChain method sendToGroup.

private boolean sendToGroup() {
    boolean sent = false;
    IntArrayList sendingGroup = sendingGroupsWorkers.get(sendGroupIndex);
    for (int j = 0; j < sendingGroup.size(); j++) {
        int worker = sendingGroup.getInt(j);
        IntArrayList workerTargets = workerToTargets.get(worker);
        int targetIndex = sendWorkerTaskIndex.get(worker);
        for (int i = targetIndex; i < workerTargets.size(); i++) {
            int target = workerTargets.get(i);
            Queue<AggregatedObjects<Object>> queue = merged.get(target);
            AggregatedObjects<Object> data = queue.peek();
            if (data == null) {
                data = empty;
            }
            RoutingParameters parameters = targetRoutes.get(target);
            // even if we have 0 tuples, we need to send at this point
            if (!delegate.sendMessage(representSource, data, target, 0, parameters)) {
                return false;
            } else {
                queue.poll();
                sent = true;
                // we are going to decrease the amount of messages in memory
                mergedInMemoryMessages -= data.size();
                // advance the index
                targetIndex++;
                sendWorkerTaskIndex.put(worker, targetIndex);
            }
        }
    }
    if (sent) {
        finishedSendGroups.add(sendGroupIndex);
    }
    return true;
}

Example 3 with AggregatedObjects

use of edu.iu.dsc.tws.comms.dfw.io.AggregatedObjects in project twister2 by DSC-SPIDAL.

the class DGatherBatchFinalReceiver method progress.

/**
 * Method used to communicationProgress work
 */
@SuppressWarnings("unchecked")
public boolean progress() {
    boolean needsFurtherProgress = false;
    for (int t : messages.keySet()) {
        FSMerger fsMerger = sortedMergers.get(t);
        if (batchDone.get(t)) {
            continue;
        }
        boolean allFinished = true;
        // now check weather we have the messages for this source
        Map<Integer, Queue<Object>> map = messages.get(t);
        Map<Integer, Boolean> finishedForTarget = finished.get(t);
        boolean found = true;
        boolean moreThanOne = false;
        for (Map.Entry<Integer, Queue<Object>> e : map.entrySet()) {
            if (e.getValue().size() == 0 && !finishedForTarget.get(e.getKey())) {
                found = false;
            } else {
                moreThanOne = true;
            }
            if (!finishedForTarget.get(e.getKey())) {
                allFinished = false;
            }
        }
        // if we have queues with 0 and more than zero we need further progress
        if (!found && moreThanOne) {
            needsFurtherProgress = true;
        }
        if (found) {
            List<Object> out = new AggregatedObjects<>();
            for (Map.Entry<Integer, Queue<Object>> e : map.entrySet()) {
                Queue<Object> valueList = e.getValue();
                if (valueList.size() > 0) {
                    Object value = valueList.poll();
                    if (value instanceof List) {
                        out.addAll((List) value);
                    } else {
                        out.add(value);
                    }
                    allFinished = false;
                }
            }
            for (Object o : out) {
                byte[] d = gather.getDataType().getDataPacker().packToByteArray(o);
                fsMerger.add(d, d.length);
            }
        } else {
            allFinished = false;
        }
        if (allFinished) {
            batchDone.put(t, true);
            fsMerger.switchToReading();
            bulkReceiver.receive(t, fsMerger.readIterator());
        }
    }
    if (!needsFurtherProgress) {
        complete = true;
    }
    return needsFurtherProgress;
}

Example 4 with AggregatedObjects

use of edu.iu.dsc.tws.comms.dfw.io.AggregatedObjects in project twister2 by DSC-SPIDAL.

the class MToNChain method receiveMessage.

@Override
public boolean receiveMessage(MessageHeader header, Object object) {
    if (finishedReceiving) {
        competedReceives++;
        return true;
    }
    int flags = header.getFlags();
    if ((flags & MessageFlags.SYNC_EMPTY) == MessageFlags.SYNC_EMPTY) {
        int worker = sourcesToWorkers.get(header.getSourceId());
        IntArrayList sourcesOfWorker = workerToSources.get(worker);
        for (int src : sourcesOfWorker) {
            boolean recv = finalReceiver.onMessage(src, CommunicationContext.DEFAULT_DESTINATION, header.getDestinationIdentifier(), header.getFlags(), object);
            if (recv) {
                addSync(src, header.getDestinationIdentifier());
            } else {
                throw new RuntimeException("Sync should be accepted");
            }
        }
        competedReceives++;
        return true;
    } else if ((flags & MessageFlags.SYNC_BARRIER) == MessageFlags.SYNC_BARRIER) {
        boolean recv = finalReceiver.onMessage(header.getSourceId(), CommunicationContext.DEFAULT_DESTINATION, header.getDestinationIdentifier(), header.getFlags(), object);
        if (recv) {
            competedReceives++;
        }
        return recv;
    }
    if (object instanceof AggregatedObjects) {
        if (((AggregatedObjects) object).size() > 0) {
            boolean recv = finalReceiver.onMessage(header.getSourceId(), CommunicationContext.DEFAULT_DESTINATION, header.getDestinationIdentifier(), header.getFlags(), object);
            if (recv) {
                competedReceives++;
            }
            return recv;
        } else {
            competedReceives++;
            return true;
        }
    } else if (object == null) {
        competedReceives++;
        return true;
    } else {
        throw new RuntimeException("we can only receive Aggregator objects");
    }
}