Examples with MessageType edu.iu.dsc.tws.api.comms.messaging.types.MessageType used on opensource projects

Example 11 with MessageType

use of edu.iu.dsc.tws.api.comms.messaging.types.MessageType in project twister2 by DSC-SPIDAL.

the class HashJoinUtils method rightOuterJoin.

public static List<Object> rightOuterJoin(List<Tuple> leftRelation, List<Tuple> rightRelation, MessageType messageType) {
    Map<Object, List<Tuple>> leftHash = new THashMap<>(messageType);
    List<Object> joinedTuples = new ArrayList<>();
    for (Tuple tuple : leftRelation) {
        leftHash.computeIfAbsent(tuple.getKey(), k -> new ArrayList<>()).add(tuple);
    }
    for (Tuple rightTuple : rightRelation) {
        List<Tuple> leftTuples = leftHash.getOrDefault(rightTuple.getKey(), Collections.emptyList());
        for (Tuple leftTuple : leftTuples) {
            joinedTuples.add(JoinedTuple.of(leftTuple.getKey(), leftTuple.getValue(), rightTuple.getValue()));
        }
        if (leftTuples.isEmpty()) {
            joinedTuples.add(JoinedTuple.of(rightTuple.getKey(), null, rightTuple.getValue()));
        }
    }
    return joinedTuples;
}

Example 12 with MessageType

use of edu.iu.dsc.tws.api.comms.messaging.types.MessageType in project twister2 by DSC-SPIDAL.

the class STAllGatherExample method buildTaskGraph.

@Override
public ComputeGraphBuilder buildTaskGraph() {
    List<Integer> taskStages = jobParameters.getTaskStages();
    int psource = taskStages.get(0);
    int psink = taskStages.get(1);
    MessageType dataType = MessageTypes.INTEGER_ARRAY;
    String edge = "edge";
    BaseSource g = new SourceTask(edge);
    ICompute r = new AllGatherSinkTask();
    computeGraphBuilder.addSource(SOURCE, g, psource);
    computeConnection = computeGraphBuilder.addCompute(SINK, r, psink);
    computeConnection.allgather(SOURCE).viaEdge(edge).withDataType(dataType);
    return computeGraphBuilder;
}

Example 13 with MessageType

use of edu.iu.dsc.tws.api.comms.messaging.types.MessageType in project twister2 by DSC-SPIDAL.

the class STPartitionExample method buildTaskGraph.

@Override
public ComputeGraphBuilder buildTaskGraph() {
    List<Integer> taskStages = jobParameters.getTaskStages();
    int sourceParallelism = taskStages.get(0);
    int sinkParallelism = taskStages.get(1);
    MessageType dataType = MessageTypes.INTEGER_ARRAY;
    String edge = "edge";
    BaseSource g = new SourceTask(edge);
    ((SourceTask) g).setMarkTimingOnlyForLowestTarget(true);
    ICompute r = new PartitionSinkTask();
    computeGraphBuilder.addSource(SOURCE, g, sourceParallelism);
    computeConnection = computeGraphBuilder.addCompute(SINK, r, sinkParallelism);
    computeConnection.partition(SOURCE).viaEdge(edge).withDataType(dataType);
    return computeGraphBuilder;
}

Example 14 with MessageType

use of edu.iu.dsc.tws.api.comms.messaging.types.MessageType in project twister2 by DSC-SPIDAL.

the class BTPartitionKeyedExample method buildTaskGraph.

@Override
public ComputeGraphBuilder buildTaskGraph() {
    List<Integer> taskStages = jobParameters.getTaskStages();
    int sourceParallelism = taskStages.get(0);
    int sinkParallelism = taskStages.get(1);
    MessageType keyType = MessageTypes.INTEGER;
    MessageType dataType = MessageTypes.INTEGER_ARRAY;
    String edge = "edge";
    BaseSource g = new SourceTask(edge, true);
    ICompute r = new BKeyedPartitionSinkTask();
    computeGraphBuilder.addSource(SOURCE, g, sourceParallelism);
    computeConnection = computeGraphBuilder.addCompute(SINK, r, sinkParallelism);
    computeConnection.keyedPartition(SOURCE).viaEdge(edge).withKeyType(keyType).withTaskPartitioner(new DeterministicTaskPartitioner()).withDataType(dataType);
    return computeGraphBuilder;
}

Example 15 with MessageType

use of edu.iu.dsc.tws.api.comms.messaging.types.MessageType in project twister2 by DSC-SPIDAL.

the class ControlledChannelOperation method onReceiveComplete.

@Override
public void onReceiveComplete(int id, int e, DataBuffer buffer) {
    // we need to try to build the message here, we may need many more messages to complete
    ByteBuffer byteBuffer = buffer.getByteBuffer();
    byteBuffer.position(buffer.getSize());
    byteBuffer.flip();
    // we have the source of the message at 0th position as an integer
    int source = byteBuffer.getInt(0);
    InMessage currentMessage = currentMessages.get(source);
    if (currentMessage == null) {
        MessageHeader header = messageDeSerializer.get(source).buildHeader(buffer, e);
        MessageType recvDType = receiveDataType;
        MessageType recvKType = receiveKeyType;
        if ((header.getFlags() & MessageFlags.SYNC_BARRIER) == MessageFlags.SYNC_BARRIER) {
            recvDType = MessageTypes.BYTE_ARRAY;
            recvKType = MessageTypes.EMPTY;
        }
        currentMessage = new InMessage(id, recvDType, this, header);
        if (isKeyed) {
            currentMessage.setKeyType(recvKType);
        }
        if (!currentMessage.addBufferAndCalculate(buffer)) {
            currentMessages.put(source, currentMessage);
        }
        // we add the message immediately to the deserialization as we can deserialize partially
        Queue<InMessage> deserializeQueue = pendingReceiveDeSerializations.get(source);
        if (!deserializeQueue.offer(currentMessage)) {
            throw new RuntimeException(executor + " We should have enough space: " + deserializeQueue.size());
        }
    } else {
        if (currentMessage.addBufferAndCalculate(buffer)) {
            currentMessages.remove(source);
        }
    }
}