Examples with MessageType edu.iu.dsc.tws.api.comms.messaging.types.MessageType used on opensource projects

Example 16 with MessageType

use of edu.iu.dsc.tws.api.comms.messaging.types.MessageType in project twister2 by DSC-SPIDAL.

the class ChannelDataFlowOperation method onReceiveComplete.

@Override
public void onReceiveComplete(int id, int e, DataBuffer buffer) {
    // we need to try to build the message here, we may need many more messages to complete
    ByteBuffer byteBuffer = buffer.getByteBuffer();
    byteBuffer.position(buffer.getSize());
    byteBuffer.flip();
    // we have the source of the message at 0th position as an integer
    int source = byteBuffer.getInt(0);
    InMessage currentMessage = currentMessages.get(source);
    if (currentMessage == null) {
        MessageHeader header = messageDeSerializer.get(source).buildHeader(buffer, e);
        MessageType recvDType = receiveDataType;
        MessageType recvKType = receiveKeyType;
        if ((header.getFlags() & MessageFlags.SYNC_BARRIER) == MessageFlags.SYNC_BARRIER) {
            recvDType = MessageTypes.BYTE_ARRAY;
            recvKType = MessageTypes.EMPTY;
        }
        currentMessage = new InMessage(id, recvDType, this, header);
        if (isKeyed) {
            currentMessage.setKeyType(recvKType);
        }
        if (!currentMessage.addBufferAndCalculate(buffer)) {
            currentMessages.put(source, currentMessage);
        }
        // we add the message immediately to the deserialization as we can deserialize partially
        Queue<InMessage> deserializeQueue = pendingReceiveDeSerializations.get(source);
        if (!deserializeQueue.offer(currentMessage)) {
            throw new RuntimeException(executor + " We should have enough space: " + deserializeQueue.size());
        }
    } else {
        if (currentMessage.addBufferAndCalculate(buffer)) {
            currentMessages.remove(source);
        }
    }
}

Example 17 with MessageType

use of edu.iu.dsc.tws.api.comms.messaging.types.MessageType in project twister2 by DSC-SPIDAL.

the class FixedSchemaKeyedDataDeSerializer method build.

/**
 * Builds the message from the data buffers in the partialObject. Since this method
 * supports multi-messages it iterates through the buffers and builds all the messages separately
 *
 * @param partialObject message object that needs to be built
 * @param edge the edge value associated with this message
 */
@Override
public void build(Object partialObject, int edge) {
    InMessage currentMessage = (InMessage) partialObject;
    MessageType keyType = currentMessage.getKeyType();
    DataPacker keyPacker = keyType.getDataPacker();
    DataPacker dataPacker = currentMessage.getDataType().getDataPacker();
    Queue<DataBuffer> buffers = currentMessage.getBuffers();
    MessageHeader header = currentMessage.getHeader();
    if (header == null) {
        throw new RuntimeException("Header must be built before the message");
    }
    List<DataBuffer> builtBuffers = new ArrayList<>();
    // get the number of objects deserialized
    DataBuffer buffer = buffers.peek();
    while (buffer != null) {
        int currentLocation = 0;
        int remaining = buffer.getSize();
        if (header.getNumberTuples() == 0) {
            builtBuffers.add(buffer);
            break;
        }
        // if we are at the beginning
        int currentObjectLength = currentMessage.getUnPkCurrentObjectLength();
        int currentKeyLength = currentMessage.getUnPkCurrentKeyLength();
        if (currentMessage.getUnPkBuffers() == 0) {
            currentLocation = DFWIOUtils.HEADER_SIZE;
            remaining = remaining - DFWIOUtils.HEADER_SIZE;
        } else {
            currentLocation = DFWIOUtils.SHORT_HEADER_SIZE;
            remaining = remaining - DFWIOUtils.SHORT_HEADER_SIZE;
        }
        if (currentObjectLength == -1 || currentMessage.getUnPkBuffers() == 0) {
            currentObjectLength = messageSchema.getMessageSize();
        }
        if (currentKeyLength == -1) {
            // we have to set the current object length
            currentObjectLength = messageSchema.getMessageSize() - messageSchema.getKeySize();
            currentKeyLength = messageSchema.getKeySize();
            currentMessage.getKeyBuilder().init(keyPacker, currentKeyLength);
            try {
                currentMessage.getDataBuilder().init(dataPacker, currentObjectLength);
                currentMessage.setUnPkCurrentKeyLength(currentKeyLength);
                currentMessage.setUnPkCurrentObjectLength(currentObjectLength);
                // we are going to read the key first
                currentMessage.setReadingKey(true);
            } catch (NegativeArraySizeException e) {
                throw new RuntimeException(e);
            }
        }
        while (remaining > 0) {
            if (currentMessage.isReadingKey()) {
                ObjectBuilderImpl keyBuilder = currentMessage.getKeyBuilder();
                int bytesRead = keyPacker.readDataFromBuffer(keyBuilder, currentLocation, buffer);
                keyBuilder.incrementCompletedSizeBy(bytesRead);
                currentLocation += bytesRead;
                remaining = remaining - bytesRead;
                currentMessage.setReadingKey(!keyBuilder.isBuilt());
                if (keyBuilder.isBuilt()) {
                    // done reading key
                    currentMessage.setReadingKey(false);
                } else {
                    break;
                }
            }
            if (!currentMessage.isReadingKey()) {
                ObjectBuilderImpl dataBuilder = currentMessage.getDataBuilder();
                // read the values from the buffer
                int byteRead = dataPacker.readDataFromBuffer(dataBuilder, currentLocation, buffer);
                dataBuilder.incrementCompletedSizeBy(byteRead);
                currentLocation += byteRead;
                remaining = remaining - byteRead;
                // okay we are done with this object
                if (dataBuilder.isBuilt()) {
                    // lets add the object
                    currentMessage.addCurrentKeyedObject();
                } else {
                    // lets break the inner while loop
                    break;
                }
                if (remaining > 0) {
                    currentObjectLength = messageSchema.getMessageSize();
                    // we have to set the current object length
                    currentObjectLength = currentObjectLength - messageSchema.getKeySize();
                    currentKeyLength = messageSchema.getKeySize();
                    currentMessage.getKeyBuilder().init(keyPacker, currentKeyLength);
                    currentMessage.getDataBuilder().init(dataPacker, currentObjectLength);
                    currentMessage.setUnPkCurrentKeyLength(currentKeyLength);
                    currentMessage.setUnPkCurrentObjectLength(currentObjectLength);
                    // we are going to read the key first
                    currentMessage.setReadingKey(true);
                } else {
                    // we have to break here as we cannot read further
                    break;
                }
            }
        }
        // lets remove this buffer
        buffers.poll();
        builtBuffers.add(buffer);
        // increment the unpacked buffers
        currentMessage.incrementUnPkBuffers();
        // lets check weather we have read everythong
        int readObjectNumber = currentMessage.getUnPkNumberObjects();
        // we need to get number of tuples and get abs because we are using -1 for single messages
        if (readObjectNumber == Math.abs(currentMessage.getHeader().getNumberTuples())) {
            break;
        }
        // lets move to next
        buffer = buffers.peek();
    }
    if (builtBuffers.size() > 0) {
        ChannelMessage channelMessage = new ChannelMessage(currentMessage.getOriginatingId(), currentMessage.getDataType(), MessageDirection.IN, currentMessage.getReleaseListener());
        channelMessage.addBuffers(builtBuffers);
        channelMessage.setHeader(currentMessage.getHeader());
        channelMessage.incrementRefCount();
        currentMessage.addBuiltMessage(channelMessage);
    }
}

Example 18 with MessageType

use of edu.iu.dsc.tws.api.comms.messaging.types.MessageType in project twister2 by DSC-SPIDAL.

the class KeyedDataDeSerializer method build.

/**
 * Builds the message from the data buffers in the partialObject. Since this method
 * supports multi-messages it iterates through the buffers and builds all the messages separately
 *
 * @param partialObject message object that needs to be built
 * @param edge the edge value associated with this message
 */
@Override
public void build(Object partialObject, int edge) {
    InMessage currentMessage = (InMessage) partialObject;
    MessageType keyType = currentMessage.getKeyType();
    DataPacker keyPacker = keyType.getDataPacker();
    DataPacker dataPacker = currentMessage.getDataType().getDataPacker();
    Queue<DataBuffer> buffers = currentMessage.getBuffers();
    MessageHeader header = currentMessage.getHeader();
    if (header == null) {
        throw new RuntimeException("Header must be built before the message");
    }
    if ((header.getFlags() & MessageFlags.SYNC_BARRIER) == MessageFlags.SYNC_BARRIER) {
        keyType = MessageTypes.BYTE_ARRAY;
        keyPacker = MessageTypes.BYTE_ARRAY.getDataPacker();
        dataPacker = keyPacker;
    }
    List<DataBuffer> builtBuffers = new ArrayList<>();
    // get the number of objects deserialized
    DataBuffer buffer = buffers.peek();
    while (buffer != null) {
        int currentLocation = 0;
        int remaining = buffer.getSize();
        if (header.getNumberTuples() == 0) {
            builtBuffers.add(buffer);
            break;
        }
        // if we are at the beginning
        int currentObjectLength = currentMessage.getUnPkCurrentObjectLength();
        int currentKeyLength = currentMessage.getUnPkCurrentKeyLength();
        if (currentMessage.getUnPkBuffers() == 0) {
            currentLocation = DFWIOUtils.HEADER_SIZE;
            remaining = remaining - DFWIOUtils.HEADER_SIZE;
        } else {
            currentLocation = DFWIOUtils.SHORT_HEADER_SIZE;
            remaining = remaining - DFWIOUtils.SHORT_HEADER_SIZE;
        }
        if (currentObjectLength == -1 || currentMessage.getUnPkBuffers() == 0) {
            currentObjectLength = buffer.getByteBuffer().getInt(currentLocation);
            remaining = remaining - Integer.BYTES;
            currentLocation += Integer.BYTES;
        }
        if (currentKeyLength == -1) {
            // we assume we can read the key length from here
            int right = DataPackerProxy.getKeyLengthRight(keyType, buffer, currentLocation);
            int left = DataPackerProxy.getKeyLengthLeft(keyType, buffer, currentLocation);
            remaining = remaining - right;
            currentLocation += right;
            // we have to set the current object length
            currentObjectLength = currentObjectLength - left - right;
            currentKeyLength = left;
            currentMessage.getKeyBuilder().init(keyPacker, currentKeyLength);
            try {
                currentMessage.getDataBuilder().init(dataPacker, currentObjectLength);
                currentMessage.setUnPkCurrentKeyLength(currentKeyLength);
                currentMessage.setUnPkCurrentObjectLength(currentObjectLength);
                // we are going to read the key first
                currentMessage.setReadingKey(true);
            } catch (NegativeArraySizeException e) {
                throw new RuntimeException(e);
            }
        }
        while (remaining > 0) {
            if (currentMessage.isReadingKey()) {
                ObjectBuilderImpl keyBuilder = currentMessage.getKeyBuilder();
                int bytesRead = keyPacker.readDataFromBuffer(keyBuilder, currentLocation, buffer);
                keyBuilder.incrementCompletedSizeBy(bytesRead);
                currentLocation += bytesRead;
                remaining = remaining - bytesRead;
                currentMessage.setReadingKey(!keyBuilder.isBuilt());
                if (keyBuilder.isBuilt()) {
                    // done reading key
                    currentMessage.setReadingKey(false);
                } else {
                    break;
                }
            }
            if (!currentMessage.isReadingKey()) {
                ObjectBuilderImpl dataBuilder = currentMessage.getDataBuilder();
                // read the values from the buffer
                int byteRead = dataPacker.readDataFromBuffer(dataBuilder, currentLocation, buffer);
                dataBuilder.incrementCompletedSizeBy(byteRead);
                currentLocation += byteRead;
                remaining = remaining - byteRead;
                // okay we are done with this object
                if (dataBuilder.isBuilt()) {
                    // lets add the object
                    currentMessage.addCurrentKeyedObject();
                } else {
                    // lets break the inner while loop
                    break;
                }
                int bytesToReadKey = 0;
                if (keyPacker.isHeaderRequired()) {
                    bytesToReadKey += Integer.BYTES;
                }
                if (remaining >= Integer.BYTES + bytesToReadKey) {
                    currentObjectLength = buffer.getByteBuffer().getInt(currentLocation);
                    remaining = remaining - Integer.BYTES;
                    currentLocation += Integer.BYTES;
                    // we assume we can read the key length from here
                    int right = DataPackerProxy.getKeyLengthRight(keyType, buffer, currentLocation);
                    int left = DataPackerProxy.getKeyLengthLeft(keyType, buffer, currentLocation);
                    remaining = remaining - right;
                    currentLocation += right;
                    // we have to set the current object length
                    currentObjectLength = currentObjectLength - left - right;
                    currentKeyLength = left;
                    currentMessage.getKeyBuilder().init(keyPacker, currentKeyLength);
                    currentMessage.getDataBuilder().init(dataPacker, currentObjectLength);
                    currentMessage.setUnPkCurrentKeyLength(currentKeyLength);
                    currentMessage.setUnPkCurrentObjectLength(currentObjectLength);
                    // we are going to read the key first
                    currentMessage.setReadingKey(true);
                } else if (remaining >= Integer.BYTES) {
                    currentObjectLength = buffer.getByteBuffer().getInt(currentLocation);
                    remaining = remaining - Integer.BYTES;
                    currentLocation += Integer.BYTES;
                    currentMessage.setUnPkCurrentObjectLength(currentObjectLength);
                    currentMessage.setUnPkCurrentKeyLength(-1);
                    currentMessage.setReadingKey(true);
                } else {
                    // we have to break here as we cannot read further
                    break;
                }
            }
        }
        // lets remove this buffer
        buffers.poll();
        builtBuffers.add(buffer);
        // increment the unpacked buffers
        currentMessage.incrementUnPkBuffers();
        // lets check weather we have read everythong
        int readObjectNumber = currentMessage.getUnPkNumberObjects();
        // we need to get number of tuples and get abs because we are using -1 for single messages
        if (readObjectNumber == Math.abs(currentMessage.getHeader().getNumberTuples())) {
            break;
        }
        // lets move to next
        buffer = buffers.peek();
    }
    if (builtBuffers.size() > 0) {
        ChannelMessage channelMessage = new ChannelMessage(currentMessage.getOriginatingId(), currentMessage.getDataType(), MessageDirection.IN, currentMessage.getReleaseListener());
        channelMessage.addBuffers(builtBuffers);
        channelMessage.setHeader(currentMessage.getHeader());
        channelMessage.incrementRefCount();
        currentMessage.addBuiltMessage(channelMessage);
    }
}

Example 19 with MessageType

use of edu.iu.dsc.tws.api.comms.messaging.types.MessageType in project twister2 by DSC-SPIDAL.

the class HashJoinUtils method leftOuterJoin.

public static List<Object> leftOuterJoin(List<Tuple> leftRelation, List<Tuple> rightRelation, MessageType messageType) {
    Map<Object, List<Tuple>> rightHash = new THashMap<>(messageType);
    List<Object> joinedTuples = new ArrayList<>();
    for (Tuple tuple : rightRelation) {
        rightHash.computeIfAbsent(tuple.getKey(), k -> new ArrayList<>()).add(tuple);
    }
    for (Tuple leftTuple : leftRelation) {
        List<Tuple> rightTuples = rightHash.getOrDefault(leftTuple.getKey(), Collections.emptyList());
        for (Tuple rightTuple : rightTuples) {
            joinedTuples.add(JoinedTuple.of(leftTuple.getKey(), leftTuple.getValue(), rightTuple.getValue()));
        }
        if (rightTuples.isEmpty()) {
            joinedTuples.add(JoinedTuple.of(leftTuple.getKey(), leftTuple.getValue(), null));
        }
    }
    return joinedTuples;
}

Example 20 with MessageType

use of edu.iu.dsc.tws.api.comms.messaging.types.MessageType in project twister2 by DSC-SPIDAL.

the class HashJoinUtils method join.

/**
 * Disk based inner join
 */
public static Iterator<JoinedTuple> join(ResettableIterator<Tuple<?, ?>> leftIt, ResettableIterator<Tuple<?, ?>> rightIt, CommunicationContext.JoinType joinType, MessageType keyType) {
    // choosing hashing and probing relations
    // if inner join:
    // hashing = left
    // probing = right
    // if left join:
    // hashing = right
    // probing = left
    // if right join:
    // hashing = left
    // probing = right
    final ResettableIterator<Tuple<?, ?>> hashingRelation = joinType.equals(CommunicationContext.JoinType.LEFT) ? rightIt : leftIt;
    final ResettableIterator<Tuple<?, ?>> probingRelation = joinType.equals(CommunicationContext.JoinType.LEFT) ? leftIt : rightIt;
    // set the memory limits based on the heap allocation
    final double lowerMemoryBound = Runtime.getRuntime().totalMemory() * 0.1;
    return new Iterator<JoinedTuple>() {

        private boolean hashingDone;

        private Map<Object, List> keyHash = new THashMap<>(keyType);

        // always keep the nextJoinTuple in memory. hasNext() will use this field
        private JoinedTuple nextJoinTuple;

        /**
         * This method will perform following actions in order
         * <ol>
         *   <li>Clear existing HashMap</li>
         *   <li>Create HashMap from the hashingRelation till it hit the memory limits</li>
         *   <li>Determine whether the hashingRelation is fully consumed</li>
         * </ol>
         */
        private void doHashing() {
            this.keyHash.clear();
            // building the hash, as long as memory permits
            while (Runtime.getRuntime().freeMemory() > lowerMemoryBound && hashingRelation.hasNext()) {
                Tuple<?, ?> nextLeft = hashingRelation.next();
                keyHash.computeIfAbsent(nextLeft.getKey(), k -> new ArrayList()).add(nextLeft.getValue());
            }
            // determine whether hashRelation is fully consumed
            hashingDone = !hashingRelation.hasNext();
            if (!hashingDone && this.keyHash.isEmpty()) {
                // problem!. We have cleared the old hash, yet there's no free memory available to proceed
                throw new Twister2RuntimeException("Couldn't progress due to memory limitations." + "Available free memory : " + Runtime.getRuntime().freeMemory() + ", Expected free memory : " + lowerMemoryBound);
            }
        }

        {
            // initially do hashing & probing
            doHashing();
            doProbing();
        }

        // when iterating over the right(probing) relation, current element
        // (which has been returned by next()) will be kept in memory since it should be combined
        // with all the tuples in leftListForCurrentKey. But this has to be done on demand, on next()
        // call of joined iterator.
        private Tuple<?, ?> currentProbingTuple;

        // list of tuples from left relation(hashing relation),
        // that matches with the currentRightTuple
        private List leftListForCurrentKey;

        // keeping the index of leftListForCurrentKey
        private int leftListIndex = 0;

        /**
         * This method should be guaranteed to create a {@link JoinedTuple}. If a tuple can't be
         * created, caller should determine that before calling this method.
         * Additionally, this method should clear everything if everything related to
         * currentRightTuple is processed.
         */
        private void progressProbing() {
            Object key = this.currentProbingTuple.getKey();
            // we have interchanged original iterators based on the join type.
            // that should be taken into consideration when creating the JoinedTuple
            Object left = joinType.equals(CommunicationContext.JoinType.LEFT) ? this.currentProbingTuple.getValue() : leftListForCurrentKey.get(leftListIndex);
            Object right = joinType.equals(CommunicationContext.JoinType.LEFT) ? leftListForCurrentKey.get(leftListIndex) : this.currentProbingTuple.getValue();
            this.nextJoinTuple = JoinedTuple.of(key, left, right);
            leftListIndex++;
            // if end of the list has reached, reset everything!
            if (leftListIndex == leftListForCurrentKey.size()) {
                currentProbingTuple = null;
                leftListForCurrentKey = null;
                leftListIndex = 0;
            }
        }

        /**
         * This method iterates through the right relation(probing relation).
         */
        private void doProbing() {
            // if there is a non null nextJoinTuple, no need of proceeding
            while (this.nextJoinTuple == null) {
                // hashed list and still in the middle of combining that list
                if (this.currentProbingTuple == null) {
                    if (probingRelation.hasNext()) {
                        this.currentProbingTuple = probingRelation.next();
                        this.leftListForCurrentKey = this.keyHash.get(currentProbingTuple.getKey());
                        if (this.leftListForCurrentKey == null) {
                            // handle left and right joins here
                            if (joinType.equals(CommunicationContext.JoinType.LEFT)) {
                                this.nextJoinTuple = JoinedTuple.of(currentProbingTuple.getKey(), currentProbingTuple.getValue(), null);
                            } else if (joinType.equals(CommunicationContext.JoinType.RIGHT)) {
                                this.nextJoinTuple = JoinedTuple.of(currentProbingTuple.getKey(), null, currentProbingTuple.getValue());
                            }
                            // any join : We are done with currentProbingTuple
                            this.currentProbingTuple = null;
                        } else {
                            progressProbing();
                        }
                    } else {
                        // right iterator has reached to an end for current HashMap.
                        if (!hashingDone) {
                            // clear current hash and reset the right iterator
                            doHashing();
                            probingRelation.reset();
                        } else {
                            // end of join operation. Yay!
                            break;
                        }
                    }
                } else {
                    progressProbing();
                }
            }
        }

        @Override
        public boolean hasNext() {
            return this.nextJoinTuple != null;
        }

        @Override
        public JoinedTuple next() {
            if (!hasNext()) {
                throw new Twister2RuntimeException("Join operation has reached to an end. " + "Use hasNext() to check the status.");
            }
            JoinedTuple currentJoinTuple = nextJoinTuple;
            nextJoinTuple = null;
            // create the next JoinTuple before returning
            doProbing();
            return currentJoinTuple;
        }
    };
}