Examples with ChannelMessage edu.iu.dsc.tws.api.comms.messaging.ChannelMessage used on opensource projects

Example 11 with ChannelMessage

use of edu.iu.dsc.tws.api.comms.messaging.ChannelMessage in project twister2 by DSC-SPIDAL.

the class ChannelDataFlowOperation method sendProgress.

/**
 * Go through the out messages, create channel messages by using the serializer send them
 *
 * @param pendingSendMessages the pending message queue
 * @param sendId send target
 */
private void sendProgress(Queue<OutMessage> pendingSendMessages, int sendId) {
    boolean canProgress = true;
    while (pendingSendMessages.size() > 0 && canProgress) {
        // take out pending messages
        OutMessage outMessage = pendingSendMessages.peek();
        Object data = outMessage.getData();
        // first lets send the message to internal destinations
        canProgress = sendInternally(outMessage, data);
        if (canProgress) {
            // we don't have an external executor to send this message
            if (outMessage.getExternalSends().size() == 0) {
                pendingSendMessages.poll();
                continue;
            }
            Queue<ChannelMessage> channelMessages = outMessage.getChannelMessages();
            // at this point lets build the message
            ChannelMessage serializeMessage = (ChannelMessage) messageSerializer.get(sendId).build(outMessage.getData(), outMessage);
            if (serializeMessage != null) {
                // we are incrementing the reference count here
                channelMessages.offer(serializeMessage);
            }
            ChannelMessage chMessage = channelMessages.peek();
            if (chMessage == null) {
                break;
            }
            List<Integer> externalRoutes = outMessage.getExternalSends();
            // okay we build the message, send it
            if (outMessage.getSendState() == OutMessage.SendState.SERIALIZED) {
                int startOfExternalRouts = chMessage.getAcceptedExternalSends();
                canProgress = sendExternally(outMessage, chMessage, externalRoutes, startOfExternalRouts);
                if (chMessage.getAcceptedExternalSends() == externalRoutes.size()) {
                    // we are done
                    pendingSendMessages.poll();
                    channelMessages.poll();
                    // the send is completed, we need to notify
                    receiver.sendCompleted(outMessage);
                }
            } else if (outMessage.getSendState() == OutMessage.SendState.PARTIALLY_SERIALIZED) {
                int startOfExternalRouts = chMessage.getAcceptedExternalSends();
                canProgress = sendExternally(outMessage, chMessage, externalRoutes, startOfExternalRouts);
                if (chMessage.getAcceptedExternalSends() == externalRoutes.size()) {
                    // we are done sending this channel message
                    channelMessages.poll();
                }
            } else {
                break;
            }
        }
    }
}

Example 12 with ChannelMessage

use of edu.iu.dsc.tws.api.comms.messaging.ChannelMessage in project twister2 by DSC-SPIDAL.

the class BaseSerializer method build.

@Override
public ChannelMessage build(Object data, Object partialBuildObject) {
    OutMessage sendMessage = (OutMessage) partialBuildObject;
    ChannelMessage channelMessage = new ChannelMessage(sendMessage.getSource(), sendMessage.getDataType(), MessageDirection.OUT, sendMessage.getReleaseCallback());
    buildHeader(sendMessage, channelMessage, 0);
    // we loop until everything is serialized
    while (sendBuffers.size() > 0 && sendMessage.getSendState() != OutMessage.SendState.SERIALIZED) {
        // we can continue only if there is a data buffer
        DataBuffer buffer = sendBuffers.poll();
        if (buffer == null) {
            break;
        }
        int lastBufferIndicatorIndex = DFWIOUtils.SHORT_HEADER_SIZE - 1;
        // this is the first time we are seeing this message
        if (sendMessage.getSendState() == OutMessage.SendState.INIT || sendMessage.getSendState() == OutMessage.SendState.SENT_INTERNALLY) {
            // we set the state here, because we can set it to serialized below
            sendMessage.setSendState(OutMessage.SendState.HEADER_BUILT);
            // build the header
            if (data instanceof AggregatedObjects) {
                // for list message we need to put the size of the list
                DFWIOUtils.buildHeader(buffer, sendMessage, ((List) data).size());
                buildHeader(sendMessage, channelMessage, ((List) data).size());
            } else {
                if ((sendMessage.getFlags() & MessageFlags.SYNC_EMPTY) == MessageFlags.SYNC_EMPTY) {
                    sendMessage.setSendState(OutMessage.SendState.SERIALIZED);
                    // we set the number of messages to 0, only header will be sent
                    DFWIOUtils.buildHeader(buffer, sendMessage, 0);
                    buildHeader(sendMessage, channelMessage, 0);
                } else {
                    // for single message we need to put the size as 1
                    DFWIOUtils.buildHeader(buffer, sendMessage, -1);
                    buildHeader(sendMessage, channelMessage, -1);
                }
            }
            lastBufferIndicatorIndex = DFWIOUtils.HEADER_SIZE - 1;
        } else {
            buffer.getByteBuffer().putInt(sendMessage.getSource());
            // indicate this is not the last buffer
            buffer.getByteBuffer().put((byte) 0);
        }
        // okay we have a body to build and it is not done fully yet
        if (sendMessage.getSendState() == OutMessage.SendState.HEADER_BUILT || sendMessage.getSendState() == OutMessage.SendState.PARTIALLY_SERIALIZED) {
            sendMessage.setSendState(OutMessage.SendState.PARTIALLY_SERIALIZED);
            serializeBody(data, sendMessage, buffer);
        }
        if (sendMessage.getSendState() == OutMessage.SendState.SERIALIZED) {
            channelMessage.setComplete(true);
            // indicate this as the final buffer
            buffer.getByteBuffer().put(lastBufferIndicatorIndex, (byte) 1);
        }
        // okay we are adding this buffer
        channelMessage.addBuffer(buffer);
    }
    // if we didn't do anything lets return null
    if (channelMessage.getBuffers().size() == 0) {
        return null;
    }
    return channelMessage;
}

Example 13 with ChannelMessage

use of edu.iu.dsc.tws.api.comms.messaging.ChannelMessage in project twister2 by DSC-SPIDAL.

the class DataDeserializer method build.

/**
 * Builds the message from the data buffers in the partialObject. Since this method
 * supports multi-messages it iterates through the buffers and builds all the messages separately
 *
 * @param partialObject message object that needs to be built
 * @param edge the edge value associated with this message
 */
@Override
public void build(Object partialObject, int edge) {
    InMessage currentMessage = (InMessage) partialObject;
    DataPacker dataPacker = currentMessage.getDataType().getDataPacker();
    Queue<DataBuffer> buffers = currentMessage.getBuffers();
    MessageHeader header = currentMessage.getHeader();
    if (header == null) {
        throw new RuntimeException("Header must be built before the message");
    }
    List<DataBuffer> builtBuffers = new ArrayList<>();
    // get the number of objects deserialized
    DataBuffer buffer = buffers.peek();
    while (buffer != null) {
        int currentLocation = 0;
        int remaining = buffer.getSize();
        if (header.getNumberTuples() == 0) {
            builtBuffers.add(buffer);
            break;
        }
        // if we are at the beginning
        int currentObjectLength = currentMessage.getUnPkCurrentObjectLength();
        if (currentMessage.getUnPkBuffers() == 0) {
            currentLocation = DFWIOUtils.HEADER_SIZE;
            remaining = remaining - DFWIOUtils.HEADER_SIZE;
        } else {
            // source(int) + last_buffer_indicator(byte)
            currentLocation = DFWIOUtils.SHORT_HEADER_SIZE;
            remaining = remaining - DFWIOUtils.SHORT_HEADER_SIZE;
        }
        if (currentObjectLength == -1 || currentMessage.getUnPkBuffers() == 0) {
            currentObjectLength = buffer.getByteBuffer().getInt(currentLocation);
            remaining = remaining - Integer.BYTES;
            currentLocation += Integer.BYTES;
            // starting to build a new object
            currentMessage.getDataBuilder().init(dataPacker, currentObjectLength);
        }
        while (remaining > 0) {
            // read the values from the buffer
            ObjectBuilderImpl dataBuilder = currentMessage.getDataBuilder();
            int bytesRead = dataPacker.readDataFromBuffer(dataBuilder, currentLocation, buffer);
            dataBuilder.incrementCompletedSizeBy(bytesRead);
            currentLocation += bytesRead;
            remaining = remaining - bytesRead;
            // okay we are done with this object
            if (dataBuilder.isBuilt()) {
                currentMessage.addCurrentObject();
                currentMessage.setUnPkCurrentObjectLength(-1);
            } else {
                // lets break the inner while loop
                break;
            }
            // could have next object length?
            if (remaining >= Integer.BYTES) {
                currentObjectLength = buffer.getByteBuffer().getInt(currentLocation);
                remaining = remaining - Integer.BYTES;
                currentLocation += Integer.BYTES;
                currentMessage.getDataBuilder().init(dataPacker, currentObjectLength);
                currentMessage.setUnPkCurrentObjectLength(currentObjectLength);
            } else {
                // we have to break here as we cannot read further
                break;
            }
        }
        // lets remove this buffer
        buffers.poll();
        builtBuffers.add(buffer);
        // increment the unpacked buffers
        currentMessage.incrementUnPkBuffers();
        // lets check weather we have read everythong
        int readObjectNumber = currentMessage.getUnPkNumberObjects();
        // we need to get number of tuples and get abs because we are using -1 for single messages
        if (readObjectNumber == Math.abs(currentMessage.getHeader().getNumberTuples())) {
            break;
        }
        // lets move to next
        buffer = buffers.peek();
    }
    if (builtBuffers.size() > 0) {
        ChannelMessage channelMessage = new ChannelMessage(currentMessage.getOriginatingId(), currentMessage.getDataType(), MessageDirection.IN, currentMessage.getReleaseListener());
        channelMessage.addBuffers(builtBuffers);
        channelMessage.setHeader(currentMessage.getHeader());
        channelMessage.incrementRefCount();
        currentMessage.addBuiltMessage(channelMessage);
    }
}

Example 14 with ChannelMessage

use of edu.iu.dsc.tws.api.comms.messaging.ChannelMessage in project twister2 by DSC-SPIDAL.

the class FixedSchemaKeyedDataDeSerializer method build.

/**
 * Builds the message from the data buffers in the partialObject. Since this method
 * supports multi-messages it iterates through the buffers and builds all the messages separately
 *
 * @param partialObject message object that needs to be built
 * @param edge the edge value associated with this message
 */
@Override
public void build(Object partialObject, int edge) {
    InMessage currentMessage = (InMessage) partialObject;
    MessageType keyType = currentMessage.getKeyType();
    DataPacker keyPacker = keyType.getDataPacker();
    DataPacker dataPacker = currentMessage.getDataType().getDataPacker();
    Queue<DataBuffer> buffers = currentMessage.getBuffers();
    MessageHeader header = currentMessage.getHeader();
    if (header == null) {
        throw new RuntimeException("Header must be built before the message");
    }
    List<DataBuffer> builtBuffers = new ArrayList<>();
    // get the number of objects deserialized
    DataBuffer buffer = buffers.peek();
    while (buffer != null) {
        int currentLocation = 0;
        int remaining = buffer.getSize();
        if (header.getNumberTuples() == 0) {
            builtBuffers.add(buffer);
            break;
        }
        // if we are at the beginning
        int currentObjectLength = currentMessage.getUnPkCurrentObjectLength();
        int currentKeyLength = currentMessage.getUnPkCurrentKeyLength();
        if (currentMessage.getUnPkBuffers() == 0) {
            currentLocation = DFWIOUtils.HEADER_SIZE;
            remaining = remaining - DFWIOUtils.HEADER_SIZE;
        } else {
            currentLocation = DFWIOUtils.SHORT_HEADER_SIZE;
            remaining = remaining - DFWIOUtils.SHORT_HEADER_SIZE;
        }
        if (currentObjectLength == -1 || currentMessage.getUnPkBuffers() == 0) {
            currentObjectLength = messageSchema.getMessageSize();
        }
        if (currentKeyLength == -1) {
            // we have to set the current object length
            currentObjectLength = messageSchema.getMessageSize() - messageSchema.getKeySize();
            currentKeyLength = messageSchema.getKeySize();
            currentMessage.getKeyBuilder().init(keyPacker, currentKeyLength);
            try {
                currentMessage.getDataBuilder().init(dataPacker, currentObjectLength);
                currentMessage.setUnPkCurrentKeyLength(currentKeyLength);
                currentMessage.setUnPkCurrentObjectLength(currentObjectLength);
                // we are going to read the key first
                currentMessage.setReadingKey(true);
            } catch (NegativeArraySizeException e) {
                throw new RuntimeException(e);
            }
        }
        while (remaining > 0) {
            if (currentMessage.isReadingKey()) {
                ObjectBuilderImpl keyBuilder = currentMessage.getKeyBuilder();
                int bytesRead = keyPacker.readDataFromBuffer(keyBuilder, currentLocation, buffer);
                keyBuilder.incrementCompletedSizeBy(bytesRead);
                currentLocation += bytesRead;
                remaining = remaining - bytesRead;
                currentMessage.setReadingKey(!keyBuilder.isBuilt());
                if (keyBuilder.isBuilt()) {
                    // done reading key
                    currentMessage.setReadingKey(false);
                } else {
                    break;
                }
            }
            if (!currentMessage.isReadingKey()) {
                ObjectBuilderImpl dataBuilder = currentMessage.getDataBuilder();
                // read the values from the buffer
                int byteRead = dataPacker.readDataFromBuffer(dataBuilder, currentLocation, buffer);
                dataBuilder.incrementCompletedSizeBy(byteRead);
                currentLocation += byteRead;
                remaining = remaining - byteRead;
                // okay we are done with this object
                if (dataBuilder.isBuilt()) {
                    // lets add the object
                    currentMessage.addCurrentKeyedObject();
                } else {
                    // lets break the inner while loop
                    break;
                }
                if (remaining > 0) {
                    currentObjectLength = messageSchema.getMessageSize();
                    // we have to set the current object length
                    currentObjectLength = currentObjectLength - messageSchema.getKeySize();
                    currentKeyLength = messageSchema.getKeySize();
                    currentMessage.getKeyBuilder().init(keyPacker, currentKeyLength);
                    currentMessage.getDataBuilder().init(dataPacker, currentObjectLength);
                    currentMessage.setUnPkCurrentKeyLength(currentKeyLength);
                    currentMessage.setUnPkCurrentObjectLength(currentObjectLength);
                    // we are going to read the key first
                    currentMessage.setReadingKey(true);
                } else {
                    // we have to break here as we cannot read further
                    break;
                }
            }
        }
        // lets remove this buffer
        buffers.poll();
        builtBuffers.add(buffer);
        // increment the unpacked buffers
        currentMessage.incrementUnPkBuffers();
        // lets check weather we have read everythong
        int readObjectNumber = currentMessage.getUnPkNumberObjects();
        // we need to get number of tuples and get abs because we are using -1 for single messages
        if (readObjectNumber == Math.abs(currentMessage.getHeader().getNumberTuples())) {
            break;
        }
        // lets move to next
        buffer = buffers.peek();
    }
    if (builtBuffers.size() > 0) {
        ChannelMessage channelMessage = new ChannelMessage(currentMessage.getOriginatingId(), currentMessage.getDataType(), MessageDirection.IN, currentMessage.getReleaseListener());
        channelMessage.addBuffers(builtBuffers);
        channelMessage.setHeader(currentMessage.getHeader());
        channelMessage.incrementRefCount();
        currentMessage.addBuiltMessage(channelMessage);
    }
}

Example 15 with ChannelMessage

use of edu.iu.dsc.tws.api.comms.messaging.ChannelMessage in project twister2 by DSC-SPIDAL.

the class KeyedDataDeSerializer method build.

/**
 * Builds the message from the data buffers in the partialObject. Since this method
 * supports multi-messages it iterates through the buffers and builds all the messages separately
 *
 * @param partialObject message object that needs to be built
 * @param edge the edge value associated with this message
 */
@Override
public void build(Object partialObject, int edge) {
    InMessage currentMessage = (InMessage) partialObject;
    MessageType keyType = currentMessage.getKeyType();
    DataPacker keyPacker = keyType.getDataPacker();
    DataPacker dataPacker = currentMessage.getDataType().getDataPacker();
    Queue<DataBuffer> buffers = currentMessage.getBuffers();
    MessageHeader header = currentMessage.getHeader();
    if (header == null) {
        throw new RuntimeException("Header must be built before the message");
    }
    if ((header.getFlags() & MessageFlags.SYNC_BARRIER) == MessageFlags.SYNC_BARRIER) {
        keyType = MessageTypes.BYTE_ARRAY;
        keyPacker = MessageTypes.BYTE_ARRAY.getDataPacker();
        dataPacker = keyPacker;
    }
    List<DataBuffer> builtBuffers = new ArrayList<>();
    // get the number of objects deserialized
    DataBuffer buffer = buffers.peek();
    while (buffer != null) {
        int currentLocation = 0;
        int remaining = buffer.getSize();
        if (header.getNumberTuples() == 0) {
            builtBuffers.add(buffer);
            break;
        }
        // if we are at the beginning
        int currentObjectLength = currentMessage.getUnPkCurrentObjectLength();
        int currentKeyLength = currentMessage.getUnPkCurrentKeyLength();
        if (currentMessage.getUnPkBuffers() == 0) {
            currentLocation = DFWIOUtils.HEADER_SIZE;
            remaining = remaining - DFWIOUtils.HEADER_SIZE;
        } else {
            currentLocation = DFWIOUtils.SHORT_HEADER_SIZE;
            remaining = remaining - DFWIOUtils.SHORT_HEADER_SIZE;
        }
        if (currentObjectLength == -1 || currentMessage.getUnPkBuffers() == 0) {
            currentObjectLength = buffer.getByteBuffer().getInt(currentLocation);
            remaining = remaining - Integer.BYTES;
            currentLocation += Integer.BYTES;
        }
        if (currentKeyLength == -1) {
            // we assume we can read the key length from here
            int right = DataPackerProxy.getKeyLengthRight(keyType, buffer, currentLocation);
            int left = DataPackerProxy.getKeyLengthLeft(keyType, buffer, currentLocation);
            remaining = remaining - right;
            currentLocation += right;
            // we have to set the current object length
            currentObjectLength = currentObjectLength - left - right;
            currentKeyLength = left;
            currentMessage.getKeyBuilder().init(keyPacker, currentKeyLength);
            try {
                currentMessage.getDataBuilder().init(dataPacker, currentObjectLength);
                currentMessage.setUnPkCurrentKeyLength(currentKeyLength);
                currentMessage.setUnPkCurrentObjectLength(currentObjectLength);
                // we are going to read the key first
                currentMessage.setReadingKey(true);
            } catch (NegativeArraySizeException e) {
                throw new RuntimeException(e);
            }
        }
        while (remaining > 0) {
            if (currentMessage.isReadingKey()) {
                ObjectBuilderImpl keyBuilder = currentMessage.getKeyBuilder();
                int bytesRead = keyPacker.readDataFromBuffer(keyBuilder, currentLocation, buffer);
                keyBuilder.incrementCompletedSizeBy(bytesRead);
                currentLocation += bytesRead;
                remaining = remaining - bytesRead;
                currentMessage.setReadingKey(!keyBuilder.isBuilt());
                if (keyBuilder.isBuilt()) {
                    // done reading key
                    currentMessage.setReadingKey(false);
                } else {
                    break;
                }
            }
            if (!currentMessage.isReadingKey()) {
                ObjectBuilderImpl dataBuilder = currentMessage.getDataBuilder();
                // read the values from the buffer
                int byteRead = dataPacker.readDataFromBuffer(dataBuilder, currentLocation, buffer);
                dataBuilder.incrementCompletedSizeBy(byteRead);
                currentLocation += byteRead;
                remaining = remaining - byteRead;
                // okay we are done with this object
                if (dataBuilder.isBuilt()) {
                    // lets add the object
                    currentMessage.addCurrentKeyedObject();
                } else {
                    // lets break the inner while loop
                    break;
                }
                int bytesToReadKey = 0;
                if (keyPacker.isHeaderRequired()) {
                    bytesToReadKey += Integer.BYTES;
                }
                if (remaining >= Integer.BYTES + bytesToReadKey) {
                    currentObjectLength = buffer.getByteBuffer().getInt(currentLocation);
                    remaining = remaining - Integer.BYTES;
                    currentLocation += Integer.BYTES;
                    // we assume we can read the key length from here
                    int right = DataPackerProxy.getKeyLengthRight(keyType, buffer, currentLocation);
                    int left = DataPackerProxy.getKeyLengthLeft(keyType, buffer, currentLocation);
                    remaining = remaining - right;
                    currentLocation += right;
                    // we have to set the current object length
                    currentObjectLength = currentObjectLength - left - right;
                    currentKeyLength = left;
                    currentMessage.getKeyBuilder().init(keyPacker, currentKeyLength);
                    currentMessage.getDataBuilder().init(dataPacker, currentObjectLength);
                    currentMessage.setUnPkCurrentKeyLength(currentKeyLength);
                    currentMessage.setUnPkCurrentObjectLength(currentObjectLength);
                    // we are going to read the key first
                    currentMessage.setReadingKey(true);
                } else if (remaining >= Integer.BYTES) {
                    currentObjectLength = buffer.getByteBuffer().getInt(currentLocation);
                    remaining = remaining - Integer.BYTES;
                    currentLocation += Integer.BYTES;
                    currentMessage.setUnPkCurrentObjectLength(currentObjectLength);
                    currentMessage.setUnPkCurrentKeyLength(-1);
                    currentMessage.setReadingKey(true);
                } else {
                    // we have to break here as we cannot read further
                    break;
                }
            }
        }
        // lets remove this buffer
        buffers.poll();
        builtBuffers.add(buffer);
        // increment the unpacked buffers
        currentMessage.incrementUnPkBuffers();
        // lets check weather we have read everythong
        int readObjectNumber = currentMessage.getUnPkNumberObjects();
        // we need to get number of tuples and get abs because we are using -1 for single messages
        if (readObjectNumber == Math.abs(currentMessage.getHeader().getNumberTuples())) {
            break;
        }
        // lets move to next
        buffer = buffers.peek();
    }
    if (builtBuffers.size() > 0) {
        ChannelMessage channelMessage = new ChannelMessage(currentMessage.getOriginatingId(), currentMessage.getDataType(), MessageDirection.IN, currentMessage.getReleaseListener());
        channelMessage.addBuffers(builtBuffers);
        channelMessage.setHeader(currentMessage.getHeader());
        channelMessage.incrementRefCount();
        currentMessage.addBuiltMessage(channelMessage);
    }
}