Examples with MPIMessage edu.iu.dsc.tws.comms.mpi.MPIMessage used on opensource projects

Example 1 with MPIMessage

use of edu.iu.dsc.tws.comms.mpi.MPIMessage in project twister2 by DSC-SPIDAL.

the class MPIMessageDeSerializer method getDataBuffers.

@Override
@SuppressWarnings("unchecked")
public Object getDataBuffers(Object partialObject, int edge) {
    MPIMessage message = (MPIMessage) partialObject;
    MessageType type = message.getType();
    // Used when handling multi messages
    List<ImmutablePair<byte[], byte[]>> results;
    if (!keyed) {
        return DataDeserializer.getAsByteBuffer(message.getBuffers(), message.getHeader().getLength(), type);
    } else {
        Pair<Integer, Object> keyPair = KeyDeserializer.getKeyAsByteBuffer(message.getKeyType(), message.getBuffers());
        MessageType keyType = message.getKeyType();
        Object data;
        if (MessageTypeUtils.isMultiMessageType(keyType)) {
            data = DataDeserializer.getAsByteBuffer(message.getBuffers(), message.getHeader().getLength() - keyPair.getKey() - 4 - 4, type, ((List) keyPair.getValue()).size());
            results = new ArrayList<>();
            List<byte[]> keyList = (List<byte[]>) keyPair.getValue();
            List<byte[]> dataList = (List<byte[]>) data;
            for (int i = 0; i < keyList.size(); i++) {
                results.add(new ImmutablePair<>(keyList.get(i), dataList.get(i)));
            }
            return results;
        } else if (!MessageTypeUtils.isPrimitiveType(keyType)) {
            data = DataDeserializer.getAsByteBuffer(message.getBuffers(), message.getHeader().getLength() - keyPair.getKey() - 4, type);
        } else {
            data = DataDeserializer.getAsByteBuffer(message.getBuffers(), message.getHeader().getLength() - keyPair.getKey(), type);
        }
        return new ImmutablePair<>(keyPair.getValue(), data);
    }
}

Example 2 with MPIMessage

use of edu.iu.dsc.tws.comms.mpi.MPIMessage in project twister2 by DSC-SPIDAL.

the class MPIMessageSerializer method build.

@Override
public Object build(Object message, Object partialBuildObject) {
    MPISendMessage sendMessage = (MPISendMessage) partialBuildObject;
    // we got an already serialized message, lets just return it
    if (sendMessage.getMPIMessage().isComplete()) {
        sendMessage.setSendState(MPISendMessage.SendState.SERIALIZED);
        return sendMessage;
    }
    if (sendMessage.getSerializationState() == null) {
        sendMessage.setSerializationState(new SerializeState());
    }
    while (sendBuffers.size() > 0 && sendMessage.serializedState() != MPISendMessage.SendState.SERIALIZED) {
        MPIBuffer buffer = sendBuffers.poll();
        if (buffer == null) {
            break;
        }
        if (sendMessage.serializedState() == MPISendMessage.SendState.INIT || sendMessage.serializedState() == MPISendMessage.SendState.SENT_INTERNALLY) {
            // build the header
            buildHeader(buffer, sendMessage);
            sendMessage.setSendState(MPISendMessage.SendState.HEADER_BUILT);
        }
        if (sendMessage.serializedState() == MPISendMessage.SendState.HEADER_BUILT || sendMessage.serializedState() == MPISendMessage.SendState.BODY_BUILT) {
            // build the body
            // first we need to serialize the body if needed
            boolean complete = serializeBody(message, sendMessage, buffer);
            if (complete) {
                sendMessage.setSendState(MPISendMessage.SendState.SERIALIZED);
            }
        }
        // okay we are adding this buffer
        sendMessage.getMPIMessage().addBuffer(buffer);
        if (sendMessage.serializedState() == MPISendMessage.SendState.SERIALIZED) {
            MPIMessage mpiMessage = sendMessage.getMPIMessage();
            SerializeState state = sendMessage.getSerializationState();
            int totalBytes = state.getTotalBytes();
            mpiMessage.getBuffers().get(0).getByteBuffer().putInt(12, totalBytes);
            MessageHeader.Builder builder = MessageHeader.newBuilder(sendMessage.getSource(), sendMessage.getEdge(), totalBytes);
            builder.destination(sendMessage.getDestintationIdentifier());
            sendMessage.getMPIMessage().setHeader(builder.build());
            state.setTotalBytes(0);
            // mark the original message as complete
            mpiMessage.setComplete(true);
        } else {
            LOG.fine("Message NOT FULLY serialized");
        }
    }
    return sendMessage;
}

Example 3 with MPIMessage

use of edu.iu.dsc.tws.comms.mpi.MPIMessage in project twister2 by DSC-SPIDAL.

the class MPIMultiMessageSerializer method serializeBody.

@SuppressWarnings("rawtypes")
private void serializeBody(Object object, MPISendMessage sendMessage, MPIBuffer buffer) {
    List objectList = (List) object;
    SerializeState state = sendMessage.getSerializationState();
    int startIndex = state.getCurrentObject();
    // we assume remaining = capacity of the buffer as we always get a fresh buffer her
    int remaining = buffer.getByteBuffer().remaining();
    // we cannot use this buffer as we cannot put the sub header
    if (remaining <= MAX_SUB_MESSAGE_HEADER_SPACE) {
        throw new RuntimeException("This buffer is too small to fit a message: " + remaining);
    }
    // we will copy until we have space left or we are have serialized all the objects
    for (int i = startIndex; i < objectList.size(); i++) {
        Object o = objectList.get(i);
        if (o instanceof MPIMessage) {
            MPIMessage mpiMessage = (MPIMessage) o;
            boolean complete = serializeBufferedMessage(mpiMessage, state, buffer);
            // we copied this completely
            if (complete) {
                state.setCurrentObject(i + 1);
            } else {
                break;
            }
        } else {
            boolean complete = serializeMessage(o, sendMessage, buffer);
            if (complete) {
                state.setCurrentObject(i + 1);
            } else {
                break;
            }
        }
        // check how much space left in this buffer
        remaining = buffer.getByteBuffer().remaining();
        // header in a contigous space, so we cannot use this buffer anymore
        if (!(remaining > MAX_SUB_MESSAGE_HEADER_SPACE && state.getCurrentObject() < objectList.size())) {
            break;
        }
    }
    if (state.getCurrentObject() == objectList.size()) {
        sendMessage.setSendState(MPISendMessage.SendState.SERIALIZED);
    }
}

Example 4 with MPIMessage

use of edu.iu.dsc.tws.comms.mpi.MPIMessage in project twister2 by DSC-SPIDAL.

the class Test method serializeObject.

private MPIMessage serializeObject(List object, int source) {
    MPIMessage mpiMessage = new MPIMessage(source, MessageType.OBJECT, MPIMessageDirection.OUT, new MessageListener());
    mpiMessage.setKeyType(MessageType.INTEGER);
    int di = -1;
    MPISendMessage sendMessage = new MPISendMessage(source, mpiMessage, 0, di, 0, 0, null, null);
    multiMessageSerializer.build(object, sendMessage);
    return mpiMessage;
}

Example 5 with MPIMessage

use of edu.iu.dsc.tws.comms.mpi.MPIMessage in project twister2 by DSC-SPIDAL.

the class MPIMultiMessageDeserializer method getDataBuffers.

@Override
public Object getDataBuffers(Object partialObject, int edge) {
    MPIMessage currentMessage = (MPIMessage) partialObject;
    int readLength = 0;
    int bufferIndex = 0;
    List<MPIBuffer> buffers = currentMessage.getBuffers();
    List<Object> returnList = new ArrayList<>();
    MessageHeader header = currentMessage.getHeader();
    if (header == null) {
        throw new RuntimeException("Header must be built before the message");
    }
    // LOG.info(String.format("%d deserilizing message", executor));
    while (readLength < header.getLength()) {
        List<MPIBuffer> messageBuffers = new ArrayList<>();
        MPIBuffer mpiBuffer = buffers.get(bufferIndex);
        ByteBuffer byteBuffer = mpiBuffer.getByteBuffer();
        // now read the length
        int length = byteBuffer.getInt();
        int tempLength = 0;
        int tempBufferIndex = bufferIndex;
        while (tempLength < length) {
            mpiBuffer = buffers.get(tempBufferIndex);
            messageBuffers.add(mpiBuffer);
            tempLength += mpiBuffer.getByteBuffer().remaining();
            tempBufferIndex++;
        // LOG.info(String.format("%d temp %d length %d readLength %d header %d buf_pos %d",
        // executor, tempLength, length, readLength, header.getLength(),
        // mpiBuffer.getByteBuffer().position()));
        }
        Object object = getSingleDataBuffers(currentMessage, messageBuffers, length);
        readLength += length + 4;
        if (keyed && !MessageTypeUtils.isPrimitiveType(currentMessage.getKeyType())) {
            // adding 4 to the length since the key length is also kept
            readLength += 4;
        }
        byteBuffer = mpiBuffer.getByteBuffer();
        if (byteBuffer.remaining() > 0) {
            bufferIndex = tempBufferIndex - 1;
        } else {
            bufferIndex = tempBufferIndex;
        }
        returnList.add(object);
    }
    return returnList;
}