Examples with MessageHeader edu.iu.dsc.tws.api.comms.messaging.MessageHeader used on opensource projects

Example 6 with MessageHeader

use of edu.iu.dsc.tws.api.comms.messaging.MessageHeader in project twister2 by DSC-SPIDAL.

the class TWSUCXChannelTest method ucxTest.

@Test
public void ucxTest() throws InterruptedException {
    final int dataSize = 25;
    final int[] data = new int[dataSize];
    for (int i = 0; i < dataSize; i++) {
        data[i] = i;
    }
    new Thread(() -> {
        TWSUCXChannel sender = new TWSUCXChannel(Config.newBuilder().build(), workerControllers.get(0));
        OutMessage outMessage = new OutMessage(0, 1, -1, 10, 0, null, null, MessageTypes.INTEGER_ARRAY, null, null, data);
        Queue<DataBuffer> sendingBuffers = new ConcurrentLinkedQueue<>();
        sendingBuffers.add(new DataBuffer(sender.createBuffer(50)));
        sendingBuffers.add(new DataBuffer(sender.createBuffer(50)));
        DataSerializer dataSerializer = new DataSerializer();
        dataSerializer.init(Config.newBuilder().build(), sendingBuffers);
        AtomicInteger sendingCount = new AtomicInteger();
        AtomicBoolean progress = new AtomicBoolean(true);
        while (progress.get()) {
            ChannelMessage channelMessage = dataSerializer.build(data, outMessage);
            if (channelMessage != null) {
                sender.sendMessage(1, channelMessage, new ChannelListener() {

                    @Override
                    public void onReceiveComplete(int id, int stream, DataBuffer message) {
                    }

                    @Override
                    public void onSendComplete(int id, int stream, ChannelMessage message) {
                        message.getBuffers().forEach(dataBuffer -> {
                            dataBuffer.setSize(0);
                            dataBuffer.getByteBuffer().clear();
                        });
                        sendingBuffers.addAll(message.getBuffers());
                        progress.set(!outMessage.getSendState().equals(OutMessage.SendState.SERIALIZED));
                    // LOG.info("Sending Count : " + sendingCount.incrementAndGet());
                    }
                });
            }
            sender.progress();
        }
        LOG.info("Exiting send loop...");
    }).start();
    TWSUCXChannel receiver = new TWSUCXChannel(Config.newBuilder().build(), workerControllers.get(1));
    Queue<DataBuffer> recvBuffers = new ConcurrentLinkedQueue<>();
    recvBuffers.add(new DataBuffer(receiver.createBuffer(50)));
    recvBuffers.add(new DataBuffer(receiver.createBuffer(50)));
    DataDeserializer dataDeserializer = new DataDeserializer();
    dataDeserializer.init(Config.newBuilder().build());
    AtomicInteger recvCount = new AtomicInteger(0);
    AtomicBoolean received = new AtomicBoolean(false);
    final int[][] receivedData = new int[1][dataSize];
    receiver.receiveMessage(0, 0, 1, new ChannelListener() {

        private InMessage inMessage;

        @Override
        public void onReceiveComplete(int id, int stream, DataBuffer message) {
            if (recvCount.incrementAndGet() == 1) {
                MessageHeader messageHeader = dataDeserializer.buildHeader(message, 1);
                this.inMessage = new InMessage(id, MessageTypes.INTEGER_ARRAY, channelMessage -> {
                    LOG.info("Release called");
                }, messageHeader);
            }
            LOG.info("Received : " + recvCount.get());
            if (this.inMessage.addBufferAndCalculate(message)) {
                LOG.info("Built");
                dataDeserializer.build(this.inMessage, 1);
                receivedData[0] = (int[]) this.inMessage.getDataBuilder().getFinalObject();
                received.set(true);
            }
            dataDeserializer.build(this.inMessage, 1);
            message.getByteBuffer().clear();
            recvBuffers.add(message);
        }

        @Override
        public void onSendComplete(int id, int stream, ChannelMessage message) {
        }
    }, recvBuffers);
    while (!received.get()) {
        receiver.progress();
    }
    Assert.assertArrayEquals(data, receivedData[0]);
}

Example 7 with MessageHeader

use of edu.iu.dsc.tws.api.comms.messaging.MessageHeader in project twister2 by DSC-SPIDAL.

the class DataSerializerTest method singleValueCase.

private InMessage singleValueCase(int numBuffers, int size, MessageType type, Object data) {
    BlockingQueue<DataBuffer> bufferQueue = createDataQueue(numBuffers, size);
    OutMessage outMessage = new OutMessage(0, 1, -1, 10, 0, null, null, type, null, null, data);
    DataSerializer serializer = new DataSerializer();
    serializer.init(Config.newBuilder().build(), bufferQueue);
    List<ChannelMessage> messages = new ArrayList<>();
    while (outMessage.getSendState() != OutMessage.SendState.SERIALIZED) {
        ChannelMessage ch = serializer.build(data, outMessage);
        messages.add(ch);
    }
    DataDeserializer deserializer = new DataDeserializer();
    deserializer.init(Config.newBuilder().build());
    MessageHeader header = deserializer.buildHeader(messages.get(0).getBuffers().get(0), 1);
    InMessage inMessage = new InMessage(0, type, null, header);
    for (ChannelMessage channelMessage : messages) {
        for (DataBuffer dataBuffer : channelMessage.getBuffers()) {
            inMessage.addBufferAndCalculate(dataBuffer);
        }
    }
    deserializer.build(inMessage, 1);
    return inMessage;
}

Example 8 with MessageHeader

use of edu.iu.dsc.tws.api.comms.messaging.MessageHeader in project twister2 by DSC-SPIDAL.

the class DataSerializerTest method listValueCase.

private InMessage listValueCase(int numBuffers, int size, List<Object> data, MessageType type) {
    BlockingQueue<DataBuffer> bufferQueue = createDataQueue(numBuffers, size);
    OutMessage outMessage = new OutMessage(0, 1, -1, 10, 0, null, null, type, null, null, data);
    DataSerializer serializer = new DataSerializer();
    serializer.init(Config.newBuilder().build(), bufferQueue);
    List<ChannelMessage> messages = new ArrayList<>();
    while (outMessage.getSendState() != OutMessage.SendState.SERIALIZED) {
        ChannelMessage ch = serializer.build(data, outMessage);
        messages.add(ch);
    }
    DataDeserializer deserializer = new DataDeserializer();
    deserializer.init(Config.newBuilder().build());
    MessageHeader header = deserializer.buildHeader(messages.get(0).getBuffers().get(0), 1);
    InMessage inMessage = new InMessage(0, type, null, header);
    for (ChannelMessage channelMessage : messages) {
        for (DataBuffer dataBuffer : channelMessage.getBuffers()) {
            inMessage.addBufferAndCalculate(dataBuffer);
        }
    }
    deserializer.build(inMessage, 1);
    return inMessage;
}

Example 9 with MessageHeader

use of edu.iu.dsc.tws.api.comms.messaging.MessageHeader in project twister2 by DSC-SPIDAL.

the class ControlledChannelOperation method onReceiveComplete.

@Override
public void onReceiveComplete(int id, int e, DataBuffer buffer) {
    // we need to try to build the message here, we may need many more messages to complete
    ByteBuffer byteBuffer = buffer.getByteBuffer();
    byteBuffer.position(buffer.getSize());
    byteBuffer.flip();
    // we have the source of the message at 0th position as an integer
    int source = byteBuffer.getInt(0);
    InMessage currentMessage = currentMessages.get(source);
    if (currentMessage == null) {
        MessageHeader header = messageDeSerializer.get(source).buildHeader(buffer, e);
        MessageType recvDType = receiveDataType;
        MessageType recvKType = receiveKeyType;
        if ((header.getFlags() & MessageFlags.SYNC_BARRIER) == MessageFlags.SYNC_BARRIER) {
            recvDType = MessageTypes.BYTE_ARRAY;
            recvKType = MessageTypes.EMPTY;
        }
        currentMessage = new InMessage(id, recvDType, this, header);
        if (isKeyed) {
            currentMessage.setKeyType(recvKType);
        }
        if (!currentMessage.addBufferAndCalculate(buffer)) {
            currentMessages.put(source, currentMessage);
        }
        // we add the message immediately to the deserialization as we can deserialize partially
        Queue<InMessage> deserializeQueue = pendingReceiveDeSerializations.get(source);
        if (!deserializeQueue.offer(currentMessage)) {
            throw new RuntimeException(executor + " We should have enough space: " + deserializeQueue.size());
        }
    } else {
        if (currentMessage.addBufferAndCalculate(buffer)) {
            currentMessages.remove(source);
        }
    }
}

Example 10 with MessageHeader

use of edu.iu.dsc.tws.api.comms.messaging.MessageHeader in project twister2 by DSC-SPIDAL.

the class ChannelDataFlowOperation method onReceiveComplete.

@Override
public void onReceiveComplete(int id, int e, DataBuffer buffer) {
    // we need to try to build the message here, we may need many more messages to complete
    ByteBuffer byteBuffer = buffer.getByteBuffer();
    byteBuffer.position(buffer.getSize());
    byteBuffer.flip();
    // we have the source of the message at 0th position as an integer
    int source = byteBuffer.getInt(0);
    InMessage currentMessage = currentMessages.get(source);
    if (currentMessage == null) {
        MessageHeader header = messageDeSerializer.get(source).buildHeader(buffer, e);
        MessageType recvDType = receiveDataType;
        MessageType recvKType = receiveKeyType;
        if ((header.getFlags() & MessageFlags.SYNC_BARRIER) == MessageFlags.SYNC_BARRIER) {
            recvDType = MessageTypes.BYTE_ARRAY;
            recvKType = MessageTypes.EMPTY;
        }
        currentMessage = new InMessage(id, recvDType, this, header);
        if (isKeyed) {
            currentMessage.setKeyType(recvKType);
        }
        if (!currentMessage.addBufferAndCalculate(buffer)) {
            currentMessages.put(source, currentMessage);
        }
        // we add the message immediately to the deserialization as we can deserialize partially
        Queue<InMessage> deserializeQueue = pendingReceiveDeSerializations.get(source);
        if (!deserializeQueue.offer(currentMessage)) {
            throw new RuntimeException(executor + " We should have enough space: " + deserializeQueue.size());
        }
    } else {
        if (currentMessage.addBufferAndCalculate(buffer)) {
            currentMessages.remove(source);
        }
    }
}