Search in sources :

Example 1 with NetworkBuffer

use of com.baidu.hugegraph.computer.core.network.buffer.NetworkBuffer in project hugegraph-computer by hugegraph.

the class AbstractMessage method encode.

@Override
public NetworkBuffer encode(ByteBuf buf) {
    this.encodeHeader(buf);
    int bodyStart = buf.writerIndex();
    NetworkBuffer networkBuffer = this.encodeBody(buf);
    int bodyEnd = buf.writerIndex();
    int bodyLength;
    if (networkBuffer != null) {
        assert bodyStart == bodyEnd;
        bodyLength = networkBuffer.length();
    } else {
        bodyLength = bodyEnd - bodyStart;
    }
    int lastWriteIndex = buf.writerIndex();
    try {
        buf.resetWriterIndex();
        buf.writeInt(bodyLength);
    } finally {
        buf.writerIndex(lastWriteIndex);
    }
    return networkBuffer;
}
Also used : NetworkBuffer(com.baidu.hugegraph.computer.core.network.buffer.NetworkBuffer)

Example 2 with NetworkBuffer

use of com.baidu.hugegraph.computer.core.network.buffer.NetworkBuffer in project hugegraph-computer by hugegraph.

the class NettyEncodeDecodeHandlerTest method testSendMsgWithEncoderExceptionMock.

@Test
public void testSendMsgWithEncoderExceptionMock() throws IOException {
    NettyTransportClient client = (NettyTransportClient) this.oneClient();
    client.startSession();
    int requestId = 1;
    int partition = 1;
    byte[] bytes = StringEncoding.encode("mock msg");
    ByteBuffer buffer = ByteBuffer.wrap(bytes);
    NetworkBuffer body = new NioBuffer(buffer);
    DataMessage dataMessage = new DataMessage(null, requestId, partition, body);
    ChannelFutureListenerOnWrite listener = new ChannelFutureListenerOnWrite(clientHandler);
    ChannelFutureListenerOnWrite spyListener = Mockito.spy(listener);
    client.channel().writeAndFlush(dataMessage).addListener(spyListener);
    Mockito.verify(clientHandler, Mockito.timeout(3000L).times(1)).onChannelActive(Mockito.any());
    Mockito.verify(clientHandler, Mockito.timeout(3000L).times(1)).exceptionCaught(Mockito.any(), Mockito.any());
    Mockito.verify(spyListener, Mockito.timeout(3000L).times(1)).onFailure(Mockito.any(), Mockito.any());
}
Also used : DataMessage(com.baidu.hugegraph.computer.core.network.message.DataMessage) NetworkBuffer(com.baidu.hugegraph.computer.core.network.buffer.NetworkBuffer) ByteBuffer(java.nio.ByteBuffer) NioBuffer(com.baidu.hugegraph.computer.core.network.buffer.NioBuffer) Test(org.junit.Test)

Example 3 with NetworkBuffer

use of com.baidu.hugegraph.computer.core.network.buffer.NetworkBuffer in project hugegraph-computer by hugegraph.

the class ComputeManagerTest method testProcess.

@Test
public void testProcess() throws IOException {
    MessageRecvManager receiveManager = this.managers.get(MessageRecvManager.NAME);
    receiveManager.onStarted(this.connectionId);
    add200VertexBuffer((NetworkBuffer buffer) -> {
        receiveManager.handle(MessageType.VERTEX, 0, buffer);
    });
    // Partition 1 only has vertex.
    add200VertexBuffer((NetworkBuffer buffer) -> {
        receiveManager.handle(MessageType.VERTEX, 1, buffer);
    });
    receiveManager.onFinished(this.connectionId);
    receiveManager.onStarted(this.connectionId);
    addSingleFreqEdgeBuffer((NetworkBuffer buffer) -> {
        receiveManager.handle(MessageType.EDGE, 0, buffer);
    });
    receiveManager.onFinished(this.connectionId);
    this.computeManager.input();
    // Superstep 0
    receiveManager.beforeSuperstep(this.config, 0);
    receiveManager.onStarted(this.connectionId);
    addMessages((NetworkBuffer buffer) -> {
        receiveManager.handle(MessageType.MSG, 0, buffer);
    });
    receiveManager.onFinished(this.connectionId);
    this.computeManager.compute(null, 0);
    receiveManager.afterSuperstep(this.config, 0);
    // Superstep 1
    this.computeManager.takeRecvedMessages();
    receiveManager.beforeSuperstep(this.config, 1);
    receiveManager.onStarted(this.connectionId);
    receiveManager.onFinished(this.connectionId);
    this.computeManager.compute(null, 1);
    receiveManager.afterSuperstep(this.config, 1);
    // Output
    this.computeManager.output();
}
Also used : MessageRecvManager(com.baidu.hugegraph.computer.core.receiver.MessageRecvManager) NetworkBuffer(com.baidu.hugegraph.computer.core.network.buffer.NetworkBuffer) Test(org.junit.Test)

Example 4 with NetworkBuffer

use of com.baidu.hugegraph.computer.core.network.buffer.NetworkBuffer in project hugegraph-computer by hugegraph.

the class EdgesInputTest method testEdgeFreq.

private void testEdgeFreq(EdgeFrequency freq) throws IOException {
    this.config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.JOB_ID, "local_001", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.JOB_PARTITIONS_COUNT, "1", ComputerOptions.WORKER_COMBINER_CLASS, // Can't combine
    Null.class.getName(), ComputerOptions.ALGORITHM_RESULT_CLASS, IdListList.class.getName(), ComputerOptions.ALGORITHM_MESSAGE_CLASS, IdList.class.getName(), ComputerOptions.WORKER_DATA_DIRS, "[data_dir1, data_dir2]", ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "10000", ComputerOptions.WORKER_WAIT_FINISH_MESSAGES_TIMEOUT, "1000", ComputerOptions.INPUT_MAX_EDGES_IN_ONE_VERTEX, "10", ComputerOptions.INPUT_EDGE_FREQ, freq.name(), ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
    this.managers = new Managers();
    FileManager fileManager = new FileManager();
    this.managers.add(fileManager);
    SortManager sortManager = new SendSortManager(context());
    this.managers.add(sortManager);
    MessageSendManager sendManager = new MessageSendManager(context(), sortManager, new MockMessageSender());
    this.managers.add(sendManager);
    MessageRecvManager receiveManager = new MessageRecvManager(context(), fileManager, sortManager);
    this.managers.add(receiveManager);
    this.managers.initAll(this.config);
    ConnectionId connectionId = new ConnectionId(new InetSocketAddress("localhost", 8081), 0);
    FileGraphPartition partition = new FileGraphPartition(context(), this.managers, 0);
    receiveManager.onStarted(connectionId);
    add200VertexBuffer((NetworkBuffer buffer) -> {
        receiveManager.handle(MessageType.VERTEX, 0, buffer);
    });
    receiveManager.onFinished(connectionId);
    receiveManager.onStarted(connectionId);
    addEdgeBuffer((NetworkBuffer buffer) -> {
        receiveManager.handle(MessageType.EDGE, 0, buffer);
    }, freq);
    receiveManager.onFinished(connectionId);
    Whitebox.invoke(partition.getClass(), new Class<?>[] { PeekableIterator.class, PeekableIterator.class }, "input", partition, receiveManager.vertexPartitions().get(0), receiveManager.edgePartitions().get(0));
    File edgeFile = Whitebox.getInternalState(partition, "edgeFile");
    EdgesInput edgesInput = new EdgesInput(context(), edgeFile);
    edgesInput.init();
    this.checkEdgesInput(edgesInput, freq);
    edgesInput.close();
}
Also used : InetSocketAddress(java.net.InetSocketAddress) MessageSendManager(com.baidu.hugegraph.computer.core.sender.MessageSendManager) MockMessageSender(com.baidu.hugegraph.computer.core.compute.MockMessageSender) FileManager(com.baidu.hugegraph.computer.core.store.FileManager) SortManager(com.baidu.hugegraph.computer.core.sort.sorting.SortManager) SendSortManager(com.baidu.hugegraph.computer.core.sort.sorting.SendSortManager) SendSortManager(com.baidu.hugegraph.computer.core.sort.sorting.SendSortManager) MessageRecvManager(com.baidu.hugegraph.computer.core.receiver.MessageRecvManager) ConnectionId(com.baidu.hugegraph.computer.core.network.ConnectionId) Managers(com.baidu.hugegraph.computer.core.manager.Managers) NetworkBuffer(com.baidu.hugegraph.computer.core.network.buffer.NetworkBuffer) FileGraphPartition(com.baidu.hugegraph.computer.core.compute.FileGraphPartition) File(java.io.File)

Example 5 with NetworkBuffer

use of com.baidu.hugegraph.computer.core.network.buffer.NetworkBuffer in project hugegraph-computer by hugegraph.

the class MessageInputTest method testMessageInput.

@Test
public void testMessageInput() throws IOException {
    MessageRecvManager receiveManager = this.managers.get(MessageRecvManager.NAME);
    receiveManager.onStarted(this.connectionId);
    // Superstep 0
    receiveManager.beforeSuperstep(this.config, 0);
    receiveManager.onStarted(this.connectionId);
    addMessages((NetworkBuffer buffer) -> {
        receiveManager.handle(MessageType.MSG, 0, buffer);
    });
    receiveManager.onFinished(this.connectionId);
    PeekableIterator<KvEntry> it = receiveManager.messagePartitions().get(0);
    MessageInput<IdList> input = new MessageInput<>(context(), it);
    Map<Id, List<IdList>> expectedMessages = expectedMessages();
    checkMessages(expectedMessages, input);
}
Also used : MessageRecvManager(com.baidu.hugegraph.computer.core.receiver.MessageRecvManager) KvEntry(com.baidu.hugegraph.computer.core.store.entry.KvEntry) NetworkBuffer(com.baidu.hugegraph.computer.core.network.buffer.NetworkBuffer) ArrayList(java.util.ArrayList) IdList(com.baidu.hugegraph.computer.core.graph.value.IdList) List(java.util.List) IdListList(com.baidu.hugegraph.computer.core.graph.value.IdListList) Id(com.baidu.hugegraph.computer.core.graph.id.Id) BytesId(com.baidu.hugegraph.computer.core.graph.id.BytesId) ConnectionId(com.baidu.hugegraph.computer.core.network.ConnectionId) IdList(com.baidu.hugegraph.computer.core.graph.value.IdList) Test(org.junit.Test)

Aggregations

NetworkBuffer (com.baidu.hugegraph.computer.core.network.buffer.NetworkBuffer)13 Test (org.junit.Test)7 ByteBuf (io.netty.buffer.ByteBuf)5 NettyBuffer (com.baidu.hugegraph.computer.core.network.buffer.NettyBuffer)4 FrameDecoder (com.baidu.hugegraph.computer.core.network.netty.codec.FrameDecoder)3 MessageRecvManager (com.baidu.hugegraph.computer.core.receiver.MessageRecvManager)3 EmbeddedChannel (io.netty.channel.embedded.EmbeddedChannel)3 ConnectionId (com.baidu.hugegraph.computer.core.network.ConnectionId)2 FileRegionBuffer (com.baidu.hugegraph.computer.core.network.buffer.FileRegionBuffer)2 NioBuffer (com.baidu.hugegraph.computer.core.network.buffer.NioBuffer)2 DataMessage (com.baidu.hugegraph.computer.core.network.message.DataMessage)2 File (java.io.File)2 TransportException (com.baidu.hugegraph.computer.core.common.exception.TransportException)1 FileGraphPartition (com.baidu.hugegraph.computer.core.compute.FileGraphPartition)1 MockMessageSender (com.baidu.hugegraph.computer.core.compute.MockMessageSender)1 BytesId (com.baidu.hugegraph.computer.core.graph.id.BytesId)1 Id (com.baidu.hugegraph.computer.core.graph.id.Id)1 IdList (com.baidu.hugegraph.computer.core.graph.value.IdList)1 IdListList (com.baidu.hugegraph.computer.core.graph.value.IdListList)1 Managers (com.baidu.hugegraph.computer.core.manager.Managers)1