Examples with MessageRecvManager com.baidu.hugegraph.computer.core.receiver.MessageRecvManager used on opensource projects

Example 1 with MessageRecvManager

use of com.baidu.hugegraph.computer.core.receiver.MessageRecvManager in project hugegraph-computer by hugegraph.

the class ComputeManagerTest method setup.

@Before
public void setup() {
    this.config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.JOB_ID, "local_001", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.JOB_PARTITIONS_COUNT, "2", ComputerOptions.BSP_MAX_SUPER_STEP, "2", ComputerOptions.WORKER_COMBINER_CLASS, // Can't combine
    Null.class.getName(), ComputerOptions.ALGORITHM_RESULT_CLASS, IdListList.class.getName(), ComputerOptions.ALGORITHM_MESSAGE_CLASS, IdList.class.getName(), ComputerOptions.WORKER_DATA_DIRS, "[data_dir1, data_dir2]", ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "10000", ComputerOptions.WORKER_WAIT_FINISH_MESSAGES_TIMEOUT, "1000", ComputerOptions.INPUT_MAX_EDGES_IN_ONE_VERTEX, "10", ComputerOptions.WORKER_COMPUTATION_CLASS, MockComputation.class.getName(), ComputerOptions.INPUT_EDGE_FREQ, "SINGLE", ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
    this.managers = new Managers();
    FileManager fileManager = new FileManager();
    this.managers.add(fileManager);
    SortManager sortManager = new SendSortManager(context());
    this.managers.add(sortManager);
    MessageSendManager sendManager = new MessageSendManager(context(), sortManager, new MockMessageSender());
    this.managers.add(sendManager);
    MessageRecvManager receiveManager = new MessageRecvManager(context(), fileManager, sortManager);
    this.managers.add(receiveManager);
    this.managers.initAll(this.config);
    this.connectionId = new ConnectionId(new InetSocketAddress("localhost", 8081), 0);
    this.computeManager = new ComputeManager(context(), this.managers);
}

Example 2 with MessageRecvManager

use of com.baidu.hugegraph.computer.core.receiver.MessageRecvManager in project hugegraph-computer by hugegraph.

the class WorkerService method initManagers.

private InetSocketAddress initManagers(ContainerInfo masterInfo) {
    // Create managers
    WorkerRpcManager rpcManager = new WorkerRpcManager();
    this.managers.add(rpcManager);
    /*
         * NOTE: this init() method will be called twice, will be ignored at
         * the 2nd time call.
         */
    WorkerRpcManager.updateRpcRemoteServerConfig(this.config, masterInfo.hostname(), masterInfo.rpcPort());
    rpcManager.init(this.config);
    WorkerAggrManager aggregatorManager = new WorkerAggrManager(this.context);
    aggregatorManager.service(rpcManager.aggregateRpcService());
    this.managers.add(aggregatorManager);
    FileManager fileManager = new FileManager();
    this.managers.add(fileManager);
    SortManager recvSortManager = new RecvSortManager(this.context);
    this.managers.add(recvSortManager);
    MessageRecvManager recvManager = new MessageRecvManager(this.context, fileManager, recvSortManager);
    this.managers.add(recvManager);
    ConnectionManager connManager = new TransportConnectionManager();
    DataServerManager serverManager = new DataServerManager(connManager, recvManager);
    this.managers.add(serverManager);
    DataClientManager clientManager = new DataClientManager(connManager, this.context);
    this.managers.add(clientManager);
    SortManager sendSortManager = new SendSortManager(this.context);
    this.managers.add(sendSortManager);
    MessageSendManager sendManager = new MessageSendManager(this.context, sendSortManager, clientManager.sender());
    this.managers.add(sendManager);
    WorkerInputManager inputManager = new WorkerInputManager(this.context, sendManager);
    inputManager.service(rpcManager.inputSplitService());
    this.managers.add(inputManager);
    // Init all managers
    this.managers.initAll(this.config);
    InetSocketAddress address = serverManager.address();
    LOG.info("{} WorkerService initialized managers with data server " + "address '{}'", this, address);
    return address;
}

Example 3 with MessageRecvManager

use of com.baidu.hugegraph.computer.core.receiver.MessageRecvManager in project hugegraph-computer by hugegraph.

the class DataServerManagerTest method test.

@Test
public void test() {
    Config config = UnitTestBase.updateWithRequiredOptions(RpcOptions.RPC_REMOTE_URL, "127.0.0.1:8090", ComputerOptions.JOB_ID, "local_001", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.BSP_LOG_INTERVAL, "30000", ComputerOptions.BSP_MAX_SUPER_STEP, "2", ComputerOptions.WORKER_COMPUTATION_CLASS, MockComputation.class.getName(), ComputerOptions.MASTER_COMPUTATION_CLASS, MockMasterComputation.class.getName());
    FileManager fileManager = new FileManager();
    fileManager.init(config);
    SortManager sortManager = new RecvSortManager(context());
    sortManager.init(config);
    MessageRecvManager recvManager = new MessageRecvManager(context(), fileManager, sortManager);
    recvManager.init(config);
    ConnectionManager connManager = new TransportConnectionManager();
    DataServerManager serverManager = new DataServerManager(connManager, recvManager);
    serverManager.init(config);
    Assert.assertEquals(DataServerManager.NAME, serverManager.name());
    InetSocketAddress address = serverManager.address();
    Assert.assertNotEquals(0, address.getPort());
    ConnectionId connectionId = ConnectionId.parseConnectionId(address.getHostName(), address.getPort());
    recvManager.onChannelActive(connectionId);
    recvManager.onChannelInactive(connectionId);
    TransportException e = new TransportException("test transport " + "exception");
    recvManager.exceptionCaught(e, connectionId);
    serverManager.close(config);
    fileManager.close(config);
    sortManager.close(config);
}

Example 4 with MessageRecvManager

use of com.baidu.hugegraph.computer.core.receiver.MessageRecvManager in project hugegraph-computer by hugegraph.

the class ComputeManagerTest method testProcess.

@Test
public void testProcess() throws IOException {
    MessageRecvManager receiveManager = this.managers.get(MessageRecvManager.NAME);
    receiveManager.onStarted(this.connectionId);
    add200VertexBuffer((NetworkBuffer buffer) -> {
        receiveManager.handle(MessageType.VERTEX, 0, buffer);
    });
    // Partition 1 only has vertex.
    add200VertexBuffer((NetworkBuffer buffer) -> {
        receiveManager.handle(MessageType.VERTEX, 1, buffer);
    });
    receiveManager.onFinished(this.connectionId);
    receiveManager.onStarted(this.connectionId);
    addSingleFreqEdgeBuffer((NetworkBuffer buffer) -> {
        receiveManager.handle(MessageType.EDGE, 0, buffer);
    });
    receiveManager.onFinished(this.connectionId);
    this.computeManager.input();
    // Superstep 0
    receiveManager.beforeSuperstep(this.config, 0);
    receiveManager.onStarted(this.connectionId);
    addMessages((NetworkBuffer buffer) -> {
        receiveManager.handle(MessageType.MSG, 0, buffer);
    });
    receiveManager.onFinished(this.connectionId);
    this.computeManager.compute(null, 0);
    receiveManager.afterSuperstep(this.config, 0);
    // Superstep 1
    this.computeManager.takeRecvedMessages();
    receiveManager.beforeSuperstep(this.config, 1);
    receiveManager.onStarted(this.connectionId);
    receiveManager.onFinished(this.connectionId);
    this.computeManager.compute(null, 1);
    receiveManager.afterSuperstep(this.config, 1);
    // Output
    this.computeManager.output();
}

Example 5 with MessageRecvManager

use of com.baidu.hugegraph.computer.core.receiver.MessageRecvManager in project hugegraph-computer by hugegraph.

the class EdgesInputTest method testEdgeFreq.

private void testEdgeFreq(EdgeFrequency freq) throws IOException {
    this.config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.JOB_ID, "local_001", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.JOB_PARTITIONS_COUNT, "1", ComputerOptions.WORKER_COMBINER_CLASS, // Can't combine
    Null.class.getName(), ComputerOptions.ALGORITHM_RESULT_CLASS, IdListList.class.getName(), ComputerOptions.ALGORITHM_MESSAGE_CLASS, IdList.class.getName(), ComputerOptions.WORKER_DATA_DIRS, "[data_dir1, data_dir2]", ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "10000", ComputerOptions.WORKER_WAIT_FINISH_MESSAGES_TIMEOUT, "1000", ComputerOptions.INPUT_MAX_EDGES_IN_ONE_VERTEX, "10", ComputerOptions.INPUT_EDGE_FREQ, freq.name(), ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
    this.managers = new Managers();
    FileManager fileManager = new FileManager();
    this.managers.add(fileManager);
    SortManager sortManager = new SendSortManager(context());
    this.managers.add(sortManager);
    MessageSendManager sendManager = new MessageSendManager(context(), sortManager, new MockMessageSender());
    this.managers.add(sendManager);
    MessageRecvManager receiveManager = new MessageRecvManager(context(), fileManager, sortManager);
    this.managers.add(receiveManager);
    this.managers.initAll(this.config);
    ConnectionId connectionId = new ConnectionId(new InetSocketAddress("localhost", 8081), 0);
    FileGraphPartition partition = new FileGraphPartition(context(), this.managers, 0);
    receiveManager.onStarted(connectionId);
    add200VertexBuffer((NetworkBuffer buffer) -> {
        receiveManager.handle(MessageType.VERTEX, 0, buffer);
    });
    receiveManager.onFinished(connectionId);
    receiveManager.onStarted(connectionId);
    addEdgeBuffer((NetworkBuffer buffer) -> {
        receiveManager.handle(MessageType.EDGE, 0, buffer);
    }, freq);
    receiveManager.onFinished(connectionId);
    Whitebox.invoke(partition.getClass(), new Class<?>[] { PeekableIterator.class, PeekableIterator.class }, "input", partition, receiveManager.vertexPartitions().get(0), receiveManager.edgePartitions().get(0));
    File edgeFile = Whitebox.getInternalState(partition, "edgeFile");
    EdgesInput edgesInput = new EdgesInput(context(), edgeFile);
    edgesInput.init();
    this.checkEdgesInput(edgesInput, freq);
    edgesInput.close();
}