Search in sources :

Example 1 with Managers

use of com.baidu.hugegraph.computer.core.manager.Managers in project hugegraph-computer by hugegraph.

the class ComputeManagerTest method setup.

@Before
public void setup() {
    this.config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.JOB_ID, "local_001", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.JOB_PARTITIONS_COUNT, "2", ComputerOptions.BSP_MAX_SUPER_STEP, "2", ComputerOptions.WORKER_COMBINER_CLASS, // Can't combine
    Null.class.getName(), ComputerOptions.ALGORITHM_RESULT_CLASS, IdListList.class.getName(), ComputerOptions.ALGORITHM_MESSAGE_CLASS, IdList.class.getName(), ComputerOptions.WORKER_DATA_DIRS, "[data_dir1, data_dir2]", ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "10000", ComputerOptions.WORKER_WAIT_FINISH_MESSAGES_TIMEOUT, "1000", ComputerOptions.INPUT_MAX_EDGES_IN_ONE_VERTEX, "10", ComputerOptions.WORKER_COMPUTATION_CLASS, MockComputation.class.getName(), ComputerOptions.INPUT_EDGE_FREQ, "SINGLE", ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
    this.managers = new Managers();
    FileManager fileManager = new FileManager();
    this.managers.add(fileManager);
    SortManager sortManager = new SendSortManager(context());
    this.managers.add(sortManager);
    MessageSendManager sendManager = new MessageSendManager(context(), sortManager, new MockMessageSender());
    this.managers.add(sendManager);
    MessageRecvManager receiveManager = new MessageRecvManager(context(), fileManager, sortManager);
    this.managers.add(receiveManager);
    this.managers.initAll(this.config);
    this.connectionId = new ConnectionId(new InetSocketAddress("localhost", 8081), 0);
    this.computeManager = new ComputeManager(context(), this.managers);
}
Also used : SendSortManager(com.baidu.hugegraph.computer.core.sort.sorting.SendSortManager) MessageRecvManager(com.baidu.hugegraph.computer.core.receiver.MessageRecvManager) ConnectionId(com.baidu.hugegraph.computer.core.network.ConnectionId) Managers(com.baidu.hugegraph.computer.core.manager.Managers) InetSocketAddress(java.net.InetSocketAddress) MessageSendManager(com.baidu.hugegraph.computer.core.sender.MessageSendManager) FileManager(com.baidu.hugegraph.computer.core.store.FileManager) SortManager(com.baidu.hugegraph.computer.core.sort.sorting.SortManager) SendSortManager(com.baidu.hugegraph.computer.core.sort.sorting.SendSortManager) Before(org.junit.Before)

Example 2 with Managers

use of com.baidu.hugegraph.computer.core.manager.Managers in project hugegraph-computer by hugegraph.

the class SenderIntegrateTest method slowSendFunc.

private void slowSendFunc(WorkerService service) throws TransportException {
    Managers managers = Whitebox.getInternalState(service, "managers");
    DataClientManager clientManager = managers.get(DataClientManager.NAME);
    ConnectionManager connManager = Whitebox.getInternalState(clientManager, "connManager");
    NettyTransportClient client = (NettyTransportClient) connManager.getOrCreateClient("127.0.0.1", 8091);
    ClientSession clientSession = Whitebox.invoke(client.getClass(), "clientSession", client);
    Function<Message, Future<Void>> sendFuncBak = Whitebox.getInternalState(clientSession, "sendFunction");
    Function<Message, Future<Void>> sendFunc = message -> {
        try {
            Thread.sleep(100);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        return sendFuncBak.apply(message);
    };
    Whitebox.setInternalState(clientSession, "sendFunction", sendFunc);
}
Also used : PageRankParams(com.baidu.hugegraph.computer.algorithm.centrality.pagerank.PageRankParams) ComputerOptions(com.baidu.hugegraph.computer.core.config.ComputerOptions) DoubleValue(com.baidu.hugegraph.computer.core.graph.value.DoubleValue) BeforeClass(org.junit.BeforeClass) Managers(com.baidu.hugegraph.computer.core.manager.Managers) RpcOptions(com.baidu.hugegraph.config.RpcOptions) Function(java.util.function.Function) MasterService(com.baidu.hugegraph.computer.core.master.MasterService) ArrayList(java.util.ArrayList) Future(java.util.concurrent.Future) Whitebox(com.baidu.hugegraph.testutil.Whitebox) TransportException(com.baidu.hugegraph.computer.core.common.exception.TransportException) ComputerContextUtil(com.baidu.hugegraph.computer.core.util.ComputerContextUtil) Message(com.baidu.hugegraph.computer.core.network.message.Message) AfterClass(org.junit.AfterClass) Logger(org.slf4j.Logger) ConnectionManager(com.baidu.hugegraph.computer.core.network.connection.ConnectionManager) Test(org.junit.Test) NettyTransportClient(com.baidu.hugegraph.computer.core.network.netty.NettyTransportClient) ClientSession(com.baidu.hugegraph.computer.core.network.session.ClientSession) Config(com.baidu.hugegraph.computer.core.config.Config) DataClientManager(com.baidu.hugegraph.computer.core.network.DataClientManager) List(java.util.List) Log(com.baidu.hugegraph.util.Log) WorkerService(com.baidu.hugegraph.computer.core.worker.WorkerService) Assert(com.baidu.hugegraph.testutil.Assert) ConnectionManager(com.baidu.hugegraph.computer.core.network.connection.ConnectionManager) Message(com.baidu.hugegraph.computer.core.network.message.Message) Managers(com.baidu.hugegraph.computer.core.manager.Managers) DataClientManager(com.baidu.hugegraph.computer.core.network.DataClientManager) NettyTransportClient(com.baidu.hugegraph.computer.core.network.netty.NettyTransportClient) ClientSession(com.baidu.hugegraph.computer.core.network.session.ClientSession) Future(java.util.concurrent.Future)

Example 3 with Managers

use of com.baidu.hugegraph.computer.core.manager.Managers in project hugegraph-computer by hugegraph.

the class EdgesInputTest method testEdgeFreq.

private void testEdgeFreq(EdgeFrequency freq) throws IOException {
    this.config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.JOB_ID, "local_001", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.JOB_PARTITIONS_COUNT, "1", ComputerOptions.WORKER_COMBINER_CLASS, // Can't combine
    Null.class.getName(), ComputerOptions.ALGORITHM_RESULT_CLASS, IdListList.class.getName(), ComputerOptions.ALGORITHM_MESSAGE_CLASS, IdList.class.getName(), ComputerOptions.WORKER_DATA_DIRS, "[data_dir1, data_dir2]", ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "10000", ComputerOptions.WORKER_WAIT_FINISH_MESSAGES_TIMEOUT, "1000", ComputerOptions.INPUT_MAX_EDGES_IN_ONE_VERTEX, "10", ComputerOptions.INPUT_EDGE_FREQ, freq.name(), ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
    this.managers = new Managers();
    FileManager fileManager = new FileManager();
    this.managers.add(fileManager);
    SortManager sortManager = new SendSortManager(context());
    this.managers.add(sortManager);
    MessageSendManager sendManager = new MessageSendManager(context(), sortManager, new MockMessageSender());
    this.managers.add(sendManager);
    MessageRecvManager receiveManager = new MessageRecvManager(context(), fileManager, sortManager);
    this.managers.add(receiveManager);
    this.managers.initAll(this.config);
    ConnectionId connectionId = new ConnectionId(new InetSocketAddress("localhost", 8081), 0);
    FileGraphPartition partition = new FileGraphPartition(context(), this.managers, 0);
    receiveManager.onStarted(connectionId);
    add200VertexBuffer((NetworkBuffer buffer) -> {
        receiveManager.handle(MessageType.VERTEX, 0, buffer);
    });
    receiveManager.onFinished(connectionId);
    receiveManager.onStarted(connectionId);
    addEdgeBuffer((NetworkBuffer buffer) -> {
        receiveManager.handle(MessageType.EDGE, 0, buffer);
    }, freq);
    receiveManager.onFinished(connectionId);
    Whitebox.invoke(partition.getClass(), new Class<?>[] { PeekableIterator.class, PeekableIterator.class }, "input", partition, receiveManager.vertexPartitions().get(0), receiveManager.edgePartitions().get(0));
    File edgeFile = Whitebox.getInternalState(partition, "edgeFile");
    EdgesInput edgesInput = new EdgesInput(context(), edgeFile);
    edgesInput.init();
    this.checkEdgesInput(edgesInput, freq);
    edgesInput.close();
}
Also used : InetSocketAddress(java.net.InetSocketAddress) MessageSendManager(com.baidu.hugegraph.computer.core.sender.MessageSendManager) MockMessageSender(com.baidu.hugegraph.computer.core.compute.MockMessageSender) FileManager(com.baidu.hugegraph.computer.core.store.FileManager) SortManager(com.baidu.hugegraph.computer.core.sort.sorting.SortManager) SendSortManager(com.baidu.hugegraph.computer.core.sort.sorting.SendSortManager) SendSortManager(com.baidu.hugegraph.computer.core.sort.sorting.SendSortManager) MessageRecvManager(com.baidu.hugegraph.computer.core.receiver.MessageRecvManager) ConnectionId(com.baidu.hugegraph.computer.core.network.ConnectionId) Managers(com.baidu.hugegraph.computer.core.manager.Managers) NetworkBuffer(com.baidu.hugegraph.computer.core.network.buffer.NetworkBuffer) FileGraphPartition(com.baidu.hugegraph.computer.core.compute.FileGraphPartition) File(java.io.File)

Example 4 with Managers

use of com.baidu.hugegraph.computer.core.manager.Managers in project hugegraph-computer by hugegraph.

the class MessageInputTest method setup.

@Before
public void setup() {
    this.config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.JOB_ID, "local_001", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.JOB_PARTITIONS_COUNT, "2", ComputerOptions.BSP_MAX_SUPER_STEP, "2", ComputerOptions.WORKER_COMBINER_CLASS, // Can't combine
    Null.class.getName(), ComputerOptions.ALGORITHM_RESULT_CLASS, IdListList.class.getName(), ComputerOptions.ALGORITHM_MESSAGE_CLASS, IdList.class.getName(), ComputerOptions.WORKER_DATA_DIRS, "[data_dir1, data_dir2]", ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "10000", ComputerOptions.WORKER_WAIT_FINISH_MESSAGES_TIMEOUT, "1000", ComputerOptions.INPUT_MAX_EDGES_IN_ONE_VERTEX, "10", ComputerOptions.WORKER_COMPUTATION_CLASS, MockComputation.class.getName(), ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
    this.managers = new Managers();
    FileManager fileManager = new FileManager();
    this.managers.add(fileManager);
    SortManager sortManager = new RecvSortManager(context());
    this.managers.add(sortManager);
    MessageRecvManager receiveManager = new MessageRecvManager(context(), fileManager, sortManager);
    this.managers.add(receiveManager);
    this.managers.initAll(this.config);
    this.connectionId = new ConnectionId(new InetSocketAddress("localhost", 8081), 0);
}
Also used : MessageRecvManager(com.baidu.hugegraph.computer.core.receiver.MessageRecvManager) ConnectionId(com.baidu.hugegraph.computer.core.network.ConnectionId) Managers(com.baidu.hugegraph.computer.core.manager.Managers) InetSocketAddress(java.net.InetSocketAddress) FileManager(com.baidu.hugegraph.computer.core.store.FileManager) RecvSortManager(com.baidu.hugegraph.computer.core.sort.sorting.RecvSortManager) SortManager(com.baidu.hugegraph.computer.core.sort.sorting.SortManager) RecvSortManager(com.baidu.hugegraph.computer.core.sort.sorting.RecvSortManager) Before(org.junit.Before)

Aggregations

Managers (com.baidu.hugegraph.computer.core.manager.Managers)4 ConnectionId (com.baidu.hugegraph.computer.core.network.ConnectionId)3 MessageRecvManager (com.baidu.hugegraph.computer.core.receiver.MessageRecvManager)3 SortManager (com.baidu.hugegraph.computer.core.sort.sorting.SortManager)3 FileManager (com.baidu.hugegraph.computer.core.store.FileManager)3 InetSocketAddress (java.net.InetSocketAddress)3 MessageSendManager (com.baidu.hugegraph.computer.core.sender.MessageSendManager)2 SendSortManager (com.baidu.hugegraph.computer.core.sort.sorting.SendSortManager)2 Before (org.junit.Before)2 PageRankParams (com.baidu.hugegraph.computer.algorithm.centrality.pagerank.PageRankParams)1 TransportException (com.baidu.hugegraph.computer.core.common.exception.TransportException)1 FileGraphPartition (com.baidu.hugegraph.computer.core.compute.FileGraphPartition)1 MockMessageSender (com.baidu.hugegraph.computer.core.compute.MockMessageSender)1 ComputerOptions (com.baidu.hugegraph.computer.core.config.ComputerOptions)1 Config (com.baidu.hugegraph.computer.core.config.Config)1 DoubleValue (com.baidu.hugegraph.computer.core.graph.value.DoubleValue)1 MasterService (com.baidu.hugegraph.computer.core.master.MasterService)1 DataClientManager (com.baidu.hugegraph.computer.core.network.DataClientManager)1 NetworkBuffer (com.baidu.hugegraph.computer.core.network.buffer.NetworkBuffer)1 ConnectionManager (com.baidu.hugegraph.computer.core.network.connection.ConnectionManager)1