Search in sources :

Example 1 with WorkerInputManager

use of com.baidu.hugegraph.computer.core.input.WorkerInputManager in project hugegraph-computer by hugegraph.

the class WorkerService method inputstep.

/**
 * Load vertices and edges from HugeGraph. There are two phases in
 * inputstep. First phase is get input splits from master, and read the
 * vertices and edges from input splits. The second phase is after all
 * workers read input splits, the workers merge the vertices and edges to
 * get the stats for each partition.
 */
private SuperstepStat inputstep() {
    LOG.info("{} WorkerService inputstep started", this);
    WorkerInputManager manager = this.managers.get(WorkerInputManager.NAME);
    manager.loadGraph();
    this.bsp4Worker.workerInputDone();
    this.bsp4Worker.waitMasterInputDone();
    WorkerStat workerStat = this.computeManager.input();
    this.bsp4Worker.workerStepDone(Constants.INPUT_SUPERSTEP, workerStat);
    SuperstepStat superstepStat = this.bsp4Worker.waitMasterStepDone(Constants.INPUT_SUPERSTEP);
    manager.close(this.config);
    LOG.info("{} WorkerService inputstep finished", this);
    return superstepStat;
}
Also used : SuperstepStat(com.baidu.hugegraph.computer.core.graph.SuperstepStat) WorkerInputManager(com.baidu.hugegraph.computer.core.input.WorkerInputManager)

Example 2 with WorkerInputManager

use of com.baidu.hugegraph.computer.core.input.WorkerInputManager in project hugegraph-computer by hugegraph.

the class WorkerService method initManagers.

private InetSocketAddress initManagers(ContainerInfo masterInfo) {
    // Create managers
    WorkerRpcManager rpcManager = new WorkerRpcManager();
    this.managers.add(rpcManager);
    /*
         * NOTE: this init() method will be called twice, will be ignored at
         * the 2nd time call.
         */
    WorkerRpcManager.updateRpcRemoteServerConfig(this.config, masterInfo.hostname(), masterInfo.rpcPort());
    rpcManager.init(this.config);
    WorkerAggrManager aggregatorManager = new WorkerAggrManager(this.context);
    aggregatorManager.service(rpcManager.aggregateRpcService());
    this.managers.add(aggregatorManager);
    FileManager fileManager = new FileManager();
    this.managers.add(fileManager);
    SortManager recvSortManager = new RecvSortManager(this.context);
    this.managers.add(recvSortManager);
    MessageRecvManager recvManager = new MessageRecvManager(this.context, fileManager, recvSortManager);
    this.managers.add(recvManager);
    ConnectionManager connManager = new TransportConnectionManager();
    DataServerManager serverManager = new DataServerManager(connManager, recvManager);
    this.managers.add(serverManager);
    DataClientManager clientManager = new DataClientManager(connManager, this.context);
    this.managers.add(clientManager);
    SortManager sendSortManager = new SendSortManager(this.context);
    this.managers.add(sendSortManager);
    MessageSendManager sendManager = new MessageSendManager(this.context, sendSortManager, clientManager.sender());
    this.managers.add(sendManager);
    WorkerInputManager inputManager = new WorkerInputManager(this.context, sendManager);
    inputManager.service(rpcManager.inputSplitService());
    this.managers.add(inputManager);
    // Init all managers
    this.managers.initAll(this.config);
    InetSocketAddress address = serverManager.address();
    LOG.info("{} WorkerService initialized managers with data server " + "address '{}'", this, address);
    return address;
}
Also used : WorkerAggrManager(com.baidu.hugegraph.computer.core.aggregator.WorkerAggrManager) DataServerManager(com.baidu.hugegraph.computer.core.network.DataServerManager) InetSocketAddress(java.net.InetSocketAddress) MessageSendManager(com.baidu.hugegraph.computer.core.sender.MessageSendManager) FileManager(com.baidu.hugegraph.computer.core.store.FileManager) RecvSortManager(com.baidu.hugegraph.computer.core.sort.sorting.RecvSortManager) SortManager(com.baidu.hugegraph.computer.core.sort.sorting.SortManager) SendSortManager(com.baidu.hugegraph.computer.core.sort.sorting.SendSortManager) WorkerRpcManager(com.baidu.hugegraph.computer.core.rpc.WorkerRpcManager) SendSortManager(com.baidu.hugegraph.computer.core.sort.sorting.SendSortManager) MessageRecvManager(com.baidu.hugegraph.computer.core.receiver.MessageRecvManager) TransportConnectionManager(com.baidu.hugegraph.computer.core.network.connection.TransportConnectionManager) ConnectionManager(com.baidu.hugegraph.computer.core.network.connection.ConnectionManager) DataClientManager(com.baidu.hugegraph.computer.core.network.DataClientManager) WorkerInputManager(com.baidu.hugegraph.computer.core.input.WorkerInputManager) RecvSortManager(com.baidu.hugegraph.computer.core.sort.sorting.RecvSortManager) TransportConnectionManager(com.baidu.hugegraph.computer.core.network.connection.TransportConnectionManager)

Aggregations

WorkerInputManager (com.baidu.hugegraph.computer.core.input.WorkerInputManager)2 WorkerAggrManager (com.baidu.hugegraph.computer.core.aggregator.WorkerAggrManager)1 SuperstepStat (com.baidu.hugegraph.computer.core.graph.SuperstepStat)1 DataClientManager (com.baidu.hugegraph.computer.core.network.DataClientManager)1 DataServerManager (com.baidu.hugegraph.computer.core.network.DataServerManager)1 ConnectionManager (com.baidu.hugegraph.computer.core.network.connection.ConnectionManager)1 TransportConnectionManager (com.baidu.hugegraph.computer.core.network.connection.TransportConnectionManager)1 MessageRecvManager (com.baidu.hugegraph.computer.core.receiver.MessageRecvManager)1 WorkerRpcManager (com.baidu.hugegraph.computer.core.rpc.WorkerRpcManager)1 MessageSendManager (com.baidu.hugegraph.computer.core.sender.MessageSendManager)1 RecvSortManager (com.baidu.hugegraph.computer.core.sort.sorting.RecvSortManager)1 SendSortManager (com.baidu.hugegraph.computer.core.sort.sorting.SendSortManager)1 SortManager (com.baidu.hugegraph.computer.core.sort.sorting.SortManager)1 FileManager (com.baidu.hugegraph.computer.core.store.FileManager)1 InetSocketAddress (java.net.InetSocketAddress)1