Search in sources :

Example 1 with DataClientManager

use of com.baidu.hugegraph.computer.core.network.DataClientManager in project hugegraph-computer by hugegraph.

the class WorkerService method initManagers.

private InetSocketAddress initManagers(ContainerInfo masterInfo) {
    // Create managers
    WorkerRpcManager rpcManager = new WorkerRpcManager();
    this.managers.add(rpcManager);
    /*
         * NOTE: this init() method will be called twice, will be ignored at
         * the 2nd time call.
         */
    WorkerRpcManager.updateRpcRemoteServerConfig(this.config, masterInfo.hostname(), masterInfo.rpcPort());
    rpcManager.init(this.config);
    WorkerAggrManager aggregatorManager = new WorkerAggrManager(this.context);
    aggregatorManager.service(rpcManager.aggregateRpcService());
    this.managers.add(aggregatorManager);
    FileManager fileManager = new FileManager();
    this.managers.add(fileManager);
    SortManager recvSortManager = new RecvSortManager(this.context);
    this.managers.add(recvSortManager);
    MessageRecvManager recvManager = new MessageRecvManager(this.context, fileManager, recvSortManager);
    this.managers.add(recvManager);
    ConnectionManager connManager = new TransportConnectionManager();
    DataServerManager serverManager = new DataServerManager(connManager, recvManager);
    this.managers.add(serverManager);
    DataClientManager clientManager = new DataClientManager(connManager, this.context);
    this.managers.add(clientManager);
    SortManager sendSortManager = new SendSortManager(this.context);
    this.managers.add(sendSortManager);
    MessageSendManager sendManager = new MessageSendManager(this.context, sendSortManager, clientManager.sender());
    this.managers.add(sendManager);
    WorkerInputManager inputManager = new WorkerInputManager(this.context, sendManager);
    inputManager.service(rpcManager.inputSplitService());
    this.managers.add(inputManager);
    // Init all managers
    this.managers.initAll(this.config);
    InetSocketAddress address = serverManager.address();
    LOG.info("{} WorkerService initialized managers with data server " + "address '{}'", this, address);
    return address;
}
Also used : WorkerAggrManager(com.baidu.hugegraph.computer.core.aggregator.WorkerAggrManager) DataServerManager(com.baidu.hugegraph.computer.core.network.DataServerManager) InetSocketAddress(java.net.InetSocketAddress) MessageSendManager(com.baidu.hugegraph.computer.core.sender.MessageSendManager) FileManager(com.baidu.hugegraph.computer.core.store.FileManager) RecvSortManager(com.baidu.hugegraph.computer.core.sort.sorting.RecvSortManager) SortManager(com.baidu.hugegraph.computer.core.sort.sorting.SortManager) SendSortManager(com.baidu.hugegraph.computer.core.sort.sorting.SendSortManager) WorkerRpcManager(com.baidu.hugegraph.computer.core.rpc.WorkerRpcManager) SendSortManager(com.baidu.hugegraph.computer.core.sort.sorting.SendSortManager) MessageRecvManager(com.baidu.hugegraph.computer.core.receiver.MessageRecvManager) TransportConnectionManager(com.baidu.hugegraph.computer.core.network.connection.TransportConnectionManager) ConnectionManager(com.baidu.hugegraph.computer.core.network.connection.ConnectionManager) DataClientManager(com.baidu.hugegraph.computer.core.network.DataClientManager) WorkerInputManager(com.baidu.hugegraph.computer.core.input.WorkerInputManager) RecvSortManager(com.baidu.hugegraph.computer.core.sort.sorting.RecvSortManager) TransportConnectionManager(com.baidu.hugegraph.computer.core.network.connection.TransportConnectionManager)

Example 2 with DataClientManager

use of com.baidu.hugegraph.computer.core.network.DataClientManager in project hugegraph-computer by hugegraph.

the class WorkerService method init.

/**
 * Init worker service, create the managers used by worker service.
 */
public void init(Config config) {
    E.checkArgument(!this.inited, "The %s has been initialized", this);
    this.serviceThread = Thread.currentThread();
    this.registerShutdownHook();
    this.config = config;
    this.workerInfo = new ContainerInfo();
    LOG.info("{} Start to initialize worker", this);
    this.bsp4Worker = new Bsp4Worker(this.config, this.workerInfo);
    /*
         * Keep the waitMasterInitDone() called before initManagers(),
         * in order to ensure master init() before worker managers init()
         */
    this.masterInfo = this.bsp4Worker.waitMasterInitDone();
    InetSocketAddress address = this.initManagers(this.masterInfo);
    this.workerInfo.updateAddress(address);
    Computation<?> computation = this.config.createObject(ComputerOptions.WORKER_COMPUTATION_CLASS);
    LOG.info("Loading computation '{}' in category '{}'", computation.name(), computation.category());
    this.combiner = this.config.createObject(ComputerOptions.WORKER_COMBINER_CLASS, false);
    if (this.combiner == null) {
        LOG.info("None combiner is provided for computation '{}'", computation.name());
    } else {
        LOG.info("Combiner '{}' is provided for computation '{}'", this.combiner.name(), computation.name());
    }
    LOG.info("{} register WorkerService", this);
    this.bsp4Worker.workerInitDone();
    List<ContainerInfo> workers = this.bsp4Worker.waitMasterAllInitDone();
    DataClientManager dm = this.managers.get(DataClientManager.NAME);
    for (ContainerInfo worker : workers) {
        this.workers.put(worker.id(), worker);
        dm.connect(worker.id(), worker.hostname(), worker.dataPort());
    }
    this.computeManager = new ComputeManager(this.context, this.managers);
    this.managers.initedAll(this.config);
    LOG.info("{} WorkerService initialized", this);
    this.inited = true;
}
Also used : InetSocketAddress(java.net.InetSocketAddress) DataClientManager(com.baidu.hugegraph.computer.core.network.DataClientManager) ContainerInfo(com.baidu.hugegraph.computer.core.common.ContainerInfo) Bsp4Worker(com.baidu.hugegraph.computer.core.bsp.Bsp4Worker) ComputeManager(com.baidu.hugegraph.computer.core.compute.ComputeManager)

Example 3 with DataClientManager

use of com.baidu.hugegraph.computer.core.network.DataClientManager in project hugegraph-computer by hugegraph.

the class SenderIntegrateTest method slowSendFunc.

private void slowSendFunc(WorkerService service) throws TransportException {
    Managers managers = Whitebox.getInternalState(service, "managers");
    DataClientManager clientManager = managers.get(DataClientManager.NAME);
    ConnectionManager connManager = Whitebox.getInternalState(clientManager, "connManager");
    NettyTransportClient client = (NettyTransportClient) connManager.getOrCreateClient("127.0.0.1", 8091);
    ClientSession clientSession = Whitebox.invoke(client.getClass(), "clientSession", client);
    Function<Message, Future<Void>> sendFuncBak = Whitebox.getInternalState(clientSession, "sendFunction");
    Function<Message, Future<Void>> sendFunc = message -> {
        try {
            Thread.sleep(100);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        return sendFuncBak.apply(message);
    };
    Whitebox.setInternalState(clientSession, "sendFunction", sendFunc);
}
Also used : PageRankParams(com.baidu.hugegraph.computer.algorithm.centrality.pagerank.PageRankParams) ComputerOptions(com.baidu.hugegraph.computer.core.config.ComputerOptions) DoubleValue(com.baidu.hugegraph.computer.core.graph.value.DoubleValue) BeforeClass(org.junit.BeforeClass) Managers(com.baidu.hugegraph.computer.core.manager.Managers) RpcOptions(com.baidu.hugegraph.config.RpcOptions) Function(java.util.function.Function) MasterService(com.baidu.hugegraph.computer.core.master.MasterService) ArrayList(java.util.ArrayList) Future(java.util.concurrent.Future) Whitebox(com.baidu.hugegraph.testutil.Whitebox) TransportException(com.baidu.hugegraph.computer.core.common.exception.TransportException) ComputerContextUtil(com.baidu.hugegraph.computer.core.util.ComputerContextUtil) Message(com.baidu.hugegraph.computer.core.network.message.Message) AfterClass(org.junit.AfterClass) Logger(org.slf4j.Logger) ConnectionManager(com.baidu.hugegraph.computer.core.network.connection.ConnectionManager) Test(org.junit.Test) NettyTransportClient(com.baidu.hugegraph.computer.core.network.netty.NettyTransportClient) ClientSession(com.baidu.hugegraph.computer.core.network.session.ClientSession) Config(com.baidu.hugegraph.computer.core.config.Config) DataClientManager(com.baidu.hugegraph.computer.core.network.DataClientManager) List(java.util.List) Log(com.baidu.hugegraph.util.Log) WorkerService(com.baidu.hugegraph.computer.core.worker.WorkerService) Assert(com.baidu.hugegraph.testutil.Assert) ConnectionManager(com.baidu.hugegraph.computer.core.network.connection.ConnectionManager) Message(com.baidu.hugegraph.computer.core.network.message.Message) Managers(com.baidu.hugegraph.computer.core.manager.Managers) DataClientManager(com.baidu.hugegraph.computer.core.network.DataClientManager) NettyTransportClient(com.baidu.hugegraph.computer.core.network.netty.NettyTransportClient) ClientSession(com.baidu.hugegraph.computer.core.network.session.ClientSession) Future(java.util.concurrent.Future)

Aggregations

DataClientManager (com.baidu.hugegraph.computer.core.network.DataClientManager)3 ConnectionManager (com.baidu.hugegraph.computer.core.network.connection.ConnectionManager)2 InetSocketAddress (java.net.InetSocketAddress)2 PageRankParams (com.baidu.hugegraph.computer.algorithm.centrality.pagerank.PageRankParams)1 WorkerAggrManager (com.baidu.hugegraph.computer.core.aggregator.WorkerAggrManager)1 Bsp4Worker (com.baidu.hugegraph.computer.core.bsp.Bsp4Worker)1 ContainerInfo (com.baidu.hugegraph.computer.core.common.ContainerInfo)1 TransportException (com.baidu.hugegraph.computer.core.common.exception.TransportException)1 ComputeManager (com.baidu.hugegraph.computer.core.compute.ComputeManager)1 ComputerOptions (com.baidu.hugegraph.computer.core.config.ComputerOptions)1 Config (com.baidu.hugegraph.computer.core.config.Config)1 DoubleValue (com.baidu.hugegraph.computer.core.graph.value.DoubleValue)1 WorkerInputManager (com.baidu.hugegraph.computer.core.input.WorkerInputManager)1 Managers (com.baidu.hugegraph.computer.core.manager.Managers)1 MasterService (com.baidu.hugegraph.computer.core.master.MasterService)1 DataServerManager (com.baidu.hugegraph.computer.core.network.DataServerManager)1 TransportConnectionManager (com.baidu.hugegraph.computer.core.network.connection.TransportConnectionManager)1 Message (com.baidu.hugegraph.computer.core.network.message.Message)1 NettyTransportClient (com.baidu.hugegraph.computer.core.network.netty.NettyTransportClient)1 ClientSession (com.baidu.hugegraph.computer.core.network.session.ClientSession)1