Search in sources :

Example 1 with FileManager

use of com.baidu.hugegraph.computer.core.store.FileManager in project hugegraph-computer by hugegraph.

the class EdgeMessageRecvPartitionTest method testNotOverwritePropertiesCombiner.

@Test
public void testNotOverwritePropertiesCombiner() throws IOException {
    this.config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.JOB_ID, "local_001", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.JOB_PARTITIONS_COUNT, "1", ComputerOptions.WORKER_DATA_DIRS, "[data_dir1, data_dir2]", ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "10000", ComputerOptions.HGKV_MERGE_FILES_NUM, "5", ComputerOptions.WORKER_EDGE_PROPERTIES_COMBINER_CLASS, MergeNewPropertiesCombiner.class.getName(), ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
    FileUtils.deleteQuietly(new File("data_dir1"));
    FileUtils.deleteQuietly(new File("data_dir2"));
    this.fileManager = new FileManager();
    this.fileManager.init(this.config);
    SuperstepFileGenerator fileGenerator = new SuperstepFileGenerator(this.fileManager, Constants.INPUT_SUPERSTEP);
    this.partition = new EdgeMessageRecvPartition(context(), fileGenerator, this.sortManager);
    Assert.assertEquals("edge", this.partition.type());
    addTenDuplicateEdgeBuffer(this.partition::addBuffer);
    checkTenEdgesWithCombinedProperties(this.partition.iterator());
}
Also used : SuperstepFileGenerator(com.baidu.hugegraph.computer.core.store.SuperstepFileGenerator) File(java.io.File) FileManager(com.baidu.hugegraph.computer.core.store.FileManager) Test(org.junit.Test)

Example 2 with FileManager

use of com.baidu.hugegraph.computer.core.store.FileManager in project hugegraph-computer by hugegraph.

the class VertexMessageRecvPartitionTest method setup.

@Before
public void setup() {
    this.config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.JOB_ID, "local_001", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.JOB_PARTITIONS_COUNT, "1", ComputerOptions.WORKER_DATA_DIRS, "[data_dir1, data_dir2]", ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "20", ComputerOptions.HGKV_MERGE_FILES_NUM, "5", ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
    FileUtils.deleteQuietly(new File("data_dir1"));
    FileUtils.deleteQuietly(new File("data_dir2"));
    this.fileManager = new FileManager();
    this.fileManager.init(this.config);
    this.sortManager = new RecvSortManager(context());
    this.sortManager.init(this.config);
    SuperstepFileGenerator fileGenerator = new SuperstepFileGenerator(this.fileManager, Constants.INPUT_SUPERSTEP);
    this.partition = new VertexMessageRecvPartition(context(), fileGenerator, this.sortManager);
}
Also used : SuperstepFileGenerator(com.baidu.hugegraph.computer.core.store.SuperstepFileGenerator) File(java.io.File) FileManager(com.baidu.hugegraph.computer.core.store.FileManager) RecvSortManager(com.baidu.hugegraph.computer.core.sort.sorting.RecvSortManager) Before(org.junit.Before)

Example 3 with FileManager

use of com.baidu.hugegraph.computer.core.store.FileManager in project hugegraph-computer by hugegraph.

the class VertexMessageRecvPartitionTest method testMergePropertiesCombiner.

@Test
public void testMergePropertiesCombiner() throws IOException {
    this.config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.JOB_ID, "local_001", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.JOB_PARTITIONS_COUNT, "1", ComputerOptions.WORKER_DATA_DIRS, "[data_dir1, data_dir2]", ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "10000", ComputerOptions.HGKV_MERGE_FILES_NUM, "5", ComputerOptions.WORKER_VERTEX_PROPERTIES_COMBINER_CLASS, MergeNewPropertiesCombiner.class.getName(), ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
    FileUtils.deleteQuietly(new File("data_dir1"));
    FileUtils.deleteQuietly(new File("data_dir2"));
    this.fileManager = new FileManager();
    this.fileManager.init(this.config);
    SuperstepFileGenerator fileGenerator = new SuperstepFileGenerator(this.fileManager, Constants.INPUT_SUPERSTEP);
    this.partition = new VertexMessageRecvPartition(context(), fileGenerator, this.sortManager);
    addTwentyDuplicateVertexBuffer(this.partition::addBuffer);
    checkTenVertexWithMergedProperties(this.partition.iterator());
    this.fileManager.close(this.config);
}
Also used : SuperstepFileGenerator(com.baidu.hugegraph.computer.core.store.SuperstepFileGenerator) File(java.io.File) FileManager(com.baidu.hugegraph.computer.core.store.FileManager) Test(org.junit.Test)

Example 4 with FileManager

use of com.baidu.hugegraph.computer.core.store.FileManager in project hugegraph-computer by hugegraph.

the class ComputeManagerTest method setup.

@Before
public void setup() {
    this.config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.JOB_ID, "local_001", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.JOB_PARTITIONS_COUNT, "2", ComputerOptions.BSP_MAX_SUPER_STEP, "2", ComputerOptions.WORKER_COMBINER_CLASS, // Can't combine
    Null.class.getName(), ComputerOptions.ALGORITHM_RESULT_CLASS, IdListList.class.getName(), ComputerOptions.ALGORITHM_MESSAGE_CLASS, IdList.class.getName(), ComputerOptions.WORKER_DATA_DIRS, "[data_dir1, data_dir2]", ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "10000", ComputerOptions.WORKER_WAIT_FINISH_MESSAGES_TIMEOUT, "1000", ComputerOptions.INPUT_MAX_EDGES_IN_ONE_VERTEX, "10", ComputerOptions.WORKER_COMPUTATION_CLASS, MockComputation.class.getName(), ComputerOptions.INPUT_EDGE_FREQ, "SINGLE", ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
    this.managers = new Managers();
    FileManager fileManager = new FileManager();
    this.managers.add(fileManager);
    SortManager sortManager = new SendSortManager(context());
    this.managers.add(sortManager);
    MessageSendManager sendManager = new MessageSendManager(context(), sortManager, new MockMessageSender());
    this.managers.add(sendManager);
    MessageRecvManager receiveManager = new MessageRecvManager(context(), fileManager, sortManager);
    this.managers.add(receiveManager);
    this.managers.initAll(this.config);
    this.connectionId = new ConnectionId(new InetSocketAddress("localhost", 8081), 0);
    this.computeManager = new ComputeManager(context(), this.managers);
}
Also used : SendSortManager(com.baidu.hugegraph.computer.core.sort.sorting.SendSortManager) MessageRecvManager(com.baidu.hugegraph.computer.core.receiver.MessageRecvManager) ConnectionId(com.baidu.hugegraph.computer.core.network.ConnectionId) Managers(com.baidu.hugegraph.computer.core.manager.Managers) InetSocketAddress(java.net.InetSocketAddress) MessageSendManager(com.baidu.hugegraph.computer.core.sender.MessageSendManager) FileManager(com.baidu.hugegraph.computer.core.store.FileManager) SortManager(com.baidu.hugegraph.computer.core.sort.sorting.SortManager) SendSortManager(com.baidu.hugegraph.computer.core.sort.sorting.SendSortManager) Before(org.junit.Before)

Example 5 with FileManager

use of com.baidu.hugegraph.computer.core.store.FileManager in project hugegraph-computer by hugegraph.

the class WorkerService method initManagers.

private InetSocketAddress initManagers(ContainerInfo masterInfo) {
    // Create managers
    WorkerRpcManager rpcManager = new WorkerRpcManager();
    this.managers.add(rpcManager);
    /*
         * NOTE: this init() method will be called twice, will be ignored at
         * the 2nd time call.
         */
    WorkerRpcManager.updateRpcRemoteServerConfig(this.config, masterInfo.hostname(), masterInfo.rpcPort());
    rpcManager.init(this.config);
    WorkerAggrManager aggregatorManager = new WorkerAggrManager(this.context);
    aggregatorManager.service(rpcManager.aggregateRpcService());
    this.managers.add(aggregatorManager);
    FileManager fileManager = new FileManager();
    this.managers.add(fileManager);
    SortManager recvSortManager = new RecvSortManager(this.context);
    this.managers.add(recvSortManager);
    MessageRecvManager recvManager = new MessageRecvManager(this.context, fileManager, recvSortManager);
    this.managers.add(recvManager);
    ConnectionManager connManager = new TransportConnectionManager();
    DataServerManager serverManager = new DataServerManager(connManager, recvManager);
    this.managers.add(serverManager);
    DataClientManager clientManager = new DataClientManager(connManager, this.context);
    this.managers.add(clientManager);
    SortManager sendSortManager = new SendSortManager(this.context);
    this.managers.add(sendSortManager);
    MessageSendManager sendManager = new MessageSendManager(this.context, sendSortManager, clientManager.sender());
    this.managers.add(sendManager);
    WorkerInputManager inputManager = new WorkerInputManager(this.context, sendManager);
    inputManager.service(rpcManager.inputSplitService());
    this.managers.add(inputManager);
    // Init all managers
    this.managers.initAll(this.config);
    InetSocketAddress address = serverManager.address();
    LOG.info("{} WorkerService initialized managers with data server " + "address '{}'", this, address);
    return address;
}
Also used : WorkerAggrManager(com.baidu.hugegraph.computer.core.aggregator.WorkerAggrManager) DataServerManager(com.baidu.hugegraph.computer.core.network.DataServerManager) InetSocketAddress(java.net.InetSocketAddress) MessageSendManager(com.baidu.hugegraph.computer.core.sender.MessageSendManager) FileManager(com.baidu.hugegraph.computer.core.store.FileManager) RecvSortManager(com.baidu.hugegraph.computer.core.sort.sorting.RecvSortManager) SortManager(com.baidu.hugegraph.computer.core.sort.sorting.SortManager) SendSortManager(com.baidu.hugegraph.computer.core.sort.sorting.SendSortManager) WorkerRpcManager(com.baidu.hugegraph.computer.core.rpc.WorkerRpcManager) SendSortManager(com.baidu.hugegraph.computer.core.sort.sorting.SendSortManager) MessageRecvManager(com.baidu.hugegraph.computer.core.receiver.MessageRecvManager) TransportConnectionManager(com.baidu.hugegraph.computer.core.network.connection.TransportConnectionManager) ConnectionManager(com.baidu.hugegraph.computer.core.network.connection.ConnectionManager) DataClientManager(com.baidu.hugegraph.computer.core.network.DataClientManager) WorkerInputManager(com.baidu.hugegraph.computer.core.input.WorkerInputManager) RecvSortManager(com.baidu.hugegraph.computer.core.sort.sorting.RecvSortManager) TransportConnectionManager(com.baidu.hugegraph.computer.core.network.connection.TransportConnectionManager)

Aggregations

FileManager (com.baidu.hugegraph.computer.core.store.FileManager)13 RecvSortManager (com.baidu.hugegraph.computer.core.sort.sorting.RecvSortManager)8 File (java.io.File)8 SortManager (com.baidu.hugegraph.computer.core.sort.sorting.SortManager)7 SuperstepFileGenerator (com.baidu.hugegraph.computer.core.store.SuperstepFileGenerator)7 InetSocketAddress (java.net.InetSocketAddress)6 Test (org.junit.Test)6 MessageRecvManager (com.baidu.hugegraph.computer.core.receiver.MessageRecvManager)5 Before (org.junit.Before)5 ConnectionId (com.baidu.hugegraph.computer.core.network.ConnectionId)4 Config (com.baidu.hugegraph.computer.core.config.Config)3 Managers (com.baidu.hugegraph.computer.core.manager.Managers)3 MessageSendManager (com.baidu.hugegraph.computer.core.sender.MessageSendManager)3 SendSortManager (com.baidu.hugegraph.computer.core.sort.sorting.SendSortManager)3 ConnectionManager (com.baidu.hugegraph.computer.core.network.connection.ConnectionManager)2 TransportConnectionManager (com.baidu.hugegraph.computer.core.network.connection.TransportConnectionManager)2 WorkerAggrManager (com.baidu.hugegraph.computer.core.aggregator.WorkerAggrManager)1 DoubleValueSumCombiner (com.baidu.hugegraph.computer.core.combiner.DoubleValueSumCombiner)1 TransportException (com.baidu.hugegraph.computer.core.common.exception.TransportException)1 FileGraphPartition (com.baidu.hugegraph.computer.core.compute.FileGraphPartition)1