Search in sources :

Example 6 with FileManager

use of com.baidu.hugegraph.computer.core.store.FileManager in project hugegraph-computer by hugegraph.

the class DataServerManagerTest method test.

@Test
public void test() {
    Config config = UnitTestBase.updateWithRequiredOptions(RpcOptions.RPC_REMOTE_URL, "127.0.0.1:8090", ComputerOptions.JOB_ID, "local_001", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.BSP_LOG_INTERVAL, "30000", ComputerOptions.BSP_MAX_SUPER_STEP, "2", ComputerOptions.WORKER_COMPUTATION_CLASS, MockComputation.class.getName(), ComputerOptions.MASTER_COMPUTATION_CLASS, MockMasterComputation.class.getName());
    FileManager fileManager = new FileManager();
    fileManager.init(config);
    SortManager sortManager = new RecvSortManager(context());
    sortManager.init(config);
    MessageRecvManager recvManager = new MessageRecvManager(context(), fileManager, sortManager);
    recvManager.init(config);
    ConnectionManager connManager = new TransportConnectionManager();
    DataServerManager serverManager = new DataServerManager(connManager, recvManager);
    serverManager.init(config);
    Assert.assertEquals(DataServerManager.NAME, serverManager.name());
    InetSocketAddress address = serverManager.address();
    Assert.assertNotEquals(0, address.getPort());
    ConnectionId connectionId = ConnectionId.parseConnectionId(address.getHostName(), address.getPort());
    recvManager.onChannelActive(connectionId);
    recvManager.onChannelInactive(connectionId);
    TransportException e = new TransportException("test transport " + "exception");
    recvManager.exceptionCaught(e, connectionId);
    serverManager.close(config);
    fileManager.close(config);
    sortManager.close(config);
}
Also used : Config(com.baidu.hugegraph.computer.core.config.Config) InetSocketAddress(java.net.InetSocketAddress) MockMasterComputation(com.baidu.hugegraph.computer.core.worker.MockMasterComputation) TransportException(com.baidu.hugegraph.computer.core.common.exception.TransportException) FileManager(com.baidu.hugegraph.computer.core.store.FileManager) RecvSortManager(com.baidu.hugegraph.computer.core.sort.sorting.RecvSortManager) SortManager(com.baidu.hugegraph.computer.core.sort.sorting.SortManager) MessageRecvManager(com.baidu.hugegraph.computer.core.receiver.MessageRecvManager) ConnectionManager(com.baidu.hugegraph.computer.core.network.connection.ConnectionManager) TransportConnectionManager(com.baidu.hugegraph.computer.core.network.connection.TransportConnectionManager) MockComputation(com.baidu.hugegraph.computer.core.worker.MockComputation) RecvSortManager(com.baidu.hugegraph.computer.core.sort.sorting.RecvSortManager) TransportConnectionManager(com.baidu.hugegraph.computer.core.network.connection.TransportConnectionManager) Test(org.junit.Test)

Example 7 with FileManager

use of com.baidu.hugegraph.computer.core.store.FileManager in project hugegraph-computer by hugegraph.

the class EdgeMessageRecvPartitionTest method setup.

@Before
public void setup() {
    this.config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.JOB_ID, "local_001", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.JOB_PARTITIONS_COUNT, "1", ComputerOptions.WORKER_DATA_DIRS, "[data_dir1, data_dir2]", ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "100", ComputerOptions.HGKV_MERGE_FILES_NUM, "2", ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
    FileUtils.deleteQuietly(new File("data_dir1"));
    FileUtils.deleteQuietly(new File("data_dir2"));
    this.fileManager = new FileManager();
    this.fileManager.init(this.config);
    this.sortManager = new RecvSortManager(context());
    this.sortManager.init(this.config);
    SuperstepFileGenerator fileGenerator = new SuperstepFileGenerator(this.fileManager, Constants.INPUT_SUPERSTEP);
    this.partition = new EdgeMessageRecvPartition(context(), fileGenerator, this.sortManager);
}
Also used : SuperstepFileGenerator(com.baidu.hugegraph.computer.core.store.SuperstepFileGenerator) File(java.io.File) FileManager(com.baidu.hugegraph.computer.core.store.FileManager) RecvSortManager(com.baidu.hugegraph.computer.core.sort.sorting.RecvSortManager) Before(org.junit.Before)

Example 8 with FileManager

use of com.baidu.hugegraph.computer.core.store.FileManager in project hugegraph-computer by hugegraph.

the class ComputeMessageRecvPartitionTest method testNotCombineMessageRecvPartition.

@Test
public void testNotCombineMessageRecvPartition() throws IOException {
    Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.JOB_ID, "local_001", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.JOB_PARTITIONS_COUNT, "1", ComputerOptions.WORKER_COMBINER_CLASS, Null.class.getName(), ComputerOptions.WORKER_DATA_DIRS, "[data_dir1, data_dir2]", ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "10", ComputerOptions.ALGORITHM_MESSAGE_CLASS, IdList.class.getName(), ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
    FileUtils.deleteQuietly(new File("data_dir1"));
    FileUtils.deleteQuietly(new File("data_dir2"));
    FileManager fileManager = new FileManager();
    fileManager.init(config);
    SortManager sortManager = new RecvSortManager(context());
    sortManager.init(config);
    SuperstepFileGenerator fileGenerator = new SuperstepFileGenerator(fileManager, 0);
    ComputeMessageRecvPartition partition = new ComputeMessageRecvPartition(context(), fileGenerator, sortManager);
    Assert.assertEquals("msg", partition.type());
    addTwentyDuplicateIdValueListMessageBuffer(partition::addBuffer);
    checkIdValueListMessages(partition.iterator());
    fileManager.close(config);
    sortManager.close(config);
}
Also used : Null(com.baidu.hugegraph.computer.core.config.Null) SuperstepFileGenerator(com.baidu.hugegraph.computer.core.store.SuperstepFileGenerator) Config(com.baidu.hugegraph.computer.core.config.Config) File(java.io.File) IdList(com.baidu.hugegraph.computer.core.graph.value.IdList) FileManager(com.baidu.hugegraph.computer.core.store.FileManager) RecvSortManager(com.baidu.hugegraph.computer.core.sort.sorting.RecvSortManager) SortManager(com.baidu.hugegraph.computer.core.sort.sorting.SortManager) RecvSortManager(com.baidu.hugegraph.computer.core.sort.sorting.RecvSortManager) Test(org.junit.Test)

Example 9 with FileManager

use of com.baidu.hugegraph.computer.core.store.FileManager in project hugegraph-computer by hugegraph.

the class ComputeMessageRecvPartitionTest method testCombineMessageRecvPartition.

@Test
public void testCombineMessageRecvPartition() throws IOException {
    Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.JOB_ID, "local_001", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.JOB_PARTITIONS_COUNT, "1", // Make sure all buffers within this limit.
    ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "1000", ComputerOptions.WORKER_COMBINER_CLASS, DoubleValueSumCombiner.class.getName(), ComputerOptions.WORKER_DATA_DIRS, "[data_dir1, data_dir2]", ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "10", ComputerOptions.ALGORITHM_MESSAGE_CLASS, DoubleValue.class.getName(), ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
    FileUtils.deleteQuietly(new File("data_dir1"));
    FileUtils.deleteQuietly(new File("data_dir2"));
    FileManager fileManager = new FileManager();
    fileManager.init(config);
    SortManager sortManager = new RecvSortManager(context());
    sortManager.init(config);
    SuperstepFileGenerator fileGenerator = new SuperstepFileGenerator(fileManager, 0);
    ComputeMessageRecvPartition partition = new ComputeMessageRecvPartition(context(), fileGenerator, sortManager);
    Assert.assertEquals("msg", partition.type());
    addTwentyCombineMessageBuffer(partition::addBuffer);
    checkTenCombineMessages(partition.iterator());
    fileManager.close(config);
    sortManager.close(config);
}
Also used : SuperstepFileGenerator(com.baidu.hugegraph.computer.core.store.SuperstepFileGenerator) DoubleValue(com.baidu.hugegraph.computer.core.graph.value.DoubleValue) Config(com.baidu.hugegraph.computer.core.config.Config) DoubleValueSumCombiner(com.baidu.hugegraph.computer.core.combiner.DoubleValueSumCombiner) File(java.io.File) FileManager(com.baidu.hugegraph.computer.core.store.FileManager) RecvSortManager(com.baidu.hugegraph.computer.core.sort.sorting.RecvSortManager) SortManager(com.baidu.hugegraph.computer.core.sort.sorting.SortManager) RecvSortManager(com.baidu.hugegraph.computer.core.sort.sorting.RecvSortManager) Test(org.junit.Test)

Example 10 with FileManager

use of com.baidu.hugegraph.computer.core.store.FileManager in project hugegraph-computer by hugegraph.

the class VertexMessageRecvPartitionTest method testOverwriteCombiner.

@Test
public void testOverwriteCombiner() throws IOException {
    this.config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.JOB_ID, "local_001", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.JOB_PARTITIONS_COUNT, "1", ComputerOptions.WORKER_DATA_DIRS, "[data_dir1, data_dir2]", ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "1000", ComputerOptions.HGKV_MERGE_FILES_NUM, "5", ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
    FileUtils.deleteQuietly(new File("data_dir1"));
    FileUtils.deleteQuietly(new File("data_dir2"));
    this.fileManager = new FileManager();
    this.fileManager.init(this.config);
    SuperstepFileGenerator fileGenerator = new SuperstepFileGenerator(this.fileManager, Constants.INPUT_SUPERSTEP);
    this.partition = new VertexMessageRecvPartition(context(), fileGenerator, this.sortManager);
    addTenVertexBuffer(this.partition::addBuffer);
    addTenVertexBuffer(this.partition::addBuffer);
    checkPartitionIterator(this.partition.iterator());
    this.fileManager.close(this.config);
}
Also used : SuperstepFileGenerator(com.baidu.hugegraph.computer.core.store.SuperstepFileGenerator) File(java.io.File) FileManager(com.baidu.hugegraph.computer.core.store.FileManager) Test(org.junit.Test)

Aggregations

FileManager (com.baidu.hugegraph.computer.core.store.FileManager)13 RecvSortManager (com.baidu.hugegraph.computer.core.sort.sorting.RecvSortManager)8 File (java.io.File)8 SortManager (com.baidu.hugegraph.computer.core.sort.sorting.SortManager)7 SuperstepFileGenerator (com.baidu.hugegraph.computer.core.store.SuperstepFileGenerator)7 InetSocketAddress (java.net.InetSocketAddress)6 Test (org.junit.Test)6 MessageRecvManager (com.baidu.hugegraph.computer.core.receiver.MessageRecvManager)5 Before (org.junit.Before)5 ConnectionId (com.baidu.hugegraph.computer.core.network.ConnectionId)4 Config (com.baidu.hugegraph.computer.core.config.Config)3 Managers (com.baidu.hugegraph.computer.core.manager.Managers)3 MessageSendManager (com.baidu.hugegraph.computer.core.sender.MessageSendManager)3 SendSortManager (com.baidu.hugegraph.computer.core.sort.sorting.SendSortManager)3 ConnectionManager (com.baidu.hugegraph.computer.core.network.connection.ConnectionManager)2 TransportConnectionManager (com.baidu.hugegraph.computer.core.network.connection.TransportConnectionManager)2 WorkerAggrManager (com.baidu.hugegraph.computer.core.aggregator.WorkerAggrManager)1 DoubleValueSumCombiner (com.baidu.hugegraph.computer.core.combiner.DoubleValueSumCombiner)1 TransportException (com.baidu.hugegraph.computer.core.common.exception.TransportException)1 FileGraphPartition (com.baidu.hugegraph.computer.core.compute.FileGraphPartition)1