Search in sources :

Example 1 with SuperstepFileGenerator

use of com.baidu.hugegraph.computer.core.store.SuperstepFileGenerator in project hugegraph-computer by hugegraph.

the class MessageRecvManager method init.

@Override
public void init(Config config) {
    SuperstepFileGenerator fileGenerator = new SuperstepFileGenerator(this.fileManager, Constants.INPUT_SUPERSTEP);
    this.vertexPartitions = new VertexMessageRecvPartitions(this.context, fileGenerator, this.sortManager);
    this.edgePartitions = new EdgeMessageRecvPartitions(this.context, fileGenerator, this.sortManager);
    this.workerCount = config.get(ComputerOptions.JOB_WORKERS_COUNT);
    // One for vertex and one for edge.
    this.expectedFinishMessages = this.workerCount * 2;
    this.finishMessagesLatch = new CountDownLatch(this.expectedFinishMessages);
    this.waitFinishMessagesTimeout = config.get(ComputerOptions.WORKER_WAIT_FINISH_MESSAGES_TIMEOUT);
}
Also used : VertexMessageRecvPartitions(com.baidu.hugegraph.computer.core.receiver.vertex.VertexMessageRecvPartitions) EdgeMessageRecvPartitions(com.baidu.hugegraph.computer.core.receiver.edge.EdgeMessageRecvPartitions) SuperstepFileGenerator(com.baidu.hugegraph.computer.core.store.SuperstepFileGenerator) CountDownLatch(java.util.concurrent.CountDownLatch)

Example 2 with SuperstepFileGenerator

use of com.baidu.hugegraph.computer.core.store.SuperstepFileGenerator in project hugegraph-computer by hugegraph.

the class EdgeMessageRecvPartitionTest method testNotOverwritePropertiesCombiner.

@Test
public void testNotOverwritePropertiesCombiner() throws IOException {
    this.config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.JOB_ID, "local_001", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.JOB_PARTITIONS_COUNT, "1", ComputerOptions.WORKER_DATA_DIRS, "[data_dir1, data_dir2]", ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "10000", ComputerOptions.HGKV_MERGE_FILES_NUM, "5", ComputerOptions.WORKER_EDGE_PROPERTIES_COMBINER_CLASS, MergeNewPropertiesCombiner.class.getName(), ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
    FileUtils.deleteQuietly(new File("data_dir1"));
    FileUtils.deleteQuietly(new File("data_dir2"));
    this.fileManager = new FileManager();
    this.fileManager.init(this.config);
    SuperstepFileGenerator fileGenerator = new SuperstepFileGenerator(this.fileManager, Constants.INPUT_SUPERSTEP);
    this.partition = new EdgeMessageRecvPartition(context(), fileGenerator, this.sortManager);
    Assert.assertEquals("edge", this.partition.type());
    addTenDuplicateEdgeBuffer(this.partition::addBuffer);
    checkTenEdgesWithCombinedProperties(this.partition.iterator());
}
Also used : SuperstepFileGenerator(com.baidu.hugegraph.computer.core.store.SuperstepFileGenerator) File(java.io.File) FileManager(com.baidu.hugegraph.computer.core.store.FileManager) Test(org.junit.Test)

Example 3 with SuperstepFileGenerator

use of com.baidu.hugegraph.computer.core.store.SuperstepFileGenerator in project hugegraph-computer by hugegraph.

the class VertexMessageRecvPartitionTest method setup.

@Before
public void setup() {
    this.config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.JOB_ID, "local_001", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.JOB_PARTITIONS_COUNT, "1", ComputerOptions.WORKER_DATA_DIRS, "[data_dir1, data_dir2]", ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "20", ComputerOptions.HGKV_MERGE_FILES_NUM, "5", ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
    FileUtils.deleteQuietly(new File("data_dir1"));
    FileUtils.deleteQuietly(new File("data_dir2"));
    this.fileManager = new FileManager();
    this.fileManager.init(this.config);
    this.sortManager = new RecvSortManager(context());
    this.sortManager.init(this.config);
    SuperstepFileGenerator fileGenerator = new SuperstepFileGenerator(this.fileManager, Constants.INPUT_SUPERSTEP);
    this.partition = new VertexMessageRecvPartition(context(), fileGenerator, this.sortManager);
}
Also used : SuperstepFileGenerator(com.baidu.hugegraph.computer.core.store.SuperstepFileGenerator) File(java.io.File) FileManager(com.baidu.hugegraph.computer.core.store.FileManager) RecvSortManager(com.baidu.hugegraph.computer.core.sort.sorting.RecvSortManager) Before(org.junit.Before)

Example 4 with SuperstepFileGenerator

use of com.baidu.hugegraph.computer.core.store.SuperstepFileGenerator in project hugegraph-computer by hugegraph.

the class VertexMessageRecvPartitionTest method testMergePropertiesCombiner.

@Test
public void testMergePropertiesCombiner() throws IOException {
    this.config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.JOB_ID, "local_001", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.JOB_PARTITIONS_COUNT, "1", ComputerOptions.WORKER_DATA_DIRS, "[data_dir1, data_dir2]", ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "10000", ComputerOptions.HGKV_MERGE_FILES_NUM, "5", ComputerOptions.WORKER_VERTEX_PROPERTIES_COMBINER_CLASS, MergeNewPropertiesCombiner.class.getName(), ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
    FileUtils.deleteQuietly(new File("data_dir1"));
    FileUtils.deleteQuietly(new File("data_dir2"));
    this.fileManager = new FileManager();
    this.fileManager.init(this.config);
    SuperstepFileGenerator fileGenerator = new SuperstepFileGenerator(this.fileManager, Constants.INPUT_SUPERSTEP);
    this.partition = new VertexMessageRecvPartition(context(), fileGenerator, this.sortManager);
    addTwentyDuplicateVertexBuffer(this.partition::addBuffer);
    checkTenVertexWithMergedProperties(this.partition.iterator());
    this.fileManager.close(this.config);
}
Also used : SuperstepFileGenerator(com.baidu.hugegraph.computer.core.store.SuperstepFileGenerator) File(java.io.File) FileManager(com.baidu.hugegraph.computer.core.store.FileManager) Test(org.junit.Test)

Example 5 with SuperstepFileGenerator

use of com.baidu.hugegraph.computer.core.store.SuperstepFileGenerator in project hugegraph-computer by hugegraph.

the class EdgeMessageRecvPartitionTest method setup.

@Before
public void setup() {
    this.config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.JOB_ID, "local_001", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.JOB_PARTITIONS_COUNT, "1", ComputerOptions.WORKER_DATA_DIRS, "[data_dir1, data_dir2]", ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "100", ComputerOptions.HGKV_MERGE_FILES_NUM, "2", ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
    FileUtils.deleteQuietly(new File("data_dir1"));
    FileUtils.deleteQuietly(new File("data_dir2"));
    this.fileManager = new FileManager();
    this.fileManager.init(this.config);
    this.sortManager = new RecvSortManager(context());
    this.sortManager.init(this.config);
    SuperstepFileGenerator fileGenerator = new SuperstepFileGenerator(this.fileManager, Constants.INPUT_SUPERSTEP);
    this.partition = new EdgeMessageRecvPartition(context(), fileGenerator, this.sortManager);
}
Also used : SuperstepFileGenerator(com.baidu.hugegraph.computer.core.store.SuperstepFileGenerator) File(java.io.File) FileManager(com.baidu.hugegraph.computer.core.store.FileManager) RecvSortManager(com.baidu.hugegraph.computer.core.sort.sorting.RecvSortManager) Before(org.junit.Before)

Aggregations

SuperstepFileGenerator (com.baidu.hugegraph.computer.core.store.SuperstepFileGenerator)9 FileManager (com.baidu.hugegraph.computer.core.store.FileManager)7 File (java.io.File)7 Test (org.junit.Test)5 RecvSortManager (com.baidu.hugegraph.computer.core.sort.sorting.RecvSortManager)4 Config (com.baidu.hugegraph.computer.core.config.Config)2 SortManager (com.baidu.hugegraph.computer.core.sort.sorting.SortManager)2 CountDownLatch (java.util.concurrent.CountDownLatch)2 Before (org.junit.Before)2 DoubleValueSumCombiner (com.baidu.hugegraph.computer.core.combiner.DoubleValueSumCombiner)1 Null (com.baidu.hugegraph.computer.core.config.Null)1 DoubleValue (com.baidu.hugegraph.computer.core.graph.value.DoubleValue)1 IdList (com.baidu.hugegraph.computer.core.graph.value.IdList)1 EdgeMessageRecvPartitions (com.baidu.hugegraph.computer.core.receiver.edge.EdgeMessageRecvPartitions)1 ComputeMessageRecvPartitions (com.baidu.hugegraph.computer.core.receiver.message.ComputeMessageRecvPartitions)1 VertexMessageRecvPartitions (com.baidu.hugegraph.computer.core.receiver.vertex.VertexMessageRecvPartitions)1