use of com.baidu.hugegraph.computer.core.store.SuperstepFileGenerator in project hugegraph-computer by hugegraph.
the class MessageRecvManager method init.
@Override
public void init(Config config) {
SuperstepFileGenerator fileGenerator = new SuperstepFileGenerator(this.fileManager, Constants.INPUT_SUPERSTEP);
this.vertexPartitions = new VertexMessageRecvPartitions(this.context, fileGenerator, this.sortManager);
this.edgePartitions = new EdgeMessageRecvPartitions(this.context, fileGenerator, this.sortManager);
this.workerCount = config.get(ComputerOptions.JOB_WORKERS_COUNT);
// One for vertex and one for edge.
this.expectedFinishMessages = this.workerCount * 2;
this.finishMessagesLatch = new CountDownLatch(this.expectedFinishMessages);
this.waitFinishMessagesTimeout = config.get(ComputerOptions.WORKER_WAIT_FINISH_MESSAGES_TIMEOUT);
}
use of com.baidu.hugegraph.computer.core.store.SuperstepFileGenerator in project hugegraph-computer by hugegraph.
the class EdgeMessageRecvPartitionTest method testNotOverwritePropertiesCombiner.
@Test
public void testNotOverwritePropertiesCombiner() throws IOException {
this.config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.JOB_ID, "local_001", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.JOB_PARTITIONS_COUNT, "1", ComputerOptions.WORKER_DATA_DIRS, "[data_dir1, data_dir2]", ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "10000", ComputerOptions.HGKV_MERGE_FILES_NUM, "5", ComputerOptions.WORKER_EDGE_PROPERTIES_COMBINER_CLASS, MergeNewPropertiesCombiner.class.getName(), ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
FileUtils.deleteQuietly(new File("data_dir1"));
FileUtils.deleteQuietly(new File("data_dir2"));
this.fileManager = new FileManager();
this.fileManager.init(this.config);
SuperstepFileGenerator fileGenerator = new SuperstepFileGenerator(this.fileManager, Constants.INPUT_SUPERSTEP);
this.partition = new EdgeMessageRecvPartition(context(), fileGenerator, this.sortManager);
Assert.assertEquals("edge", this.partition.type());
addTenDuplicateEdgeBuffer(this.partition::addBuffer);
checkTenEdgesWithCombinedProperties(this.partition.iterator());
}
use of com.baidu.hugegraph.computer.core.store.SuperstepFileGenerator in project hugegraph-computer by hugegraph.
the class VertexMessageRecvPartitionTest method setup.
@Before
public void setup() {
this.config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.JOB_ID, "local_001", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.JOB_PARTITIONS_COUNT, "1", ComputerOptions.WORKER_DATA_DIRS, "[data_dir1, data_dir2]", ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "20", ComputerOptions.HGKV_MERGE_FILES_NUM, "5", ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
FileUtils.deleteQuietly(new File("data_dir1"));
FileUtils.deleteQuietly(new File("data_dir2"));
this.fileManager = new FileManager();
this.fileManager.init(this.config);
this.sortManager = new RecvSortManager(context());
this.sortManager.init(this.config);
SuperstepFileGenerator fileGenerator = new SuperstepFileGenerator(this.fileManager, Constants.INPUT_SUPERSTEP);
this.partition = new VertexMessageRecvPartition(context(), fileGenerator, this.sortManager);
}
use of com.baidu.hugegraph.computer.core.store.SuperstepFileGenerator in project hugegraph-computer by hugegraph.
the class VertexMessageRecvPartitionTest method testMergePropertiesCombiner.
@Test
public void testMergePropertiesCombiner() throws IOException {
this.config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.JOB_ID, "local_001", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.JOB_PARTITIONS_COUNT, "1", ComputerOptions.WORKER_DATA_DIRS, "[data_dir1, data_dir2]", ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "10000", ComputerOptions.HGKV_MERGE_FILES_NUM, "5", ComputerOptions.WORKER_VERTEX_PROPERTIES_COMBINER_CLASS, MergeNewPropertiesCombiner.class.getName(), ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
FileUtils.deleteQuietly(new File("data_dir1"));
FileUtils.deleteQuietly(new File("data_dir2"));
this.fileManager = new FileManager();
this.fileManager.init(this.config);
SuperstepFileGenerator fileGenerator = new SuperstepFileGenerator(this.fileManager, Constants.INPUT_SUPERSTEP);
this.partition = new VertexMessageRecvPartition(context(), fileGenerator, this.sortManager);
addTwentyDuplicateVertexBuffer(this.partition::addBuffer);
checkTenVertexWithMergedProperties(this.partition.iterator());
this.fileManager.close(this.config);
}
use of com.baidu.hugegraph.computer.core.store.SuperstepFileGenerator in project hugegraph-computer by hugegraph.
the class EdgeMessageRecvPartitionTest method setup.
@Before
public void setup() {
this.config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.JOB_ID, "local_001", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.JOB_PARTITIONS_COUNT, "1", ComputerOptions.WORKER_DATA_DIRS, "[data_dir1, data_dir2]", ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "100", ComputerOptions.HGKV_MERGE_FILES_NUM, "2", ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
FileUtils.deleteQuietly(new File("data_dir1"));
FileUtils.deleteQuietly(new File("data_dir2"));
this.fileManager = new FileManager();
this.fileManager.init(this.config);
this.sortManager = new RecvSortManager(context());
this.sortManager.init(this.config);
SuperstepFileGenerator fileGenerator = new SuperstepFileGenerator(this.fileManager, Constants.INPUT_SUPERSTEP);
this.partition = new EdgeMessageRecvPartition(context(), fileGenerator, this.sortManager);
}
Aggregations