use of com.baidu.hugegraph.computer.core.store.SuperstepFileGenerator in project hugegraph-computer by hugegraph.
the class ComputeMessageRecvPartitionTest method testNotCombineMessageRecvPartition.
@Test
public void testNotCombineMessageRecvPartition() throws IOException {
Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.JOB_ID, "local_001", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.JOB_PARTITIONS_COUNT, "1", ComputerOptions.WORKER_COMBINER_CLASS, Null.class.getName(), ComputerOptions.WORKER_DATA_DIRS, "[data_dir1, data_dir2]", ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "10", ComputerOptions.ALGORITHM_MESSAGE_CLASS, IdList.class.getName(), ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
FileUtils.deleteQuietly(new File("data_dir1"));
FileUtils.deleteQuietly(new File("data_dir2"));
FileManager fileManager = new FileManager();
fileManager.init(config);
SortManager sortManager = new RecvSortManager(context());
sortManager.init(config);
SuperstepFileGenerator fileGenerator = new SuperstepFileGenerator(fileManager, 0);
ComputeMessageRecvPartition partition = new ComputeMessageRecvPartition(context(), fileGenerator, sortManager);
Assert.assertEquals("msg", partition.type());
addTwentyDuplicateIdValueListMessageBuffer(partition::addBuffer);
checkIdValueListMessages(partition.iterator());
fileManager.close(config);
sortManager.close(config);
}
use of com.baidu.hugegraph.computer.core.store.SuperstepFileGenerator in project hugegraph-computer by hugegraph.
the class ComputeMessageRecvPartitionTest method testCombineMessageRecvPartition.
@Test
public void testCombineMessageRecvPartition() throws IOException {
Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.JOB_ID, "local_001", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.JOB_PARTITIONS_COUNT, "1", // Make sure all buffers within this limit.
ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "1000", ComputerOptions.WORKER_COMBINER_CLASS, DoubleValueSumCombiner.class.getName(), ComputerOptions.WORKER_DATA_DIRS, "[data_dir1, data_dir2]", ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "10", ComputerOptions.ALGORITHM_MESSAGE_CLASS, DoubleValue.class.getName(), ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
FileUtils.deleteQuietly(new File("data_dir1"));
FileUtils.deleteQuietly(new File("data_dir2"));
FileManager fileManager = new FileManager();
fileManager.init(config);
SortManager sortManager = new RecvSortManager(context());
sortManager.init(config);
SuperstepFileGenerator fileGenerator = new SuperstepFileGenerator(fileManager, 0);
ComputeMessageRecvPartition partition = new ComputeMessageRecvPartition(context(), fileGenerator, sortManager);
Assert.assertEquals("msg", partition.type());
addTwentyCombineMessageBuffer(partition::addBuffer);
checkTenCombineMessages(partition.iterator());
fileManager.close(config);
sortManager.close(config);
}
use of com.baidu.hugegraph.computer.core.store.SuperstepFileGenerator in project hugegraph-computer by hugegraph.
the class VertexMessageRecvPartitionTest method testOverwriteCombiner.
@Test
public void testOverwriteCombiner() throws IOException {
this.config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.JOB_ID, "local_001", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.JOB_PARTITIONS_COUNT, "1", ComputerOptions.WORKER_DATA_DIRS, "[data_dir1, data_dir2]", ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "1000", ComputerOptions.HGKV_MERGE_FILES_NUM, "5", ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
FileUtils.deleteQuietly(new File("data_dir1"));
FileUtils.deleteQuietly(new File("data_dir2"));
this.fileManager = new FileManager();
this.fileManager.init(this.config);
SuperstepFileGenerator fileGenerator = new SuperstepFileGenerator(this.fileManager, Constants.INPUT_SUPERSTEP);
this.partition = new VertexMessageRecvPartition(context(), fileGenerator, this.sortManager);
addTenVertexBuffer(this.partition::addBuffer);
addTenVertexBuffer(this.partition::addBuffer);
checkPartitionIterator(this.partition.iterator());
this.fileManager.close(this.config);
}
use of com.baidu.hugegraph.computer.core.store.SuperstepFileGenerator in project hugegraph-computer by hugegraph.
the class MessageRecvManager method beforeSuperstep.
@Override
public void beforeSuperstep(Config config, int superstep) {
SuperstepFileGenerator fileGenerator = new SuperstepFileGenerator(this.fileManager, superstep);
this.messagePartitions = new ComputeMessageRecvPartitions(this.context, fileGenerator, this.sortManager);
this.expectedFinishMessages = this.workerCount;
this.finishMessagesLatch = new CountDownLatch(this.expectedFinishMessages);
this.superstep = superstep;
if (this.superstep == Constants.INPUT_SUPERSTEP + 1) {
assert this.vertexPartitions != null;
this.vertexPartitions.clearOldFiles(Constants.INPUT_SUPERSTEP);
this.vertexPartitions = null;
assert this.edgePartitions != null;
this.edgePartitions.clearOldFiles(Constants.INPUT_SUPERSTEP);
this.edgePartitions = null;
}
}
Aggregations