use of com.baidu.hugegraph.computer.core.store.FileManager in project hugegraph-computer by hugegraph.
the class EdgeMessageRecvPartitionTest method testNotOverwritePropertiesCombiner.
@Test
public void testNotOverwritePropertiesCombiner() throws IOException {
this.config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.JOB_ID, "local_001", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.JOB_PARTITIONS_COUNT, "1", ComputerOptions.WORKER_DATA_DIRS, "[data_dir1, data_dir2]", ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "10000", ComputerOptions.HGKV_MERGE_FILES_NUM, "5", ComputerOptions.WORKER_EDGE_PROPERTIES_COMBINER_CLASS, MergeNewPropertiesCombiner.class.getName(), ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
FileUtils.deleteQuietly(new File("data_dir1"));
FileUtils.deleteQuietly(new File("data_dir2"));
this.fileManager = new FileManager();
this.fileManager.init(this.config);
SuperstepFileGenerator fileGenerator = new SuperstepFileGenerator(this.fileManager, Constants.INPUT_SUPERSTEP);
this.partition = new EdgeMessageRecvPartition(context(), fileGenerator, this.sortManager);
Assert.assertEquals("edge", this.partition.type());
addTenDuplicateEdgeBuffer(this.partition::addBuffer);
checkTenEdgesWithCombinedProperties(this.partition.iterator());
}
use of com.baidu.hugegraph.computer.core.store.FileManager in project hugegraph-computer by hugegraph.
the class VertexMessageRecvPartitionTest method setup.
@Before
public void setup() {
this.config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.JOB_ID, "local_001", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.JOB_PARTITIONS_COUNT, "1", ComputerOptions.WORKER_DATA_DIRS, "[data_dir1, data_dir2]", ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "20", ComputerOptions.HGKV_MERGE_FILES_NUM, "5", ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
FileUtils.deleteQuietly(new File("data_dir1"));
FileUtils.deleteQuietly(new File("data_dir2"));
this.fileManager = new FileManager();
this.fileManager.init(this.config);
this.sortManager = new RecvSortManager(context());
this.sortManager.init(this.config);
SuperstepFileGenerator fileGenerator = new SuperstepFileGenerator(this.fileManager, Constants.INPUT_SUPERSTEP);
this.partition = new VertexMessageRecvPartition(context(), fileGenerator, this.sortManager);
}
use of com.baidu.hugegraph.computer.core.store.FileManager in project hugegraph-computer by hugegraph.
the class VertexMessageRecvPartitionTest method testMergePropertiesCombiner.
@Test
public void testMergePropertiesCombiner() throws IOException {
this.config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.JOB_ID, "local_001", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.JOB_PARTITIONS_COUNT, "1", ComputerOptions.WORKER_DATA_DIRS, "[data_dir1, data_dir2]", ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "10000", ComputerOptions.HGKV_MERGE_FILES_NUM, "5", ComputerOptions.WORKER_VERTEX_PROPERTIES_COMBINER_CLASS, MergeNewPropertiesCombiner.class.getName(), ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
FileUtils.deleteQuietly(new File("data_dir1"));
FileUtils.deleteQuietly(new File("data_dir2"));
this.fileManager = new FileManager();
this.fileManager.init(this.config);
SuperstepFileGenerator fileGenerator = new SuperstepFileGenerator(this.fileManager, Constants.INPUT_SUPERSTEP);
this.partition = new VertexMessageRecvPartition(context(), fileGenerator, this.sortManager);
addTwentyDuplicateVertexBuffer(this.partition::addBuffer);
checkTenVertexWithMergedProperties(this.partition.iterator());
this.fileManager.close(this.config);
}
use of com.baidu.hugegraph.computer.core.store.FileManager in project hugegraph-computer by hugegraph.
the class ComputeManagerTest method setup.
@Before
public void setup() {
this.config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.JOB_ID, "local_001", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.JOB_PARTITIONS_COUNT, "2", ComputerOptions.BSP_MAX_SUPER_STEP, "2", ComputerOptions.WORKER_COMBINER_CLASS, // Can't combine
Null.class.getName(), ComputerOptions.ALGORITHM_RESULT_CLASS, IdListList.class.getName(), ComputerOptions.ALGORITHM_MESSAGE_CLASS, IdList.class.getName(), ComputerOptions.WORKER_DATA_DIRS, "[data_dir1, data_dir2]", ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "10000", ComputerOptions.WORKER_WAIT_FINISH_MESSAGES_TIMEOUT, "1000", ComputerOptions.INPUT_MAX_EDGES_IN_ONE_VERTEX, "10", ComputerOptions.WORKER_COMPUTATION_CLASS, MockComputation.class.getName(), ComputerOptions.INPUT_EDGE_FREQ, "SINGLE", ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
this.managers = new Managers();
FileManager fileManager = new FileManager();
this.managers.add(fileManager);
SortManager sortManager = new SendSortManager(context());
this.managers.add(sortManager);
MessageSendManager sendManager = new MessageSendManager(context(), sortManager, new MockMessageSender());
this.managers.add(sendManager);
MessageRecvManager receiveManager = new MessageRecvManager(context(), fileManager, sortManager);
this.managers.add(receiveManager);
this.managers.initAll(this.config);
this.connectionId = new ConnectionId(new InetSocketAddress("localhost", 8081), 0);
this.computeManager = new ComputeManager(context(), this.managers);
}
use of com.baidu.hugegraph.computer.core.store.FileManager in project hugegraph-computer by hugegraph.
the class WorkerService method initManagers.
private InetSocketAddress initManagers(ContainerInfo masterInfo) {
// Create managers
WorkerRpcManager rpcManager = new WorkerRpcManager();
this.managers.add(rpcManager);
/*
* NOTE: this init() method will be called twice, will be ignored at
* the 2nd time call.
*/
WorkerRpcManager.updateRpcRemoteServerConfig(this.config, masterInfo.hostname(), masterInfo.rpcPort());
rpcManager.init(this.config);
WorkerAggrManager aggregatorManager = new WorkerAggrManager(this.context);
aggregatorManager.service(rpcManager.aggregateRpcService());
this.managers.add(aggregatorManager);
FileManager fileManager = new FileManager();
this.managers.add(fileManager);
SortManager recvSortManager = new RecvSortManager(this.context);
this.managers.add(recvSortManager);
MessageRecvManager recvManager = new MessageRecvManager(this.context, fileManager, recvSortManager);
this.managers.add(recvManager);
ConnectionManager connManager = new TransportConnectionManager();
DataServerManager serverManager = new DataServerManager(connManager, recvManager);
this.managers.add(serverManager);
DataClientManager clientManager = new DataClientManager(connManager, this.context);
this.managers.add(clientManager);
SortManager sendSortManager = new SendSortManager(this.context);
this.managers.add(sendSortManager);
MessageSendManager sendManager = new MessageSendManager(this.context, sendSortManager, clientManager.sender());
this.managers.add(sendManager);
WorkerInputManager inputManager = new WorkerInputManager(this.context, sendManager);
inputManager.service(rpcManager.inputSplitService());
this.managers.add(inputManager);
// Init all managers
this.managers.initAll(this.config);
InetSocketAddress address = serverManager.address();
LOG.info("{} WorkerService initialized managers with data server " + "address '{}'", this, address);
return address;
}
Aggregations