Examples with Config com.baidu.hugegraph.computer.core.config.Config used on opensource projects

Example 26 with Config

use of com.baidu.hugegraph.computer.core.config.Config in project hugegraph-computer by hugegraph.

the class ComputeMessageRecvPartitionTest method testCombineMessageRecvPartition.

@Test
public void testCombineMessageRecvPartition() throws IOException {
    Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.JOB_ID, "local_001", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.JOB_PARTITIONS_COUNT, "1", // Make sure all buffers within this limit.
    ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "1000", ComputerOptions.WORKER_COMBINER_CLASS, DoubleValueSumCombiner.class.getName(), ComputerOptions.WORKER_DATA_DIRS, "[data_dir1, data_dir2]", ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "10", ComputerOptions.ALGORITHM_MESSAGE_CLASS, DoubleValue.class.getName(), ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
    FileUtils.deleteQuietly(new File("data_dir1"));
    FileUtils.deleteQuietly(new File("data_dir2"));
    FileManager fileManager = new FileManager();
    fileManager.init(config);
    SortManager sortManager = new RecvSortManager(context());
    sortManager.init(config);
    SuperstepFileGenerator fileGenerator = new SuperstepFileGenerator(fileManager, 0);
    ComputeMessageRecvPartition partition = new ComputeMessageRecvPartition(context(), fileGenerator, sortManager);
    Assert.assertEquals("msg", partition.type());
    addTwentyCombineMessageBuffer(partition::addBuffer);
    checkTenCombineMessages(partition.iterator());
    fileManager.close(config);
    sortManager.close(config);
}

Example 27 with Config

use of com.baidu.hugegraph.computer.core.config.Config in project hugegraph-computer by hugegraph.

the class SorterTest method testMergeSubKvFiles.

@Test
public void testMergeSubKvFiles() throws Exception {
    Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.INPUT_MAX_EDGES_IN_ONE_VERTEX, "2", ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
    this.testMergeSubKvFiles(config);
    config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.INPUT_MAX_EDGES_IN_ONE_VERTEX, "2", ComputerOptions.TRANSPORT_RECV_FILE_MODE, "true");
    this.testMergeSubKvFiles(config);
}

Example 28 with Config

use of com.baidu.hugegraph.computer.core.config.Config in project hugegraph-computer by hugegraph.

the class SortLargeDataTest method testDiffNumEntriesFileMerge.

@Test
public void testDiffNumEntriesFileMerge() throws Exception {
    Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.HGKV_MERGE_FILES_NUM, "3", ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
    List<Integer> sizeList = ImmutableList.of(200, 500, 20, 50, 300, 250, 10, 33, 900, 89, 20);
    List<String> inputs = new ArrayList<>();
    for (int j = 0; j < sizeList.size(); j++) {
        String file = StoreTestUtil.availablePathById(j + 10);
        inputs.add(file);
        try (KvEntryFileWriter builder = new HgkvDirBuilderImpl(config, file)) {
            for (int i = 0; i < sizeList.get(j); i++) {
                byte[] keyBytes = StoreTestUtil.intToByteArray(i);
                byte[] valueBytes = StoreTestUtil.intToByteArray(1);
                Pointer key = new InlinePointer(keyBytes);
                Pointer value = new InlinePointer(valueBytes);
                KvEntry entry = new DefaultKvEntry(key, value);
                builder.write(entry);
            }
        }
    }
    List<String> outputs = ImmutableList.of(StoreTestUtil.availablePathById(0), StoreTestUtil.availablePathById(1), StoreTestUtil.availablePathById(2), StoreTestUtil.availablePathById(3));
    Sorter sorter = SorterTestUtil.createSorter(config);
    sorter.mergeInputs(inputs, new KvOuterSortFlusher(), outputs, false);
    int total = sizeList.stream().mapToInt(i -> i).sum();
    int mergeTotal = 0;
    for (String output : outputs) {
        mergeTotal += HgkvDirImpl.open(output).numEntries();
    }
    Assert.assertEquals(total, mergeTotal);
}

Example 29 with Config

use of com.baidu.hugegraph.computer.core.config.Config in project hugegraph-computer by hugegraph.

the class FileManagerTest method testInitWithFile.

@Test
public void testInitWithFile() throws IOException {
    File file = new File("exist");
    file.createNewFile();
    Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.JOB_ID, "local_001", ComputerOptions.WORKER_DATA_DIRS, "[" + file.getAbsolutePath() + "]");
    FileManager dataFileManager = new FileManager();
    Assert.assertEquals(FileManager.NAME, dataFileManager.name());
    Assert.assertThrows(ComputerException.class, () -> {
        dataFileManager.init(config);
    }, e -> {
        Assert.assertContains("Can't create dir ", e.getMessage());
    });
    file.delete();
}

Example 30 with Config

use of com.baidu.hugegraph.computer.core.config.Config in project hugegraph-computer by hugegraph.

the class FileManagerTest method testInitWithReadOnlyDir.

@Test
public void testInitWithReadOnlyDir() {
    Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.JOB_ID, "local_001", ComputerOptions.WORKER_DATA_DIRS, "[/etc]");
    FileManager dataFileManager = new FileManager();
    Assert.assertThrows(ComputerException.class, () -> {
        dataFileManager.init(config);
    }, e -> {
        Assert.assertContains("Can't create dir", e.getMessage());
    });
}