Examples with HgkvDirBuilderImpl com.baidu.hugegraph.computer.core.store.file.hgkvfile.builder.HgkvDirBuilderImpl used on opensource projects

Example 1 with HgkvDirBuilderImpl

use of com.baidu.hugegraph.computer.core.store.file.hgkvfile.builder.HgkvDirBuilderImpl in project hugegraph-computer by hugegraph.

the class HgkvDirTest method testHgkvDirBuilder.

@Test
public void testHgkvDirBuilder() throws IOException {
    // The data must be ordered
    List<Integer> data = ImmutableList.of(2, 3, 2, 1, 5, 2, 5, 9, 6, 2);
    List<KvEntry> kvEntries = StoreTestUtil.kvEntriesFromMap(data);
    String path = StoreTestUtil.availablePathById("1");
    try (KvEntryFileWriter builder = new HgkvDirBuilderImpl(CONFIG, path)) {
        for (KvEntry entry : kvEntries) {
            builder.write(entry);
        }
        builder.finish();
        // Open the file and determine the footer is as expected
        HgkvDir dir = HgkvDirImpl.open(path);
        Assert.assertEquals(HgkvFileImpl.MAGIC, dir.magic());
        String version = HgkvDirImpl.MAJOR_VERSION + "." + HgkvDirImpl.MINOR_VERSION;
        Assert.assertEquals(version, dir.version());
        Assert.assertEquals(5, dir.numEntries());
        int maxKey = StoreTestUtil.byteArrayToInt(dir.max());
        Assert.assertEquals(6, maxKey);
        int minKey = StoreTestUtil.byteArrayToInt(dir.min());
        Assert.assertEquals(2, minKey);
    } finally {
        FileUtils.deleteQuietly(new File(path));
    }
}

Example 2 with HgkvDirBuilderImpl

use of com.baidu.hugegraph.computer.core.store.file.hgkvfile.builder.HgkvDirBuilderImpl in project hugegraph-computer by hugegraph.

the class StoreTestUtil method hgkvDirFromKvMap.

public static void hgkvDirFromKvMap(Config config, List<Integer> map, String path) throws IOException {
    File file = new File(path);
    try (KvEntryFileWriter builder = new HgkvDirBuilderImpl(config, path)) {
        List<KvEntry> entries = StoreTestUtil.kvEntriesFromMap(map);
        for (KvEntry entry : entries) {
            builder.write(entry);
        }
        builder.finish();
    } catch (Exception e) {
        FileUtils.deleteQuietly(file);
        throw e;
    }
}

Example 3 with HgkvDirBuilderImpl

use of com.baidu.hugegraph.computer.core.store.file.hgkvfile.builder.HgkvDirBuilderImpl in project hugegraph-computer by hugegraph.

the class HgkvFileSorter method mergeInputs.

@Override
public void mergeInputs(List<String> inputs, OuterSortFlusher flusher, List<String> outputs, boolean withSubKv) throws Exception {
    Function<String, EntryIterator> fileToInput;
    Function<String, KvEntryFileWriter> fileToWriter;
    if (withSubKv) {
        fileToInput = o -> new HgkvDir4SubKvReaderImpl(o).iterator();
    } else {
        fileToInput = o -> new HgkvDirReaderImpl(o).iterator();
    }
    fileToWriter = path -> new HgkvDirBuilderImpl(this.config, path);
    InputFilesSelector selector = new DisperseEvenlySelector();
    List<SelectedFiles> selectResult = selector.selectedByHgkvFile(inputs, outputs);
    this.sorter.mergeFile(selectResult, fileToInput, fileToWriter, flusher);
}

Example 4 with HgkvDirBuilderImpl

use of com.baidu.hugegraph.computer.core.store.file.hgkvfile.builder.HgkvDirBuilderImpl in project hugegraph-computer by hugegraph.

the class StoreTestUtil method hgkvDirFromSubKvMap.

public static void hgkvDirFromSubKvMap(Config config, List<List<Integer>> map, String path) throws IOException {
    BytesInput input = SorterTestUtil.inputFromSubKvMap(map);
    KvEntriesInput iter = new KvEntriesInput(input, true);
    try (KvEntryFileWriter builder = new HgkvDirBuilderImpl(config, path)) {
        while (iter.hasNext()) {
            builder.write(iter.next());
        }
    }
    iter.close();
}

Example 5 with HgkvDirBuilderImpl

use of com.baidu.hugegraph.computer.core.store.file.hgkvfile.builder.HgkvDirBuilderImpl in project hugegraph-computer by hugegraph.

the class SortLargeDataTest method testDiffNumEntriesFileMerge.

@Test
public void testDiffNumEntriesFileMerge() throws Exception {
    Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.HGKV_MERGE_FILES_NUM, "3", ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
    List<Integer> sizeList = ImmutableList.of(200, 500, 20, 50, 300, 250, 10, 33, 900, 89, 20);
    List<String> inputs = new ArrayList<>();
    for (int j = 0; j < sizeList.size(); j++) {
        String file = StoreTestUtil.availablePathById(j + 10);
        inputs.add(file);
        try (KvEntryFileWriter builder = new HgkvDirBuilderImpl(config, file)) {
            for (int i = 0; i < sizeList.get(j); i++) {
                byte[] keyBytes = StoreTestUtil.intToByteArray(i);
                byte[] valueBytes = StoreTestUtil.intToByteArray(1);
                Pointer key = new InlinePointer(keyBytes);
                Pointer value = new InlinePointer(valueBytes);
                KvEntry entry = new DefaultKvEntry(key, value);
                builder.write(entry);
            }
        }
    }
    List<String> outputs = ImmutableList.of(StoreTestUtil.availablePathById(0), StoreTestUtil.availablePathById(1), StoreTestUtil.availablePathById(2), StoreTestUtil.availablePathById(3));
    Sorter sorter = SorterTestUtil.createSorter(config);
    sorter.mergeInputs(inputs, new KvOuterSortFlusher(), outputs, false);
    int total = sizeList.stream().mapToInt(i -> i).sum();
    int mergeTotal = 0;
    for (String output : outputs) {
        mergeTotal += HgkvDirImpl.open(output).numEntries();
    }
    Assert.assertEquals(total, mergeTotal);
}