Examples with HgkvDirReaderImpl com.baidu.hugegraph.computer.core.store.file.hgkvfile.reader.HgkvDirReaderImpl used on opensource projects

Example 1 with HgkvDirReaderImpl

use of com.baidu.hugegraph.computer.core.store.file.hgkvfile.reader.HgkvDirReaderImpl in project hugegraph-computer by hugegraph.

the class HgkvFileSorter method mergeInputs.

@Override
public void mergeInputs(List<String> inputs, OuterSortFlusher flusher, List<String> outputs, boolean withSubKv) throws Exception {
    Function<String, EntryIterator> fileToInput;
    Function<String, KvEntryFileWriter> fileToWriter;
    if (withSubKv) {
        fileToInput = o -> new HgkvDir4SubKvReaderImpl(o).iterator();
    } else {
        fileToInput = o -> new HgkvDirReaderImpl(o).iterator();
    }
    fileToWriter = path -> new HgkvDirBuilderImpl(this.config, path);
    InputFilesSelector selector = new DisperseEvenlySelector();
    List<SelectedFiles> selectResult = selector.selectedByHgkvFile(inputs, outputs);
    this.sorter.mergeFile(selectResult, fileToInput, fileToWriter, flusher);
}

Example 2 with HgkvDirReaderImpl

use of com.baidu.hugegraph.computer.core.store.file.hgkvfile.reader.HgkvDirReaderImpl in project hugegraph-computer by hugegraph.

the class SorterTest method testSortKvBuffers.

@Test
public void testSortKvBuffers() throws Exception {
    Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.HGKV_MAX_FILE_SIZE, "32", ComputerOptions.HGKV_DATABLOCK_SIZE, "16", ComputerOptions.HGKV_MERGE_FILES_NUM, "3", ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
    List<Integer> map1 = ImmutableList.of(2, 3, 2, 1, 5, 2, 6, 9, 6, 2);
    List<Integer> map2 = ImmutableList.of(1, 3, 1, 1, 3, 2, 6, 9, 8, 2);
    String path = StoreTestUtil.availablePathById("1");
    // Merge 4 sorted input
    List<RandomAccessInput> inputs = ImmutableList.of(SorterTestUtil.inputFromKvMap(map1), SorterTestUtil.inputFromKvMap(map2), SorterTestUtil.inputFromKvMap(map1), SorterTestUtil.inputFromKvMap(map2));
    Sorter sorter = SorterTestUtil.createSorter(config);
    PointerCombiner combiner = SorterTestUtil.createPointerCombiner(IntValue::new, new IntValueSumCombiner());
    sorter.mergeBuffers(inputs, new CombineKvOuterSortFlusher(combiner), path, false);
    // Assert merge result from target hgkvDir
    KvEntryFileReader reader = new HgkvDirReaderImpl(path, false);
    EntryIterator iter = reader.iterator();
    SorterTestUtil.assertKvEntry(iter.next(), 1, 8);
    SorterTestUtil.assertKvEntry(iter.next(), 2, 8);
    SorterTestUtil.assertKvEntry(iter.next(), 3, 4);
    SorterTestUtil.assertKvEntry(iter.next(), 5, 4);
    SorterTestUtil.assertKvEntry(iter.next(), 6, 40);
    SorterTestUtil.assertKvEntry(iter.next(), 8, 4);
    Assert.assertFalse(iter.hasNext());
}

Example 3 with HgkvDirReaderImpl

use of com.baidu.hugegraph.computer.core.store.file.hgkvfile.reader.HgkvDirReaderImpl in project hugegraph-computer by hugegraph.

the class HgkvDirTest method testHgkvDirReader.

@Test
public void testHgkvDirReader() throws Exception {
    // The keys in the data must be ordered
    List<Integer> data = ImmutableList.of(2, 3, 2, 1, 5, 2, 5, 5, 5, 9, 6, 2);
    String path = StoreTestUtil.availablePathById("1");
    StoreTestUtil.hgkvDirFromKvMap(CONFIG, data, path);
    KvEntryFileReader reader = new HgkvDirReaderImpl(path, false);
    try (EntryIterator iterator = reader.iterator()) {
        int i = 0;
        while (iterator.hasNext()) {
            KvEntry entry = iterator.next();
            int key = StoreTestUtil.byteArrayToInt(entry.key().bytes());
            Assert.assertEquals(data.get(i).intValue(), key);
            i += 2;
        }
        Assert.assertThrows(NoSuchElementException.class, iterator::next);
    }
}