use of com.baidu.hugegraph.computer.core.store.EntryIterator in project hugegraph-computer by hugegraph.
the class HgkvFileSorter method mergeInputs.
@Override
public void mergeInputs(List<String> inputs, OuterSortFlusher flusher, List<String> outputs, boolean withSubKv) throws Exception {
Function<String, EntryIterator> fileToInput;
Function<String, KvEntryFileWriter> fileToWriter;
if (withSubKv) {
fileToInput = o -> new HgkvDir4SubKvReaderImpl(o).iterator();
} else {
fileToInput = o -> new HgkvDirReaderImpl(o).iterator();
}
fileToWriter = path -> new HgkvDirBuilderImpl(this.config, path);
InputFilesSelector selector = new DisperseEvenlySelector();
List<SelectedFiles> selectResult = selector.selectedByHgkvFile(inputs, outputs);
this.sorter.mergeFile(selectResult, fileToInput, fileToWriter, flusher);
}
use of com.baidu.hugegraph.computer.core.store.EntryIterator in project hugegraph-computer by hugegraph.
the class BufferFileSorter method mergeInputs.
@Override
public void mergeInputs(List<String> inputs, OuterSortFlusher flusher, List<String> outputs, boolean withSubKv) throws Exception {
Function<String, EntryIterator> fileToInput;
Function<String, KvEntryFileWriter> fileToWriter;
if (withSubKv) {
fileToInput = o -> new BufferFileSubEntryReader(o).iterator();
} else {
fileToInput = o -> new BufferFileEntryReader(o).iterator();
}
fileToWriter = BufferFileEntryBuilder::new;
InputFilesSelector selector = new DisperseEvenlySelector();
List<SelectedFiles> selectResult = selector.selectedByBufferFile(inputs, outputs);
this.sorter.mergeFile(selectResult, fileToInput, fileToWriter, flusher);
}
use of com.baidu.hugegraph.computer.core.store.EntryIterator in project hugegraph-computer by hugegraph.
the class SorterTest method testSortKvBuffers.
@Test
public void testSortKvBuffers() throws Exception {
Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.HGKV_MAX_FILE_SIZE, "32", ComputerOptions.HGKV_DATABLOCK_SIZE, "16", ComputerOptions.HGKV_MERGE_FILES_NUM, "3", ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
List<Integer> map1 = ImmutableList.of(2, 3, 2, 1, 5, 2, 6, 9, 6, 2);
List<Integer> map2 = ImmutableList.of(1, 3, 1, 1, 3, 2, 6, 9, 8, 2);
String path = StoreTestUtil.availablePathById("1");
// Merge 4 sorted input
List<RandomAccessInput> inputs = ImmutableList.of(SorterTestUtil.inputFromKvMap(map1), SorterTestUtil.inputFromKvMap(map2), SorterTestUtil.inputFromKvMap(map1), SorterTestUtil.inputFromKvMap(map2));
Sorter sorter = SorterTestUtil.createSorter(config);
PointerCombiner combiner = SorterTestUtil.createPointerCombiner(IntValue::new, new IntValueSumCombiner());
sorter.mergeBuffers(inputs, new CombineKvOuterSortFlusher(combiner), path, false);
// Assert merge result from target hgkvDir
KvEntryFileReader reader = new HgkvDirReaderImpl(path, false);
EntryIterator iter = reader.iterator();
SorterTestUtil.assertKvEntry(iter.next(), 1, 8);
SorterTestUtil.assertKvEntry(iter.next(), 2, 8);
SorterTestUtil.assertKvEntry(iter.next(), 3, 4);
SorterTestUtil.assertKvEntry(iter.next(), 5, 4);
SorterTestUtil.assertKvEntry(iter.next(), 6, 40);
SorterTestUtil.assertKvEntry(iter.next(), 8, 4);
Assert.assertFalse(iter.hasNext());
}
use of com.baidu.hugegraph.computer.core.store.EntryIterator in project hugegraph-computer by hugegraph.
the class SorterTest method testSortSubKvBuffer.
@Test
public void testSortSubKvBuffer() throws Exception {
Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.INPUT_MAX_EDGES_IN_ONE_VERTEX, "2");
/*
* Assert result
* key 1 subKv 3 1, 5 1
* key 2 subKv 5 1, 8 2
* key 2 subKv 9 1
* key 3 subKv 2 2, 3 1
* key 3 subKv 4 1
*/
BytesInput input = this.sortedSubKvBuffer(config);
EntryIterator iter = new KvEntriesInput(input, true);
SorterTestUtil.assertSubKvByKv(iter.next(), 1, 3, 1, 5, 1);
SorterTestUtil.assertSubKvByKv(iter.next(), 2, 5, 1, 8, 2);
SorterTestUtil.assertSubKvByKv(iter.next(), 2, 9, 1);
SorterTestUtil.assertSubKvByKv(iter.next(), 3, 2, 2, 3, 1);
SorterTestUtil.assertSubKvByKv(iter.next(), 3, 4, 1);
iter.close();
}
use of com.baidu.hugegraph.computer.core.store.EntryIterator in project hugegraph-computer by hugegraph.
the class FlusherTest method testKvInnerSortFlusher.
@Test
public void testKvInnerSortFlusher() throws Exception {
List<Integer> map = ImmutableList.of(2, 1, 3, 1, 2, 1, 4, 1);
BytesInput input = SorterTestUtil.inputFromKvMap(map);
BytesOutput output = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
Sorter sorter = SorterTestUtil.createSorter(CONFIG);
sorter.sortBuffer(input, new KvInnerSortFlusher(output), false);
BytesInput result = EntriesUtil.inputFromOutput(output);
EntryIterator iter = new KvEntriesInput(result);
SorterTestUtil.assertKvEntry(iter.next(), 2, 1);
SorterTestUtil.assertKvEntry(iter.next(), 2, 1);
SorterTestUtil.assertKvEntry(iter.next(), 3, 1);
SorterTestUtil.assertKvEntry(iter.next(), 4, 1);
iter.close();
}
Aggregations