use of com.baidu.hugegraph.computer.core.store.buffer.KvEntriesInput in project hugegraph-computer by hugegraph.
the class StoreTestUtil method kvEntriesFromMap.
public static List<KvEntry> kvEntriesFromMap(List<Integer> map) throws IOException {
BytesOutput data = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
Iterator<Integer> iterator = map.iterator();
while (iterator.hasNext()) {
// Write key length
writeData(data, iterator.next());
// Write value length
writeData(data, iterator.next());
}
BytesInput input = IOFactory.createBytesInput(data.buffer(), (int) data.position());
KvEntriesInput iter = new KvEntriesInput(input);
List<KvEntry> entries = new ArrayList<>();
while (iter.hasNext()) {
entries.add(iter.next());
}
iter.close();
return entries;
}
use of com.baidu.hugegraph.computer.core.store.buffer.KvEntriesInput in project hugegraph-computer by hugegraph.
the class SorterTest method testSortSubKvBuffer.
@Test
public void testSortSubKvBuffer() throws Exception {
Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.INPUT_MAX_EDGES_IN_ONE_VERTEX, "2");
/*
* Assert result
* key 1 subKv 3 1, 5 1
* key 2 subKv 5 1, 8 2
* key 2 subKv 9 1
* key 3 subKv 2 2, 3 1
* key 3 subKv 4 1
*/
BytesInput input = this.sortedSubKvBuffer(config);
EntryIterator iter = new KvEntriesInput(input, true);
SorterTestUtil.assertSubKvByKv(iter.next(), 1, 3, 1, 5, 1);
SorterTestUtil.assertSubKvByKv(iter.next(), 2, 5, 1, 8, 2);
SorterTestUtil.assertSubKvByKv(iter.next(), 2, 9, 1);
SorterTestUtil.assertSubKvByKv(iter.next(), 3, 2, 2, 3, 1);
SorterTestUtil.assertSubKvByKv(iter.next(), 3, 4, 1);
iter.close();
}
use of com.baidu.hugegraph.computer.core.store.buffer.KvEntriesInput in project hugegraph-computer by hugegraph.
the class SorterTest method testSortKvBuffer.
@Test
public void testSortKvBuffer() throws Exception {
Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.HGKV_MAX_FILE_SIZE, "32", ComputerOptions.HGKV_DATABLOCK_SIZE, "16", ComputerOptions.HGKV_MERGE_FILES_NUM, "3");
List<Integer> map = ImmutableList.of(2, 3, 1, 23, 6, 2, 5, 9, 2, 2, 6, 1, 1, 20);
BytesInput input = SorterTestUtil.inputFromKvMap(map);
BytesOutput output = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
Sorter sorter = SorterTestUtil.createSorter(config);
PointerCombiner combiner = SorterTestUtil.createPointerCombiner(IntValue::new, new IntValueSumCombiner());
sorter.sortBuffer(input, new CombineKvInnerSortFlusher(output, combiner), false);
BytesInput resultInput = EntriesUtil.inputFromOutput(output);
KvEntriesInput iter = new KvEntriesInput(resultInput);
SorterTestUtil.assertKvEntry(iter.next(), 1, 43);
SorterTestUtil.assertKvEntry(iter.next(), 2, 5);
SorterTestUtil.assertKvEntry(iter.next(), 5, 9);
SorterTestUtil.assertKvEntry(iter.next(), 6, 3);
iter.close();
}
use of com.baidu.hugegraph.computer.core.store.buffer.KvEntriesInput in project hugegraph-computer by hugegraph.
the class FlusherTest method testKvInnerSortFlusher.
@Test
public void testKvInnerSortFlusher() throws Exception {
List<Integer> map = ImmutableList.of(2, 1, 3, 1, 2, 1, 4, 1);
BytesInput input = SorterTestUtil.inputFromKvMap(map);
BytesOutput output = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
Sorter sorter = SorterTestUtil.createSorter(CONFIG);
sorter.sortBuffer(input, new KvInnerSortFlusher(output), false);
BytesInput result = EntriesUtil.inputFromOutput(output);
EntryIterator iter = new KvEntriesInput(result);
SorterTestUtil.assertKvEntry(iter.next(), 2, 1);
SorterTestUtil.assertKvEntry(iter.next(), 2, 1);
SorterTestUtil.assertKvEntry(iter.next(), 3, 1);
SorterTestUtil.assertKvEntry(iter.next(), 4, 1);
iter.close();
}
use of com.baidu.hugegraph.computer.core.store.buffer.KvEntriesInput in project hugegraph-computer by hugegraph.
the class FlusherTest method testOverwriteCombiner.
@Test
public void testOverwriteCombiner() throws Exception {
List<Integer> data = ImmutableList.of(1, 2, 3, 5, 1, 3, 1, 1, 3, 4);
BytesInput input = SorterTestUtil.inputFromKvMap(data);
BytesOutput output = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
PointerCombiner combiner = SorterTestUtil.createPointerCombiner(IntValue::new, new OverwriteCombiner<>());
InnerSortFlusher flusher = new CombineKvInnerSortFlusher(output, combiner);
Sorter sorter = SorterTestUtil.createSorter(CONFIG);
sorter.sortBuffer(input, flusher, false);
BytesInput result = EntriesUtil.inputFromOutput(output);
// Assert result
KvEntriesInput iter = new KvEntriesInput(result);
SorterTestUtil.assertKvEntry(iter.next(), 1, 1);
SorterTestUtil.assertKvEntry(iter.next(), 3, 4);
iter.close();
}
Aggregations