use of com.baidu.hugegraph.computer.core.store.entry.DefaultKvEntry in project hugegraph-computer by hugegraph.
the class CombinableSorterFlusher method flush.
public void flush(Iterator<KvEntry> entries) throws IOException {
E.checkArgument(entries.hasNext(), "Parameter entries can't be empty");
KvEntry last = entries.next();
Pointer combineValue = last.value();
while (true) {
KvEntry current = null;
if (entries.hasNext()) {
current = entries.next();
if (last.compareTo(current) == 0) {
combineValue = this.combiner.combine(combineValue, current.value());
continue;
}
}
this.writeKvEntry(new DefaultKvEntry(last.key(), combineValue));
if (current == null) {
break;
}
last = current;
combineValue = last.value();
}
}
use of com.baidu.hugegraph.computer.core.store.entry.DefaultKvEntry in project hugegraph-computer by hugegraph.
the class SortLargeDataTest method testDiffNumEntriesFileMerge.
@Test
public void testDiffNumEntriesFileMerge() throws Exception {
Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.HGKV_MERGE_FILES_NUM, "3", ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
List<Integer> sizeList = ImmutableList.of(200, 500, 20, 50, 300, 250, 10, 33, 900, 89, 20);
List<String> inputs = new ArrayList<>();
for (int j = 0; j < sizeList.size(); j++) {
String file = StoreTestUtil.availablePathById(j + 10);
inputs.add(file);
try (KvEntryFileWriter builder = new HgkvDirBuilderImpl(config, file)) {
for (int i = 0; i < sizeList.get(j); i++) {
byte[] keyBytes = StoreTestUtil.intToByteArray(i);
byte[] valueBytes = StoreTestUtil.intToByteArray(1);
Pointer key = new InlinePointer(keyBytes);
Pointer value = new InlinePointer(valueBytes);
KvEntry entry = new DefaultKvEntry(key, value);
builder.write(entry);
}
}
}
List<String> outputs = ImmutableList.of(StoreTestUtil.availablePathById(0), StoreTestUtil.availablePathById(1), StoreTestUtil.availablePathById(2), StoreTestUtil.availablePathById(3));
Sorter sorter = SorterTestUtil.createSorter(config);
sorter.mergeInputs(inputs, new KvOuterSortFlusher(), outputs, false);
int total = sizeList.stream().mapToInt(i -> i).sum();
int mergeTotal = 0;
for (String output : outputs) {
mergeTotal += HgkvDirImpl.open(output).numEntries();
}
Assert.assertEquals(total, mergeTotal);
}
Aggregations