use of com.baidu.hugegraph.computer.core.combiner.PointerCombiner in project hugegraph-computer by hugegraph.
the class SortManager method createSortFlusher.
private InnerSortFlusher createSortFlusher(MessageType type, RandomAccessOutput output, int flushThreshold) {
PointerCombiner combiner;
boolean needSortSubKv;
switch(type) {
case VERTEX:
combiner = new VertexValueCombiner(this.context);
needSortSubKv = false;
break;
case EDGE:
combiner = new EdgeValueCombiner(this.context);
needSortSubKv = true;
break;
case MSG:
combiner = this.createMessageCombiner();
needSortSubKv = false;
break;
default:
throw new ComputerException("Unsupported combine message " + "type for %s", type);
}
InnerSortFlusher flusher;
if (combiner == null) {
flusher = new KvInnerSortFlusher(output);
} else {
if (needSortSubKv) {
flusher = new CombineSubKvInnerSortFlusher(output, combiner, flushThreshold);
} else {
flusher = new CombineKvInnerSortFlusher(output, combiner);
}
}
return flusher;
}
use of com.baidu.hugegraph.computer.core.combiner.PointerCombiner in project hugegraph-computer by hugegraph.
the class SortLargeDataTest method mergeFiles.
private static void mergeFiles(Sorter sorter, List<String> files, List<String> outputs) throws Exception {
PointerCombiner combiner = SorterTestUtil.createPointerCombiner(IntValue::new, new IntValueSumCombiner());
OuterSortFlusher flusher = new CombineKvOuterSortFlusher(combiner);
sorter.mergeInputs(files, flusher, outputs, false);
}
use of com.baidu.hugegraph.computer.core.combiner.PointerCombiner in project hugegraph-computer by hugegraph.
the class SortLargeDataTest method sortBuffer.
private static RandomAccessInput sortBuffer(Sorter sorter, RandomAccessInput input) throws Exception {
BytesOutput output = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
PointerCombiner combiner = SorterTestUtil.createPointerCombiner(IntValue::new, new IntValueSumCombiner());
InnerSortFlusher flusher = new CombineKvInnerSortFlusher(output, combiner);
sorter.sortBuffer(input, flusher, false);
return EntriesUtil.inputFromOutput(output);
}
use of com.baidu.hugegraph.computer.core.combiner.PointerCombiner in project hugegraph-computer by hugegraph.
the class SorterTest method testSortKvBuffers.
@Test
public void testSortKvBuffers() throws Exception {
Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.HGKV_MAX_FILE_SIZE, "32", ComputerOptions.HGKV_DATABLOCK_SIZE, "16", ComputerOptions.HGKV_MERGE_FILES_NUM, "3", ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
List<Integer> map1 = ImmutableList.of(2, 3, 2, 1, 5, 2, 6, 9, 6, 2);
List<Integer> map2 = ImmutableList.of(1, 3, 1, 1, 3, 2, 6, 9, 8, 2);
String path = StoreTestUtil.availablePathById("1");
// Merge 4 sorted input
List<RandomAccessInput> inputs = ImmutableList.of(SorterTestUtil.inputFromKvMap(map1), SorterTestUtil.inputFromKvMap(map2), SorterTestUtil.inputFromKvMap(map1), SorterTestUtil.inputFromKvMap(map2));
Sorter sorter = SorterTestUtil.createSorter(config);
PointerCombiner combiner = SorterTestUtil.createPointerCombiner(IntValue::new, new IntValueSumCombiner());
sorter.mergeBuffers(inputs, new CombineKvOuterSortFlusher(combiner), path, false);
// Assert merge result from target hgkvDir
KvEntryFileReader reader = new HgkvDirReaderImpl(path, false);
EntryIterator iter = reader.iterator();
SorterTestUtil.assertKvEntry(iter.next(), 1, 8);
SorterTestUtil.assertKvEntry(iter.next(), 2, 8);
SorterTestUtil.assertKvEntry(iter.next(), 3, 4);
SorterTestUtil.assertKvEntry(iter.next(), 5, 4);
SorterTestUtil.assertKvEntry(iter.next(), 6, 40);
SorterTestUtil.assertKvEntry(iter.next(), 8, 4);
Assert.assertFalse(iter.hasNext());
}
use of com.baidu.hugegraph.computer.core.combiner.PointerCombiner in project hugegraph-computer by hugegraph.
the class SorterTest method testMergeKvInputs.
private void testMergeKvInputs(Config config) throws Exception {
List<Integer> map1 = ImmutableList.of(2, 3, 2, 1, 5, 2, 6, 9, 6, 2);
List<Integer> map2 = ImmutableList.of(1, 3, 1, 2, 3, 2);
// Input hgkvDirs
String file1Name = StoreTestUtil.availablePathById("1");
String file2Name = StoreTestUtil.availablePathById("2");
String file3Name = StoreTestUtil.availablePathById("3");
String file4Name = StoreTestUtil.availablePathById("4");
String file5Name = StoreTestUtil.availablePathById("5");
String file6Name = StoreTestUtil.availablePathById("6");
String file7Name = StoreTestUtil.availablePathById("7");
String file8Name = StoreTestUtil.availablePathById("8");
String file9Name = StoreTestUtil.availablePathById("9");
String file10Name = StoreTestUtil.availablePathById("10");
List<String> inputs = Lists.newArrayList(file1Name, file2Name, file3Name, file4Name, file5Name, file6Name, file7Name, file8Name, file9Name, file10Name);
// Output hgkvDirs
String output1 = StoreTestUtil.availablePathById("20");
String output2 = StoreTestUtil.availablePathById("21");
List<String> outputs = ImmutableList.of(output1, output2);
for (int i = 0; i < inputs.size(); i++) {
List<Integer> map;
if ((i & 1) == 0) {
map = map1;
} else {
map = map2;
}
if (config.get(ComputerOptions.TRANSPORT_RECV_FILE_MODE)) {
StoreTestUtil.bufferFileFromKvMap(map, inputs.get(i));
} else {
StoreTestUtil.hgkvDirFromKvMap(config, map, inputs.get(i));
}
}
// Merge file
Sorter sorter = SorterTestUtil.createSorter(config);
PointerCombiner combiner = SorterTestUtil.createPointerCombiner(IntValue::new, new IntValueSumCombiner());
sorter.mergeInputs(inputs, new CombineKvOuterSortFlusher(combiner), outputs, false);
// Assert sort result
List<Integer> result = ImmutableList.of(1, 25, 2, 20, 3, 10, 5, 10, 6, 55);
Iterator<Integer> resultIter = result.iterator();
Iterator<KvEntry> iterator = sorter.iterator(outputs, false);
KvEntry last = iterator.next();
int value = StoreTestUtil.dataFromPointer(last.value());
while (true) {
KvEntry current = null;
if (iterator.hasNext()) {
current = iterator.next();
if (last.compareTo(current) == 0) {
value += StoreTestUtil.dataFromPointer(current.value());
continue;
}
}
Assert.assertEquals(StoreTestUtil.dataFromPointer(last.key()), resultIter.next());
Assert.assertEquals(value, resultIter.next());
if (current == null) {
break;
}
last = current;
value = StoreTestUtil.dataFromPointer(last.value());
}
Assert.assertFalse(resultIter.hasNext());
FileUtil.deleteFilesQuietly(inputs);
FileUtil.deleteFilesQuietly(outputs);
}
Aggregations