use of com.baidu.hugegraph.computer.core.store.entry.KvEntry in project hugegraph-computer by hugegraph.
the class SorterTest method testMergeKvInputs.
private void testMergeKvInputs(Config config) throws Exception {
List<Integer> map1 = ImmutableList.of(2, 3, 2, 1, 5, 2, 6, 9, 6, 2);
List<Integer> map2 = ImmutableList.of(1, 3, 1, 2, 3, 2);
// Input hgkvDirs
String file1Name = StoreTestUtil.availablePathById("1");
String file2Name = StoreTestUtil.availablePathById("2");
String file3Name = StoreTestUtil.availablePathById("3");
String file4Name = StoreTestUtil.availablePathById("4");
String file5Name = StoreTestUtil.availablePathById("5");
String file6Name = StoreTestUtil.availablePathById("6");
String file7Name = StoreTestUtil.availablePathById("7");
String file8Name = StoreTestUtil.availablePathById("8");
String file9Name = StoreTestUtil.availablePathById("9");
String file10Name = StoreTestUtil.availablePathById("10");
List<String> inputs = Lists.newArrayList(file1Name, file2Name, file3Name, file4Name, file5Name, file6Name, file7Name, file8Name, file9Name, file10Name);
// Output hgkvDirs
String output1 = StoreTestUtil.availablePathById("20");
String output2 = StoreTestUtil.availablePathById("21");
List<String> outputs = ImmutableList.of(output1, output2);
for (int i = 0; i < inputs.size(); i++) {
List<Integer> map;
if ((i & 1) == 0) {
map = map1;
} else {
map = map2;
}
if (config.get(ComputerOptions.TRANSPORT_RECV_FILE_MODE)) {
StoreTestUtil.bufferFileFromKvMap(map, inputs.get(i));
} else {
StoreTestUtil.hgkvDirFromKvMap(config, map, inputs.get(i));
}
}
// Merge file
Sorter sorter = SorterTestUtil.createSorter(config);
PointerCombiner combiner = SorterTestUtil.createPointerCombiner(IntValue::new, new IntValueSumCombiner());
sorter.mergeInputs(inputs, new CombineKvOuterSortFlusher(combiner), outputs, false);
// Assert sort result
List<Integer> result = ImmutableList.of(1, 25, 2, 20, 3, 10, 5, 10, 6, 55);
Iterator<Integer> resultIter = result.iterator();
Iterator<KvEntry> iterator = sorter.iterator(outputs, false);
KvEntry last = iterator.next();
int value = StoreTestUtil.dataFromPointer(last.value());
while (true) {
KvEntry current = null;
if (iterator.hasNext()) {
current = iterator.next();
if (last.compareTo(current) == 0) {
value += StoreTestUtil.dataFromPointer(current.value());
continue;
}
}
Assert.assertEquals(StoreTestUtil.dataFromPointer(last.key()), resultIter.next());
Assert.assertEquals(value, resultIter.next());
if (current == null) {
break;
}
last = current;
value = StoreTestUtil.dataFromPointer(last.value());
}
Assert.assertFalse(resultIter.hasNext());
FileUtil.deleteFilesQuietly(inputs);
FileUtil.deleteFilesQuietly(outputs);
}
use of com.baidu.hugegraph.computer.core.store.entry.KvEntry in project hugegraph-computer by hugegraph.
the class SorterTest method testSortSubKvBuffers.
@Test
public void testSortSubKvBuffers() throws Exception {
Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.INPUT_MAX_EDGES_IN_ONE_VERTEX, "2", ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
int flushThreshold = config.get(ComputerOptions.INPUT_MAX_EDGES_IN_ONE_VERTEX);
BytesInput i1 = this.sortedSubKvBuffer(config);
BytesInput i2 = this.sortedSubKvBuffer(config);
BytesInput i3 = this.sortedSubKvBuffer(config);
List<RandomAccessInput> buffers = ImmutableList.of(i1, i2, i3);
Sorter sorter = SorterTestUtil.createSorter(config);
PointerCombiner combiner = SorterTestUtil.createPointerCombiner(IntValue::new, new IntValueSumCombiner());
OuterSortFlusher flusher = new CombineSubKvOuterSortFlusher(combiner, flushThreshold);
flusher.sources(buffers.size());
String outputFile = StoreTestUtil.availablePathById("1");
sorter.mergeBuffers(buffers, flusher, outputFile, true);
/*
* Assert result
* key 1 subKv 3 3, 5 3
* key 2 subKv 5 3, 8 6
* key 2 subKv 9 3
* key 3 subKv 2 6, 3 3
* key 3 subKv 4 3
*/
ImmutableList<String> outputs = ImmutableList.of(outputFile);
Iterator<KvEntry> kvIter = sorter.iterator(outputs, true);
SorterTestUtil.assertSubKvByKv(kvIter.next(), 1, 3, 3, 5, 3);
SorterTestUtil.assertSubKvByKv(kvIter.next(), 2, 5, 3, 8, 6);
SorterTestUtil.assertSubKvByKv(kvIter.next(), 2, 9, 3);
SorterTestUtil.assertSubKvByKv(kvIter.next(), 3, 2, 6, 3, 3);
SorterTestUtil.assertSubKvByKv(kvIter.next(), 3, 4, 3);
// Assert file properties
HgkvDir dir = HgkvDirImpl.open(outputFile);
Assert.assertEquals(5, dir.numEntries());
Assert.assertEquals(8, dir.numSubEntries());
}
use of com.baidu.hugegraph.computer.core.store.entry.KvEntry in project hugegraph-computer by hugegraph.
the class SorterTestUtil method assertSubKvByKv.
public static void assertSubKvByKv(KvEntry entry, int... expect) throws IOException {
int index = 0;
Assert.assertEquals(expect[index++], StoreTestUtil.dataFromPointer(entry.key()));
Iterator<KvEntry> subKvs = EntriesUtil.subKvIterFromEntry(entry);
while (subKvs.hasNext()) {
KvEntry subKv = subKvs.next();
Assert.assertEquals(expect[index++], StoreTestUtil.dataFromPointer(subKv.key()));
Assert.assertEquals(expect[index++], StoreTestUtil.dataFromPointer(subKv.value()));
}
}
use of com.baidu.hugegraph.computer.core.store.entry.KvEntry in project hugegraph-computer by hugegraph.
the class FlusherTest method testExceptionCaseForFlusher.
@Test
public void testExceptionCaseForFlusher() {
BytesOutput output = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
InnerSortFlusher flusher = new KvInnerSortFlusher(output);
List<KvEntry> entries = new ArrayList<>();
Assert.assertThrows(IllegalArgumentException.class, () -> {
flusher.flush(entries.iterator());
}, e -> {
String errorMsg = "Parameter entries can't be empty";
Assert.assertContains(errorMsg, e.getMessage());
});
}
use of com.baidu.hugegraph.computer.core.store.entry.KvEntry in project hugegraph-computer by hugegraph.
the class EdgeMessageRecvPartitionTest method checkTenEdgesWithCombinedProperties.
private static void checkTenEdgesWithCombinedProperties(Iterator<KvEntry> it) throws IOException {
for (long i = 0L; i < 10L; i++) {
Assert.assertTrue(it.hasNext());
KvEntry entry = it.next();
Id id = ReceiverUtil.readId(entry.key());
Assert.assertEquals(BytesId.of(i), id);
EntryIterator subKvIt = EntriesUtil.subKvIterFromEntry(entry);
for (long j = i + 1; j < i + 3; j++) {
Assert.assertTrue(subKvIt.hasNext());
KvEntry subKv = subKvIt.next();
Id targetId = ReceiverUtil.readId(subKv.key());
Assert.assertEquals(BytesId.of(j), targetId);
Properties properties = graphFactory().createProperties();
ReceiverUtil.readValue(subKv.value(), properties);
Assert.assertEquals(2, properties.size());
LongValue v1 = properties.get("p1");
Assert.assertEquals(new LongValue(i), v1);
LongValue v2 = properties.get("p2");
Assert.assertEquals(new LongValue(2L * i), v2);
}
}
Assert.assertFalse(it.hasNext());
}
Aggregations