use of com.baidu.hugegraph.computer.core.io.BytesInput in project hugegraph-computer by hugegraph.
the class EntriesUtil method kvEntryWithFirstSubKv.
public static KvEntryWithFirstSubKv kvEntryWithFirstSubKv(KvEntry entry) {
try {
BytesInput input = IOFactory.createBytesInput(entry.value().bytes());
// Read sub-entry size
long subKvNum = input.readFixedInt();
KvEntry firstSubKv = EntriesUtil.subKvEntryFromInput(input, true);
return new KvEntryWithFirstSubKv(entry.key(), entry.value(), firstSubKv, subKvNum);
} catch (IOException e) {
throw new ComputerException(e.getMessage(), e);
}
}
use of com.baidu.hugegraph.computer.core.io.BytesInput in project hugegraph-computer by hugegraph.
the class SortLargeDataTest method testAllProcess.
@Test
public void testAllProcess() throws Exception {
StopWatch watcher = new StopWatch();
final long bufferSize = Bytes.MB;
final int mergeBufferNum = 300;
final int dataSize = 1000000;
long value = 0;
Random random = new Random();
BytesOutput output = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
List<RandomAccessInput> buffers = new ArrayList<>(mergeBufferNum);
List<String> mergeBufferFiles = new ArrayList<>();
int fileNum = 10;
Sorter sorter = SorterTestUtil.createSorter(CONFIG);
watcher.start();
for (int i = 0; i < dataSize; i++) {
SorterTestUtil.writeData(output, random.nextInt(dataSize));
int entryValue = random.nextInt(5);
SorterTestUtil.writeData(output, entryValue);
value = value + entryValue;
// Write data to buffer and sort buffer
if (output.position() >= bufferSize || (i + 1) == dataSize) {
BytesInput input = EntriesUtil.inputFromOutput(output);
buffers.add(sortBuffer(sorter, input));
output.seek(0);
}
// Merge buffers to HgkvDir
if (buffers.size() >= mergeBufferNum || (i + 1) == dataSize) {
String outputFile = StoreTestUtil.availablePathById(fileNum++);
mergeBufferFiles.add(outputFile);
mergeBuffers(sorter, buffers, outputFile);
buffers.clear();
}
}
// Merge file
String resultFile = StoreTestUtil.availablePathById("0");
mergeFiles(sorter, mergeBufferFiles, Lists.newArrayList(resultFile));
watcher.stop();
LOG.info("testAllProcess sort time: {}", watcher.getTime());
long result = sumOfEntryValue(sorter, ImmutableList.of(resultFile));
Assert.assertEquals(value, result);
}
use of com.baidu.hugegraph.computer.core.io.BytesInput in project hugegraph-computer by hugegraph.
the class SorterTest method testSortSubKvBuffer.
@Test
public void testSortSubKvBuffer() throws Exception {
Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.INPUT_MAX_EDGES_IN_ONE_VERTEX, "2");
/*
* Assert result
* key 1 subKv 3 1, 5 1
* key 2 subKv 5 1, 8 2
* key 2 subKv 9 1
* key 3 subKv 2 2, 3 1
* key 3 subKv 4 1
*/
BytesInput input = this.sortedSubKvBuffer(config);
EntryIterator iter = new KvEntriesInput(input, true);
SorterTestUtil.assertSubKvByKv(iter.next(), 1, 3, 1, 5, 1);
SorterTestUtil.assertSubKvByKv(iter.next(), 2, 5, 1, 8, 2);
SorterTestUtil.assertSubKvByKv(iter.next(), 2, 9, 1);
SorterTestUtil.assertSubKvByKv(iter.next(), 3, 2, 2, 3, 1);
SorterTestUtil.assertSubKvByKv(iter.next(), 3, 4, 1);
iter.close();
}
use of com.baidu.hugegraph.computer.core.io.BytesInput in project hugegraph-computer by hugegraph.
the class SorterTest method testSortKvBuffer.
@Test
public void testSortKvBuffer() throws Exception {
Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.HGKV_MAX_FILE_SIZE, "32", ComputerOptions.HGKV_DATABLOCK_SIZE, "16", ComputerOptions.HGKV_MERGE_FILES_NUM, "3");
List<Integer> map = ImmutableList.of(2, 3, 1, 23, 6, 2, 5, 9, 2, 2, 6, 1, 1, 20);
BytesInput input = SorterTestUtil.inputFromKvMap(map);
BytesOutput output = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
Sorter sorter = SorterTestUtil.createSorter(config);
PointerCombiner combiner = SorterTestUtil.createPointerCombiner(IntValue::new, new IntValueSumCombiner());
sorter.sortBuffer(input, new CombineKvInnerSortFlusher(output, combiner), false);
BytesInput resultInput = EntriesUtil.inputFromOutput(output);
KvEntriesInput iter = new KvEntriesInput(resultInput);
SorterTestUtil.assertKvEntry(iter.next(), 1, 43);
SorterTestUtil.assertKvEntry(iter.next(), 2, 5);
SorterTestUtil.assertKvEntry(iter.next(), 5, 9);
SorterTestUtil.assertKvEntry(iter.next(), 6, 3);
iter.close();
}
use of com.baidu.hugegraph.computer.core.io.BytesInput in project hugegraph-computer by hugegraph.
the class SorterTest method testSortSubKvBuffers.
@Test
public void testSortSubKvBuffers() throws Exception {
Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.INPUT_MAX_EDGES_IN_ONE_VERTEX, "2", ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
int flushThreshold = config.get(ComputerOptions.INPUT_MAX_EDGES_IN_ONE_VERTEX);
BytesInput i1 = this.sortedSubKvBuffer(config);
BytesInput i2 = this.sortedSubKvBuffer(config);
BytesInput i3 = this.sortedSubKvBuffer(config);
List<RandomAccessInput> buffers = ImmutableList.of(i1, i2, i3);
Sorter sorter = SorterTestUtil.createSorter(config);
PointerCombiner combiner = SorterTestUtil.createPointerCombiner(IntValue::new, new IntValueSumCombiner());
OuterSortFlusher flusher = new CombineSubKvOuterSortFlusher(combiner, flushThreshold);
flusher.sources(buffers.size());
String outputFile = StoreTestUtil.availablePathById("1");
sorter.mergeBuffers(buffers, flusher, outputFile, true);
/*
* Assert result
* key 1 subKv 3 3, 5 3
* key 2 subKv 5 3, 8 6
* key 2 subKv 9 3
* key 3 subKv 2 6, 3 3
* key 3 subKv 4 3
*/
ImmutableList<String> outputs = ImmutableList.of(outputFile);
Iterator<KvEntry> kvIter = sorter.iterator(outputs, true);
SorterTestUtil.assertSubKvByKv(kvIter.next(), 1, 3, 3, 5, 3);
SorterTestUtil.assertSubKvByKv(kvIter.next(), 2, 5, 3, 8, 6);
SorterTestUtil.assertSubKvByKv(kvIter.next(), 2, 9, 3);
SorterTestUtil.assertSubKvByKv(kvIter.next(), 3, 2, 6, 3, 3);
SorterTestUtil.assertSubKvByKv(kvIter.next(), 3, 4, 3);
// Assert file properties
HgkvDir dir = HgkvDirImpl.open(outputFile);
Assert.assertEquals(5, dir.numEntries());
Assert.assertEquals(8, dir.numSubEntries());
}
Aggregations