use of com.baidu.hugegraph.computer.core.sort.flusher.KvOuterSortFlusher in project hugegraph-computer by hugegraph.
the class SortLargeDataTest method testMergeBuffers.
@Test
public void testMergeBuffers() throws Exception {
StopWatch watcher = new StopWatch();
// Sort buffers total size 100M, each buffer is 50KB
final long bufferSize = Bytes.KB * 50;
final long bufferNum = 2000;
final int keyRange = 10000000;
long totalValue = 0L;
Random random = new Random();
List<RandomAccessInput> buffers = new ArrayList<>();
for (int i = 0; i < bufferNum; i++) {
BytesOutput buffer = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
while (buffer.position() < bufferSize) {
// Write data
int key = random.nextInt(keyRange);
SorterTestUtil.writeData(buffer, key);
int value = random.nextInt(100);
SorterTestUtil.writeData(buffer, value);
totalValue += value;
}
buffers.add(EntriesUtil.inputFromOutput(buffer));
}
// Sort buffer
Sorter sorter = SorterTestUtil.createSorter(CONFIG);
watcher.start();
List<RandomAccessInput> sortedBuffers = new ArrayList<>();
for (RandomAccessInput buffer : buffers) {
RandomAccessInput sortedBuffer = sortBuffer(sorter, buffer);
sortedBuffers.add(sortedBuffer);
}
watcher.stop();
LOG.info("testMergeBuffers sort buffer cost time: {}", watcher.getTime());
String resultFile = StoreTestUtil.availablePathById("0");
// Sort buffers
watcher.reset();
watcher.start();
sorter.mergeBuffers(sortedBuffers, new KvOuterSortFlusher(), resultFile, false);
watcher.stop();
LOG.info("testMergeBuffers merge buffers cost time: {}", watcher.getTime());
// Assert result
long result = sumOfEntryValue(sorter, ImmutableList.of(resultFile));
Assert.assertEquals(totalValue, result);
assertFileOrder(sorter, ImmutableList.of(resultFile));
}
use of com.baidu.hugegraph.computer.core.sort.flusher.KvOuterSortFlusher in project hugegraph-computer by hugegraph.
the class FlusherTest method testKvOuterSortFlusher.
@Test
public void testKvOuterSortFlusher() throws Exception {
List<Integer> map1 = ImmutableList.of(2, 1, 2, 1, 3, 1, 4, 1);
List<Integer> map2 = ImmutableList.of(1, 1, 3, 1, 6, 1);
BytesInput input1 = SorterTestUtil.inputFromKvMap(map1);
BytesInput input2 = SorterTestUtil.inputFromKvMap(map2);
List<RandomAccessInput> inputs = ImmutableList.of(input1, input2);
String resultFile = StoreTestUtil.availablePathById("1");
Sorter sorter = SorterTestUtil.createSorter(CONFIG);
sorter.mergeBuffers(inputs, new KvOuterSortFlusher(), resultFile, false);
ImmutableList<String> outputs = ImmutableList.of(resultFile);
Iterator<KvEntry> iter = sorter.iterator(outputs, false);
SorterTestUtil.assertKvEntry(iter.next(), 1, 1);
SorterTestUtil.assertKvEntry(iter.next(), 2, 1);
SorterTestUtil.assertKvEntry(iter.next(), 2, 1);
SorterTestUtil.assertKvEntry(iter.next(), 3, 1);
SorterTestUtil.assertKvEntry(iter.next(), 3, 1);
SorterTestUtil.assertKvEntry(iter.next(), 4, 1);
SorterTestUtil.assertKvEntry(iter.next(), 6, 1);
}
use of com.baidu.hugegraph.computer.core.sort.flusher.KvOuterSortFlusher in project hugegraph-computer by hugegraph.
the class SortLargeDataTest method testDiffNumEntriesFileMerge.
@Test
public void testDiffNumEntriesFileMerge() throws Exception {
Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.HGKV_MERGE_FILES_NUM, "3", ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
List<Integer> sizeList = ImmutableList.of(200, 500, 20, 50, 300, 250, 10, 33, 900, 89, 20);
List<String> inputs = new ArrayList<>();
for (int j = 0; j < sizeList.size(); j++) {
String file = StoreTestUtil.availablePathById(j + 10);
inputs.add(file);
try (KvEntryFileWriter builder = new HgkvDirBuilderImpl(config, file)) {
for (int i = 0; i < sizeList.get(j); i++) {
byte[] keyBytes = StoreTestUtil.intToByteArray(i);
byte[] valueBytes = StoreTestUtil.intToByteArray(1);
Pointer key = new InlinePointer(keyBytes);
Pointer value = new InlinePointer(valueBytes);
KvEntry entry = new DefaultKvEntry(key, value);
builder.write(entry);
}
}
}
List<String> outputs = ImmutableList.of(StoreTestUtil.availablePathById(0), StoreTestUtil.availablePathById(1), StoreTestUtil.availablePathById(2), StoreTestUtil.availablePathById(3));
Sorter sorter = SorterTestUtil.createSorter(config);
sorter.mergeInputs(inputs, new KvOuterSortFlusher(), outputs, false);
int total = sizeList.stream().mapToInt(i -> i).sum();
int mergeTotal = 0;
for (String output : outputs) {
mergeTotal += HgkvDirImpl.open(output).numEntries();
}
Assert.assertEquals(total, mergeTotal);
}
Aggregations