Search in sources :

Example 1 with KvOuterSortFlusher

use of com.baidu.hugegraph.computer.core.sort.flusher.KvOuterSortFlusher in project hugegraph-computer by hugegraph.

the class SortLargeDataTest method testMergeBuffers.

@Test
public void testMergeBuffers() throws Exception {
    StopWatch watcher = new StopWatch();
    // Sort buffers total size 100M, each buffer is 50KB
    final long bufferSize = Bytes.KB * 50;
    final long bufferNum = 2000;
    final int keyRange = 10000000;
    long totalValue = 0L;
    Random random = new Random();
    List<RandomAccessInput> buffers = new ArrayList<>();
    for (int i = 0; i < bufferNum; i++) {
        BytesOutput buffer = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
        while (buffer.position() < bufferSize) {
            // Write data
            int key = random.nextInt(keyRange);
            SorterTestUtil.writeData(buffer, key);
            int value = random.nextInt(100);
            SorterTestUtil.writeData(buffer, value);
            totalValue += value;
        }
        buffers.add(EntriesUtil.inputFromOutput(buffer));
    }
    // Sort buffer
    Sorter sorter = SorterTestUtil.createSorter(CONFIG);
    watcher.start();
    List<RandomAccessInput> sortedBuffers = new ArrayList<>();
    for (RandomAccessInput buffer : buffers) {
        RandomAccessInput sortedBuffer = sortBuffer(sorter, buffer);
        sortedBuffers.add(sortedBuffer);
    }
    watcher.stop();
    LOG.info("testMergeBuffers sort buffer cost time: {}", watcher.getTime());
    String resultFile = StoreTestUtil.availablePathById("0");
    // Sort buffers
    watcher.reset();
    watcher.start();
    sorter.mergeBuffers(sortedBuffers, new KvOuterSortFlusher(), resultFile, false);
    watcher.stop();
    LOG.info("testMergeBuffers merge buffers cost time: {}", watcher.getTime());
    // Assert result
    long result = sumOfEntryValue(sorter, ImmutableList.of(resultFile));
    Assert.assertEquals(totalValue, result);
    assertFileOrder(sorter, ImmutableList.of(resultFile));
}
Also used : RandomAccessInput(com.baidu.hugegraph.computer.core.io.RandomAccessInput) Random(java.util.Random) BytesOutput(com.baidu.hugegraph.computer.core.io.BytesOutput) KvOuterSortFlusher(com.baidu.hugegraph.computer.core.sort.flusher.KvOuterSortFlusher) CombineKvOuterSortFlusher(com.baidu.hugegraph.computer.core.sort.flusher.CombineKvOuterSortFlusher) ArrayList(java.util.ArrayList) Sorter(com.baidu.hugegraph.computer.core.sort.Sorter) StopWatch(org.apache.commons.lang3.time.StopWatch) Test(org.junit.Test)

Example 2 with KvOuterSortFlusher

use of com.baidu.hugegraph.computer.core.sort.flusher.KvOuterSortFlusher in project hugegraph-computer by hugegraph.

the class FlusherTest method testKvOuterSortFlusher.

@Test
public void testKvOuterSortFlusher() throws Exception {
    List<Integer> map1 = ImmutableList.of(2, 1, 2, 1, 3, 1, 4, 1);
    List<Integer> map2 = ImmutableList.of(1, 1, 3, 1, 6, 1);
    BytesInput input1 = SorterTestUtil.inputFromKvMap(map1);
    BytesInput input2 = SorterTestUtil.inputFromKvMap(map2);
    List<RandomAccessInput> inputs = ImmutableList.of(input1, input2);
    String resultFile = StoreTestUtil.availablePathById("1");
    Sorter sorter = SorterTestUtil.createSorter(CONFIG);
    sorter.mergeBuffers(inputs, new KvOuterSortFlusher(), resultFile, false);
    ImmutableList<String> outputs = ImmutableList.of(resultFile);
    Iterator<KvEntry> iter = sorter.iterator(outputs, false);
    SorterTestUtil.assertKvEntry(iter.next(), 1, 1);
    SorterTestUtil.assertKvEntry(iter.next(), 2, 1);
    SorterTestUtil.assertKvEntry(iter.next(), 2, 1);
    SorterTestUtil.assertKvEntry(iter.next(), 3, 1);
    SorterTestUtil.assertKvEntry(iter.next(), 3, 1);
    SorterTestUtil.assertKvEntry(iter.next(), 4, 1);
    SorterTestUtil.assertKvEntry(iter.next(), 6, 1);
}
Also used : RandomAccessInput(com.baidu.hugegraph.computer.core.io.RandomAccessInput) BytesInput(com.baidu.hugegraph.computer.core.io.BytesInput) KvOuterSortFlusher(com.baidu.hugegraph.computer.core.sort.flusher.KvOuterSortFlusher) Sorter(com.baidu.hugegraph.computer.core.sort.Sorter) KvEntry(com.baidu.hugegraph.computer.core.store.entry.KvEntry) Test(org.junit.Test)

Example 3 with KvOuterSortFlusher

use of com.baidu.hugegraph.computer.core.sort.flusher.KvOuterSortFlusher in project hugegraph-computer by hugegraph.

the class SortLargeDataTest method testDiffNumEntriesFileMerge.

@Test
public void testDiffNumEntriesFileMerge() throws Exception {
    Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.HGKV_MERGE_FILES_NUM, "3", ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
    List<Integer> sizeList = ImmutableList.of(200, 500, 20, 50, 300, 250, 10, 33, 900, 89, 20);
    List<String> inputs = new ArrayList<>();
    for (int j = 0; j < sizeList.size(); j++) {
        String file = StoreTestUtil.availablePathById(j + 10);
        inputs.add(file);
        try (KvEntryFileWriter builder = new HgkvDirBuilderImpl(config, file)) {
            for (int i = 0; i < sizeList.get(j); i++) {
                byte[] keyBytes = StoreTestUtil.intToByteArray(i);
                byte[] valueBytes = StoreTestUtil.intToByteArray(1);
                Pointer key = new InlinePointer(keyBytes);
                Pointer value = new InlinePointer(valueBytes);
                KvEntry entry = new DefaultKvEntry(key, value);
                builder.write(entry);
            }
        }
    }
    List<String> outputs = ImmutableList.of(StoreTestUtil.availablePathById(0), StoreTestUtil.availablePathById(1), StoreTestUtil.availablePathById(2), StoreTestUtil.availablePathById(3));
    Sorter sorter = SorterTestUtil.createSorter(config);
    sorter.mergeInputs(inputs, new KvOuterSortFlusher(), outputs, false);
    int total = sizeList.stream().mapToInt(i -> i).sum();
    int mergeTotal = 0;
    for (String output : outputs) {
        mergeTotal += HgkvDirImpl.open(output).numEntries();
    }
    Assert.assertEquals(total, mergeTotal);
}
Also used : ComputerOptions(com.baidu.hugegraph.computer.core.config.ComputerOptions) BeforeClass(org.junit.BeforeClass) Random(java.util.Random) EntriesUtil(com.baidu.hugegraph.computer.core.store.entry.EntriesUtil) Pointer(com.baidu.hugegraph.computer.core.store.entry.Pointer) ArrayList(java.util.ArrayList) IntValueSumCombiner(com.baidu.hugegraph.computer.core.combiner.IntValueSumCombiner) IOFactory(com.baidu.hugegraph.computer.core.io.IOFactory) Lists(com.google.common.collect.Lists) ImmutableList(com.google.common.collect.ImmutableList) After(org.junit.After) StoreTestUtil(com.baidu.hugegraph.computer.core.store.StoreTestUtil) UnitTestBase(com.baidu.hugegraph.computer.suite.unit.UnitTestBase) Before(org.junit.Before) Logger(org.slf4j.Logger) OuterSortFlusher(com.baidu.hugegraph.computer.core.sort.flusher.OuterSortFlusher) Constants(com.baidu.hugegraph.computer.core.common.Constants) IOException(java.io.IOException) FileUtils(org.apache.commons.io.FileUtils) Test(org.junit.Test) HgkvDir(com.baidu.hugegraph.computer.core.store.file.hgkvfile.HgkvDir) StopWatch(org.apache.commons.lang3.time.StopWatch) CombineKvInnerSortFlusher(com.baidu.hugegraph.computer.core.sort.flusher.CombineKvInnerSortFlusher) HgkvDirImpl(com.baidu.hugegraph.computer.core.store.file.hgkvfile.HgkvDirImpl) DefaultKvEntry(com.baidu.hugegraph.computer.core.store.entry.DefaultKvEntry) KvEntry(com.baidu.hugegraph.computer.core.store.entry.KvEntry) File(java.io.File) Config(com.baidu.hugegraph.computer.core.config.Config) KvOuterSortFlusher(com.baidu.hugegraph.computer.core.sort.flusher.KvOuterSortFlusher) Bytes(com.baidu.hugegraph.util.Bytes) List(java.util.List) Log(com.baidu.hugegraph.util.Log) CombineKvOuterSortFlusher(com.baidu.hugegraph.computer.core.sort.flusher.CombineKvOuterSortFlusher) IntValue(com.baidu.hugegraph.computer.core.graph.value.IntValue) HgkvDirBuilderImpl(com.baidu.hugegraph.computer.core.store.file.hgkvfile.builder.HgkvDirBuilderImpl) PointerCombiner(com.baidu.hugegraph.computer.core.combiner.PointerCombiner) BytesInput(com.baidu.hugegraph.computer.core.io.BytesInput) BytesOutput(com.baidu.hugegraph.computer.core.io.BytesOutput) SorterTestUtil(com.baidu.hugegraph.computer.core.sort.SorterTestUtil) KvEntryFileWriter(com.baidu.hugegraph.computer.core.store.KvEntryFileWriter) Sorter(com.baidu.hugegraph.computer.core.sort.Sorter) Assert(com.baidu.hugegraph.testutil.Assert) PeekableIterator(com.baidu.hugegraph.computer.core.sort.flusher.PeekableIterator) RandomAccessInput(com.baidu.hugegraph.computer.core.io.RandomAccessInput) InlinePointer(com.baidu.hugegraph.computer.core.store.entry.InlinePointer) InnerSortFlusher(com.baidu.hugegraph.computer.core.sort.flusher.InnerSortFlusher) KvOuterSortFlusher(com.baidu.hugegraph.computer.core.sort.flusher.KvOuterSortFlusher) CombineKvOuterSortFlusher(com.baidu.hugegraph.computer.core.sort.flusher.CombineKvOuterSortFlusher) Config(com.baidu.hugegraph.computer.core.config.Config) InlinePointer(com.baidu.hugegraph.computer.core.store.entry.InlinePointer) ArrayList(java.util.ArrayList) DefaultKvEntry(com.baidu.hugegraph.computer.core.store.entry.DefaultKvEntry) KvEntry(com.baidu.hugegraph.computer.core.store.entry.KvEntry) Pointer(com.baidu.hugegraph.computer.core.store.entry.Pointer) InlinePointer(com.baidu.hugegraph.computer.core.store.entry.InlinePointer) DefaultKvEntry(com.baidu.hugegraph.computer.core.store.entry.DefaultKvEntry) Sorter(com.baidu.hugegraph.computer.core.sort.Sorter) KvEntryFileWriter(com.baidu.hugegraph.computer.core.store.KvEntryFileWriter) HgkvDirBuilderImpl(com.baidu.hugegraph.computer.core.store.file.hgkvfile.builder.HgkvDirBuilderImpl) Test(org.junit.Test)

Aggregations

RandomAccessInput (com.baidu.hugegraph.computer.core.io.RandomAccessInput)3 Sorter (com.baidu.hugegraph.computer.core.sort.Sorter)3 KvOuterSortFlusher (com.baidu.hugegraph.computer.core.sort.flusher.KvOuterSortFlusher)3 Test (org.junit.Test)3 BytesInput (com.baidu.hugegraph.computer.core.io.BytesInput)2 BytesOutput (com.baidu.hugegraph.computer.core.io.BytesOutput)2 CombineKvOuterSortFlusher (com.baidu.hugegraph.computer.core.sort.flusher.CombineKvOuterSortFlusher)2 KvEntry (com.baidu.hugegraph.computer.core.store.entry.KvEntry)2 ArrayList (java.util.ArrayList)2 Random (java.util.Random)2 IntValueSumCombiner (com.baidu.hugegraph.computer.core.combiner.IntValueSumCombiner)1 PointerCombiner (com.baidu.hugegraph.computer.core.combiner.PointerCombiner)1 Constants (com.baidu.hugegraph.computer.core.common.Constants)1 ComputerOptions (com.baidu.hugegraph.computer.core.config.ComputerOptions)1 Config (com.baidu.hugegraph.computer.core.config.Config)1 IntValue (com.baidu.hugegraph.computer.core.graph.value.IntValue)1 IOFactory (com.baidu.hugegraph.computer.core.io.IOFactory)1 SorterTestUtil (com.baidu.hugegraph.computer.core.sort.SorterTestUtil)1 CombineKvInnerSortFlusher (com.baidu.hugegraph.computer.core.sort.flusher.CombineKvInnerSortFlusher)1 InnerSortFlusher (com.baidu.hugegraph.computer.core.sort.flusher.InnerSortFlusher)1