Examples with BytesOutput com.baidu.hugegraph.computer.core.io.BytesOutput used on opensource projects

Example 6 with BytesOutput

use of com.baidu.hugegraph.computer.core.io.BytesOutput in project hugegraph-computer by hugegraph.

the class SorterTest method testSortKvBuffer.

@Test
public void testSortKvBuffer() throws Exception {
    Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.HGKV_MAX_FILE_SIZE, "32", ComputerOptions.HGKV_DATABLOCK_SIZE, "16", ComputerOptions.HGKV_MERGE_FILES_NUM, "3");
    List<Integer> map = ImmutableList.of(2, 3, 1, 23, 6, 2, 5, 9, 2, 2, 6, 1, 1, 20);
    BytesInput input = SorterTestUtil.inputFromKvMap(map);
    BytesOutput output = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
    Sorter sorter = SorterTestUtil.createSorter(config);
    PointerCombiner combiner = SorterTestUtil.createPointerCombiner(IntValue::new, new IntValueSumCombiner());
    sorter.sortBuffer(input, new CombineKvInnerSortFlusher(output, combiner), false);
    BytesInput resultInput = EntriesUtil.inputFromOutput(output);
    KvEntriesInput iter = new KvEntriesInput(resultInput);
    SorterTestUtil.assertKvEntry(iter.next(), 1, 43);
    SorterTestUtil.assertKvEntry(iter.next(), 2, 5);
    SorterTestUtil.assertKvEntry(iter.next(), 5, 9);
    SorterTestUtil.assertKvEntry(iter.next(), 6, 3);
    iter.close();
}

Example 7 with BytesOutput

use of com.baidu.hugegraph.computer.core.io.BytesOutput in project hugegraph-computer by hugegraph.

the class FlusherTest method testKvInnerSortFlusher.

@Test
public void testKvInnerSortFlusher() throws Exception {
    List<Integer> map = ImmutableList.of(2, 1, 3, 1, 2, 1, 4, 1);
    BytesInput input = SorterTestUtil.inputFromKvMap(map);
    BytesOutput output = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
    Sorter sorter = SorterTestUtil.createSorter(CONFIG);
    sorter.sortBuffer(input, new KvInnerSortFlusher(output), false);
    BytesInput result = EntriesUtil.inputFromOutput(output);
    EntryIterator iter = new KvEntriesInput(result);
    SorterTestUtil.assertKvEntry(iter.next(), 2, 1);
    SorterTestUtil.assertKvEntry(iter.next(), 2, 1);
    SorterTestUtil.assertKvEntry(iter.next(), 3, 1);
    SorterTestUtil.assertKvEntry(iter.next(), 4, 1);
    iter.close();
}

Example 8 with BytesOutput

use of com.baidu.hugegraph.computer.core.io.BytesOutput in project hugegraph-computer by hugegraph.

the class FlusherTest method testExceptionCaseForFlusher.

@Test
public void testExceptionCaseForFlusher() {
    BytesOutput output = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
    InnerSortFlusher flusher = new KvInnerSortFlusher(output);
    List<KvEntry> entries = new ArrayList<>();
    Assert.assertThrows(IllegalArgumentException.class, () -> {
        flusher.flush(entries.iterator());
    }, e -> {
        String errorMsg = "Parameter entries can't be empty";
        Assert.assertContains(errorMsg, e.getMessage());
    });
}

Example 9 with BytesOutput

use of com.baidu.hugegraph.computer.core.io.BytesOutput in project hugegraph-computer by hugegraph.

the class FlusherTest method testOverwriteCombiner.

@Test
public void testOverwriteCombiner() throws Exception {
    List<Integer> data = ImmutableList.of(1, 2, 3, 5, 1, 3, 1, 1, 3, 4);
    BytesInput input = SorterTestUtil.inputFromKvMap(data);
    BytesOutput output = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
    PointerCombiner combiner = SorterTestUtil.createPointerCombiner(IntValue::new, new OverwriteCombiner<>());
    InnerSortFlusher flusher = new CombineKvInnerSortFlusher(output, combiner);
    Sorter sorter = SorterTestUtil.createSorter(CONFIG);
    sorter.sortBuffer(input, flusher, false);
    BytesInput result = EntriesUtil.inputFromOutput(output);
    // Assert result
    KvEntriesInput iter = new KvEntriesInput(result);
    SorterTestUtil.assertKvEntry(iter.next(), 1, 1);
    SorterTestUtil.assertKvEntry(iter.next(), 3, 4);
    iter.close();
}

Example 10 with BytesOutput

use of com.baidu.hugegraph.computer.core.io.BytesOutput in project hugegraph-computer by hugegraph.

the class SortLargeDataTest method testMergeBuffersAllSameKey.

@Test
public void testMergeBuffersAllSameKey() throws Exception {
    List<RandomAccessInput> buffers = new ArrayList<>();
    for (int i = 0; i < 1000; i++) {
        BytesOutput buffer = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
        for (int j = 0; j < 100; j++) {
            // Write data
            SorterTestUtil.writeData(buffer, 1);
            SorterTestUtil.writeData(buffer, 1);
        }
        buffers.add(EntriesUtil.inputFromOutput(buffer));
    }
    String resultFile = StoreTestUtil.availablePathById("0");
    Sorter sorter = SorterTestUtil.createSorter(CONFIG);
    mergeBuffers(sorter, buffers, resultFile);
    // Assert result
    long result = sumOfEntryValue(sorter, ImmutableList.of(resultFile));
    Assert.assertEquals(1000 * 100, result);
}