Examples with PointerCombiner com.baidu.hugegraph.computer.core.combiner.PointerCombiner used on opensource projects

Example 6 with PointerCombiner

use of com.baidu.hugegraph.computer.core.combiner.PointerCombiner in project hugegraph-computer by hugegraph.

the class SorterTest method testSortKvBuffer.

@Test
public void testSortKvBuffer() throws Exception {
    Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.HGKV_MAX_FILE_SIZE, "32", ComputerOptions.HGKV_DATABLOCK_SIZE, "16", ComputerOptions.HGKV_MERGE_FILES_NUM, "3");
    List<Integer> map = ImmutableList.of(2, 3, 1, 23, 6, 2, 5, 9, 2, 2, 6, 1, 1, 20);
    BytesInput input = SorterTestUtil.inputFromKvMap(map);
    BytesOutput output = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
    Sorter sorter = SorterTestUtil.createSorter(config);
    PointerCombiner combiner = SorterTestUtil.createPointerCombiner(IntValue::new, new IntValueSumCombiner());
    sorter.sortBuffer(input, new CombineKvInnerSortFlusher(output, combiner), false);
    BytesInput resultInput = EntriesUtil.inputFromOutput(output);
    KvEntriesInput iter = new KvEntriesInput(resultInput);
    SorterTestUtil.assertKvEntry(iter.next(), 1, 43);
    SorterTestUtil.assertKvEntry(iter.next(), 2, 5);
    SorterTestUtil.assertKvEntry(iter.next(), 5, 9);
    SorterTestUtil.assertKvEntry(iter.next(), 6, 3);
    iter.close();
}

Example 7 with PointerCombiner

use of com.baidu.hugegraph.computer.core.combiner.PointerCombiner in project hugegraph-computer by hugegraph.

the class SorterTest method testSortSubKvBuffers.

@Test
public void testSortSubKvBuffers() throws Exception {
    Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.INPUT_MAX_EDGES_IN_ONE_VERTEX, "2", ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
    int flushThreshold = config.get(ComputerOptions.INPUT_MAX_EDGES_IN_ONE_VERTEX);
    BytesInput i1 = this.sortedSubKvBuffer(config);
    BytesInput i2 = this.sortedSubKvBuffer(config);
    BytesInput i3 = this.sortedSubKvBuffer(config);
    List<RandomAccessInput> buffers = ImmutableList.of(i1, i2, i3);
    Sorter sorter = SorterTestUtil.createSorter(config);
    PointerCombiner combiner = SorterTestUtil.createPointerCombiner(IntValue::new, new IntValueSumCombiner());
    OuterSortFlusher flusher = new CombineSubKvOuterSortFlusher(combiner, flushThreshold);
    flusher.sources(buffers.size());
    String outputFile = StoreTestUtil.availablePathById("1");
    sorter.mergeBuffers(buffers, flusher, outputFile, true);
    /*
         * Assert result
         * key 1 subKv 3 3, 5 3
         * key 2 subKv 5 3, 8 6
         * key 2 subKv 9 3
         * key 3 subKv 2 6, 3 3
         * key 3 subKv 4 3
         */
    ImmutableList<String> outputs = ImmutableList.of(outputFile);
    Iterator<KvEntry> kvIter = sorter.iterator(outputs, true);
    SorterTestUtil.assertSubKvByKv(kvIter.next(), 1, 3, 3, 5, 3);
    SorterTestUtil.assertSubKvByKv(kvIter.next(), 2, 5, 3, 8, 6);
    SorterTestUtil.assertSubKvByKv(kvIter.next(), 2, 9, 3);
    SorterTestUtil.assertSubKvByKv(kvIter.next(), 3, 2, 6, 3, 3);
    SorterTestUtil.assertSubKvByKv(kvIter.next(), 3, 4, 3);
    // Assert file properties
    HgkvDir dir = HgkvDirImpl.open(outputFile);
    Assert.assertEquals(5, dir.numEntries());
    Assert.assertEquals(8, dir.numSubEntries());
}

Example 8 with PointerCombiner

use of com.baidu.hugegraph.computer.core.combiner.PointerCombiner in project hugegraph-computer by hugegraph.

the class FlusherTest method testOverwriteCombiner.

@Test
public void testOverwriteCombiner() throws Exception {
    List<Integer> data = ImmutableList.of(1, 2, 3, 5, 1, 3, 1, 1, 3, 4);
    BytesInput input = SorterTestUtil.inputFromKvMap(data);
    BytesOutput output = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
    PointerCombiner combiner = SorterTestUtil.createPointerCombiner(IntValue::new, new OverwriteCombiner<>());
    InnerSortFlusher flusher = new CombineKvInnerSortFlusher(output, combiner);
    Sorter sorter = SorterTestUtil.createSorter(CONFIG);
    sorter.sortBuffer(input, flusher, false);
    BytesInput result = EntriesUtil.inputFromOutput(output);
    // Assert result
    KvEntriesInput iter = new KvEntriesInput(result);
    SorterTestUtil.assertKvEntry(iter.next(), 1, 1);
    SorterTestUtil.assertKvEntry(iter.next(), 3, 4);
    iter.close();
}

Example 9 with PointerCombiner

use of com.baidu.hugegraph.computer.core.combiner.PointerCombiner in project hugegraph-computer by hugegraph.

the class SortLargeDataTest method mergeBuffers.

private static void mergeBuffers(Sorter sorter, List<RandomAccessInput> buffers, String output) throws Exception {
    PointerCombiner combiner = SorterTestUtil.createPointerCombiner(IntValue::new, new IntValueSumCombiner());
    OuterSortFlusher flusher = new CombineKvOuterSortFlusher(combiner);
    sorter.mergeBuffers(buffers, flusher, output, false);
}

Example 10 with PointerCombiner

use of com.baidu.hugegraph.computer.core.combiner.PointerCombiner in project hugegraph-computer by hugegraph.

the class SorterTest method testMergeSubKvFiles.

private void testMergeSubKvFiles(Config config) throws Exception {
    int flushThreshold = config.get(ComputerOptions.INPUT_MAX_EDGES_IN_ONE_VERTEX);
    List<Integer> kv1 = ImmutableList.of(1, 2, 1, 4, 1);
    List<Integer> kv2 = ImmutableList.of(4, 2, 1, 3, 1);
    List<Integer> kv3 = ImmutableList.of(4, 6, 1, 8, 1);
    List<Integer> kv4 = ImmutableList.of(1, 1, 1, 2, 1);
    List<Integer> kv5 = ImmutableList.of(1, 5, 1, 7, 1);
    List<Integer> kv6 = ImmutableList.of(2, 2, 1, 5, 1);
    List<List<Integer>> data1 = ImmutableList.of(kv1, kv2, kv3);
    List<List<Integer>> data2 = ImmutableList.of(kv4, kv5, kv6);
    List<List<Integer>> data3 = ImmutableList.of(kv4, kv1, kv3);
    List<List<List<Integer>>> datas = ImmutableList.of(data1, data2, data3);
    String input1 = StoreTestUtil.availablePathById(1);
    String input2 = StoreTestUtil.availablePathById(2);
    String input3 = StoreTestUtil.availablePathById(3);
    String output = StoreTestUtil.availablePathById(0);
    List<String> inputs = ImmutableList.of(input1, input2, input3);
    List<String> outputs = ImmutableList.of(output);
    boolean useBufferFile = config.get(ComputerOptions.TRANSPORT_RECV_FILE_MODE);
    for (int i = 0; i < inputs.size(); i++) {
        String input = inputs.get(i);
        List<List<Integer>> data = datas.get(i);
        if (useBufferFile) {
            StoreTestUtil.bufferFileFromSubKvMap(data, input);
        } else {
            StoreTestUtil.hgkvDirFromSubKvMap(config, data, input);
        }
    }
    Sorter sorter = SorterTestUtil.createSorter(config);
    PointerCombiner combiner = SorterTestUtil.createPointerCombiner(IntValue::new, new IntValueSumCombiner());
    OuterSortFlusher flusher = new CombineSubKvOuterSortFlusher(combiner, flushThreshold);
    flusher.sources(inputs.size());
    sorter.mergeInputs(inputs, flusher, outputs, true);
    /* Assert result
         * key 1 subKv 1 2 2 4
         * key 1 subKv 4 2 5 1
         * key 1 subKv 7 1
         * key 2 subKv 2 1 5 1
         * key 4 subKv 2 1 3 1
         * key 4 subKv 6 2 8 2
         */
    try (CIter<KvEntry> kvIter = sorter.iterator(outputs, true)) {
        SorterTestUtil.assertSubKvByKv(kvIter.next(), 1, 1, 2, 2, 4);
        SorterTestUtil.assertSubKvByKv(kvIter.next(), 1, 4, 2, 5, 1);
        SorterTestUtil.assertSubKvByKv(kvIter.next(), 1, 7, 1);
        SorterTestUtil.assertSubKvByKv(kvIter.next(), 2, 2, 1, 5, 1);
        SorterTestUtil.assertSubKvByKv(kvIter.next(), 4, 2, 1, 3, 1);
        SorterTestUtil.assertSubKvByKv(kvIter.next(), 4, 6, 2, 8, 2);
    }
    FileUtil.deleteFilesQuietly(inputs);
    FileUtil.deleteFilesQuietly(outputs);
}