Examples with ReusingMergeInnerJoinIterator org.apache.flink.runtime.operators.sort.ReusingMergeInnerJoinIterator used on opensource projects

Example 1 with ReusingMergeInnerJoinIterator

use of org.apache.flink.runtime.operators.sort.ReusingMergeInnerJoinIterator in project flink by apache.

the class HashVsSortMiniBenchmark method testSortBothMerge.

@Test
public void testSortBothMerge() {
    try {
        TestData.TupleGenerator generator1 = new TestData.TupleGenerator(SEED1, INPUT_1_SIZE / 10, 100, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
        TestData.TupleGenerator generator2 = new TestData.TupleGenerator(SEED2, INPUT_2_SIZE, 100, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
        final TestData.TupleGeneratorIterator input1 = new TestData.TupleGeneratorIterator(generator1, INPUT_1_SIZE);
        final TestData.TupleGeneratorIterator input2 = new TestData.TupleGeneratorIterator(generator2, INPUT_2_SIZE);
        final FlatJoinFunction matcher = new NoOpMatcher();
        final Collector<Tuple2<Integer, String>> collector = new DiscardingOutputCollector<>();
        long start = System.nanoTime();
        final UnilateralSortMerger<Tuple2<Integer, String>> sorter1 = new UnilateralSortMerger<>(this.memoryManager, this.ioManager, input1, this.parentTask, this.serializer1, this.comparator1.duplicate(), (double) MEMORY_FOR_SORTER / MEMORY_SIZE, 128, 0.8f, true, /*use large record handler*/
        true);
        final UnilateralSortMerger<Tuple2<Integer, String>> sorter2 = new UnilateralSortMerger<>(this.memoryManager, this.ioManager, input2, this.parentTask, this.serializer2, this.comparator2.duplicate(), (double) MEMORY_FOR_SORTER / MEMORY_SIZE, 128, 0.8f, true, /*use large record handler*/
        true);
        final MutableObjectIterator<Tuple2<Integer, String>> sortedInput1 = sorter1.getIterator();
        final MutableObjectIterator<Tuple2<Integer, String>> sortedInput2 = sorter2.getIterator();
        // compare with iterator values
        ReusingMergeInnerJoinIterator<Tuple2<Integer, String>, Tuple2<Integer, String>, Tuple2<Integer, String>> iterator = new ReusingMergeInnerJoinIterator<>(sortedInput1, sortedInput2, this.serializer1.getSerializer(), this.comparator1, this.serializer2.getSerializer(), this.comparator2, this.pairComparator11, this.memoryManager, this.ioManager, MEMORY_PAGES_FOR_MERGE, this.parentTask);
        iterator.open();
        while (iterator.callWithNextKey(matcher, collector)) ;
        iterator.close();
        sorter1.close();
        sorter2.close();
        long elapsed = System.nanoTime() - start;
        double msecs = elapsed / (1000 * 1000);
        System.out.println("Sort-Merge Took " + msecs + " msecs.");
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("An exception occurred during the test: " + e.getMessage());
    }
}