Examples with FlatJoinFunction org.apache.flink.api.common.functions.FlatJoinFunction used on opensource projects

Example 16 with FlatJoinFunction

use of org.apache.flink.api.common.functions.FlatJoinFunction in project flink by apache.

the class ReusingHashJoinIteratorITCase method testBuildFirstAndProbeSideOuterJoin.

@Test
public void testBuildFirstAndProbeSideOuterJoin() {
    try {
        TestData.TupleGenerator generator1 = new TestData.TupleGenerator(SEED1, 500, 4096, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
        TestData.TupleGenerator generator2 = new TestData.TupleGenerator(SEED2, 1000, 2048, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
        final TestData.TupleGeneratorIterator input1 = new TestData.TupleGeneratorIterator(generator1, INPUT_1_SIZE);
        final TestData.TupleGeneratorIterator input2 = new TestData.TupleGeneratorIterator(generator2, INPUT_2_SIZE);
        // collect expected data
        final Map<Integer, Collection<TupleMatch>> expectedMatchesMap = rightOuterJoinTuples(collectTupleData(input1), collectTupleData(input2));
        final FlatJoinFunction matcher = new TupleMatchRemovingJoin(expectedMatchesMap);
        final Collector<Tuple2<Integer, String>> collector = new DiscardingOutputCollector<>();
        // reset the generators
        generator1.reset();
        generator2.reset();
        input1.reset();
        input2.reset();
        // compare with iterator values
        ReusingBuildFirstHashJoinIterator<Tuple2<Integer, String>, Tuple2<Integer, String>, Tuple2<Integer, String>> iterator = new ReusingBuildFirstHashJoinIterator<>(input1, input2, this.recordSerializer, this.record1Comparator, this.recordSerializer, this.record2Comparator, this.recordPairComparator, this.memoryManager, ioManager, this.parentTask, 1.0, true, false, false);
        iterator.open();
        while (iterator.callWithNextKey(matcher, collector)) ;
        iterator.close();
        // assert that each expected match was seen
        for (Entry<Integer, Collection<TupleMatch>> entry : expectedMatchesMap.entrySet()) {
            if (!entry.getValue().isEmpty()) {
                Assert.fail("Collection for key " + entry.getKey() + " is not empty");
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("An exception occurred during the test: " + e.getMessage());
    }
}

Example 17 with FlatJoinFunction

use of org.apache.flink.api.common.functions.FlatJoinFunction in project flink by apache.

the class ReusingHashJoinIteratorITCase method testBuildSecondAndProbeSideOuterJoin.

@Test
public void testBuildSecondAndProbeSideOuterJoin() {
    try {
        TestData.TupleGenerator generator1 = new TestData.TupleGenerator(SEED1, 1000, 4096, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
        TestData.TupleGenerator generator2 = new TestData.TupleGenerator(SEED2, 500, 2048, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
        final TestData.TupleGeneratorIterator input1 = new TestData.TupleGeneratorIterator(generator1, INPUT_1_SIZE);
        final TestData.TupleGeneratorIterator input2 = new TestData.TupleGeneratorIterator(generator2, INPUT_2_SIZE);
        // collect expected data
        final Map<Integer, Collection<TupleMatch>> expectedMatchesMap = leftOuterJoinTuples(collectTupleData(input1), collectTupleData(input2));
        final FlatJoinFunction matcher = new TupleMatchRemovingJoin(expectedMatchesMap);
        final Collector<Tuple2<Integer, String>> collector = new DiscardingOutputCollector<>();
        // reset the generators
        generator1.reset();
        generator2.reset();
        input1.reset();
        input2.reset();
        // compare with iterator values
        ReusingBuildSecondHashJoinIterator<Tuple2<Integer, String>, Tuple2<Integer, String>, Tuple2<Integer, String>> iterator = new ReusingBuildSecondHashJoinIterator<>(input1, input2, this.recordSerializer, this.record1Comparator, this.recordSerializer, this.record2Comparator, this.recordPairComparator, this.memoryManager, ioManager, this.parentTask, 1.0, true, false, false);
        iterator.open();
        while (iterator.callWithNextKey(matcher, collector)) ;
        iterator.close();
        // assert that each expected match was seen
        for (Entry<Integer, Collection<TupleMatch>> entry : expectedMatchesMap.entrySet()) {
            if (!entry.getValue().isEmpty()) {
                Assert.fail("Collection for key " + entry.getKey() + " is not empty");
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("An exception occurred during the test: " + e.getMessage());
    }
}

Example 18 with FlatJoinFunction

use of org.apache.flink.api.common.functions.FlatJoinFunction in project flink by apache.

the class ReusingHashJoinIteratorITCase method testBuildSecondAndBuildSideOuterJoin.

@Test
public void testBuildSecondAndBuildSideOuterJoin() {
    try {
        TestData.TupleGenerator generator1 = new TestData.TupleGenerator(SEED1, 1000, 4096, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
        TestData.TupleGenerator generator2 = new TestData.TupleGenerator(SEED2, 500, 2048, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
        final TestData.TupleGeneratorIterator input1 = new TestData.TupleGeneratorIterator(generator1, INPUT_1_SIZE);
        final TestData.TupleGeneratorIterator input2 = new TestData.TupleGeneratorIterator(generator2, INPUT_2_SIZE);
        // collect expected data
        final Map<Integer, Collection<TupleMatch>> expectedMatchesMap = rightOuterJoinTuples(collectTupleData(input1), collectTupleData(input2));
        final FlatJoinFunction matcher = new TupleMatchRemovingJoin(expectedMatchesMap);
        final Collector<Tuple2<Integer, String>> collector = new DiscardingOutputCollector<>();
        // reset the generators
        generator1.reset();
        generator2.reset();
        input1.reset();
        input2.reset();
        // compare with iterator values
        ReusingBuildSecondHashJoinIterator<Tuple2<Integer, String>, Tuple2<Integer, String>, Tuple2<Integer, String>> iterator = new ReusingBuildSecondHashJoinIterator<>(input1, input2, this.recordSerializer, this.record1Comparator, this.recordSerializer, this.record2Comparator, this.recordPairComparator, this.memoryManager, ioManager, this.parentTask, 1.0, false, true, false);
        iterator.open();
        while (iterator.callWithNextKey(matcher, collector)) ;
        iterator.close();
        // assert that each expected match was seen
        for (Entry<Integer, Collection<TupleMatch>> entry : expectedMatchesMap.entrySet()) {
            if (!entry.getValue().isEmpty()) {
                Assert.fail("Collection for key " + entry.getKey() + " is not empty");
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("An exception occurred during the test: " + e.getMessage());
    }
}