Examples with FlatJoinFunction org.apache.flink.api.common.functions.FlatJoinFunction used on opensource projects

Example 1 with FlatJoinFunction

use of org.apache.flink.api.common.functions.FlatJoinFunction in project flink by apache.

the class NamesTest method testJoinWith.

@Test
public void testJoinWith() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    List<Tuple1<String>> strLi = new ArrayList<Tuple1<String>>();
    strLi.add(new Tuple1<String>("a"));
    strLi.add(new Tuple1<String>("b"));
    DataSet<Tuple1<String>> strs = env.fromCollection(strLi);
    DataSet<Tuple1<String>> strs1 = env.fromCollection(strLi);
    strs.join(strs1).where(0).equalTo(0).with(new FlatJoinFunction<Tuple1<String>, Tuple1<String>, String>() {

        @Override
        public void join(Tuple1<String> first, Tuple1<String> second, Collector<String> out) throws Exception {
        //
        }
    }).output(new DiscardingOutputFormat<String>());
    Plan plan = env.createProgramPlan();
    plan.accept(new Visitor<Operator<?>>() {

        @Override
        public boolean preVisit(Operator<?> visitable) {
            if (visitable instanceof InnerJoinOperatorBase) {
                Assert.assertEquals("Join at testJoinWith(NamesTest.java:93)", visitable.getName());
            }
            return true;
        }

        @Override
        public void postVisit(Operator<?> visitable) {
        }
    });
}

Example 2 with FlatJoinFunction

use of org.apache.flink.api.common.functions.FlatJoinFunction in project flink by apache.

the class ReusingHashJoinIteratorITCase method testBuildSecondWithHighNumberOfCommonKeys.

@Test
public void testBuildSecondWithHighNumberOfCommonKeys() {
    // the size of the left and right inputs
    final int INPUT_1_SIZE = 200;
    final int INPUT_2_SIZE = 100;
    final int INPUT_1_DUPLICATES = 10;
    final int INPUT_2_DUPLICATES = 2000;
    final int DUPLICATE_KEY = 13;
    try {
        TestData.TupleGenerator generator1 = new TestData.TupleGenerator(SEED1, 500, 4096, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
        TestData.TupleGenerator generator2 = new TestData.TupleGenerator(SEED2, 500, 2048, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
        final TestData.TupleGeneratorIterator gen1Iter = new TestData.TupleGeneratorIterator(generator1, INPUT_1_SIZE);
        final TestData.TupleGeneratorIterator gen2Iter = new TestData.TupleGeneratorIterator(generator2, INPUT_2_SIZE);
        final TestData.TupleConstantValueIterator const1Iter = new TestData.TupleConstantValueIterator(DUPLICATE_KEY, "LEFT String for Duplicate Keys", INPUT_1_DUPLICATES);
        final TestData.TupleConstantValueIterator const2Iter = new TestData.TupleConstantValueIterator(DUPLICATE_KEY, "RIGHT String for Duplicate Keys", INPUT_2_DUPLICATES);
        final List<MutableObjectIterator<Tuple2<Integer, String>>> inList1 = new ArrayList<>();
        inList1.add(gen1Iter);
        inList1.add(const1Iter);
        final List<MutableObjectIterator<Tuple2<Integer, String>>> inList2 = new ArrayList<>();
        inList2.add(gen2Iter);
        inList2.add(const2Iter);
        MutableObjectIterator<Tuple2<Integer, String>> input1 = new UnionIterator<>(inList1);
        MutableObjectIterator<Tuple2<Integer, String>> input2 = new UnionIterator<>(inList2);
        // collect expected data
        final Map<Integer, Collection<TupleMatch>> expectedMatchesMap = joinTuples(collectTupleData(input1), collectTupleData(input2));
        // re-create the whole thing for actual processing
        // reset the generators and iterators
        generator1.reset();
        generator2.reset();
        const1Iter.reset();
        const2Iter.reset();
        gen1Iter.reset();
        gen2Iter.reset();
        inList1.clear();
        inList1.add(gen1Iter);
        inList1.add(const1Iter);
        inList2.clear();
        inList2.add(gen2Iter);
        inList2.add(const2Iter);
        input1 = new UnionIterator<>(inList1);
        input2 = new UnionIterator<>(inList2);
        final FlatJoinFunction matcher = new TupleMatchRemovingJoin(expectedMatchesMap);
        final Collector<Tuple2<Integer, String>> collector = new DiscardingOutputCollector<>();
        ReusingBuildSecondHashJoinIterator<Tuple2<Integer, String>, Tuple2<Integer, String>, Tuple2<Integer, String>> iterator = new ReusingBuildSecondHashJoinIterator<>(input1, input2, this.recordSerializer, this.record1Comparator, this.recordSerializer, this.record2Comparator, this.recordPairComparator, this.memoryManager, ioManager, this.parentTask, 1.0, false, false, true);
        iterator.open();
        while (iterator.callWithNextKey(matcher, collector)) ;
        iterator.close();
        // assert that each expected match was seen
        for (Entry<Integer, Collection<TupleMatch>> entry : expectedMatchesMap.entrySet()) {
            if (!entry.getValue().isEmpty()) {
                Assert.fail("Collection for key " + entry.getKey() + " is not empty");
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("An exception occurred during the test: " + e.getMessage());
    }
}

Example 3 with FlatJoinFunction

use of org.apache.flink.api.common.functions.FlatJoinFunction in project flink by apache.

the class ReusingHashJoinIteratorITCase method testBuildFirst.

@Test
public void testBuildFirst() {
    try {
        TestData.TupleGenerator generator1 = new TestData.TupleGenerator(SEED1, 500, 4096, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
        TestData.TupleGenerator generator2 = new TestData.TupleGenerator(SEED2, 500, 2048, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
        final TestData.TupleGeneratorIterator input1 = new TestData.TupleGeneratorIterator(generator1, INPUT_1_SIZE);
        final TestData.TupleGeneratorIterator input2 = new TestData.TupleGeneratorIterator(generator2, INPUT_2_SIZE);
        // collect expected data
        final Map<Integer, Collection<TupleMatch>> expectedMatchesMap = joinTuples(collectTupleData(input1), collectTupleData(input2));
        final FlatJoinFunction matcher = new TupleMatchRemovingJoin(expectedMatchesMap);
        final Collector<Tuple2<Integer, String>> collector = new DiscardingOutputCollector<>();
        // reset the generators
        generator1.reset();
        generator2.reset();
        input1.reset();
        input2.reset();
        // compare with iterator values
        ReusingBuildFirstHashJoinIterator<Tuple2<Integer, String>, Tuple2<Integer, String>, Tuple2<Integer, String>> iterator = new ReusingBuildFirstHashJoinIterator<>(input1, input2, this.recordSerializer, this.record1Comparator, this.recordSerializer, this.record2Comparator, this.recordPairComparator, this.memoryManager, ioManager, this.parentTask, 1.0, false, false, true);
        iterator.open();
        while (iterator.callWithNextKey(matcher, collector)) ;
        iterator.close();
        // assert that each expected match was seen
        for (Entry<Integer, Collection<TupleMatch>> entry : expectedMatchesMap.entrySet()) {
            if (!entry.getValue().isEmpty()) {
                Assert.fail("Collection for key " + entry.getKey() + " is not empty");
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("An exception occurred during the test: " + e.getMessage());
    }
}

Example 4 with FlatJoinFunction

use of org.apache.flink.api.common.functions.FlatJoinFunction in project flink by apache.

the class ReusingReOpenableHashTableITCase method doTest.

protected void doTest(TestData.TupleGeneratorIterator buildInput, TestData.TupleGeneratorIterator probeInput, TupleGenerator bgen, TupleGenerator pgen) throws Exception {
    // collect expected data
    final Map<Integer, Collection<TupleMatch>> expectedFirstMatchesMap = joinTuples(collectTupleData(buildInput), collectTupleData(probeInput));
    final List<Map<Integer, Collection<TupleMatch>>> expectedNMatchesMapList = new ArrayList<>(NUM_PROBES);
    final FlatJoinFunction[] nMatcher = new TupleMatchRemovingJoin[NUM_PROBES];
    for (int i = 0; i < NUM_PROBES; i++) {
        Map<Integer, Collection<TupleMatch>> tmp;
        expectedNMatchesMapList.add(tmp = deepCopy(expectedFirstMatchesMap));
        nMatcher[i] = new TupleMatchRemovingJoin(tmp);
    }
    final FlatJoinFunction firstMatcher = new TupleMatchRemovingJoin(expectedFirstMatchesMap);
    final Collector<Tuple2<Integer, String>> collector = new DiscardingOutputCollector<>();
    // reset the generators
    bgen.reset();
    pgen.reset();
    buildInput.reset();
    probeInput.reset();
    // compare with iterator values
    ReusingBuildFirstReOpenableHashJoinIterator<Tuple2<Integer, String>, Tuple2<Integer, String>, Tuple2<Integer, String>> iterator = new ReusingBuildFirstReOpenableHashJoinIterator<>(buildInput, probeInput, this.recordSerializer, this.record1Comparator, this.recordSerializer, this.record2Comparator, this.recordPairComparator, this.memoryManager, ioManager, this.parentTask, 1.0, false, false, true);
    iterator.open();
    // do first join with both inputs
    while (iterator.callWithNextKey(firstMatcher, collector)) ;
    // assert that each expected match was seen for the first input
    for (Entry<Integer, Collection<TupleMatch>> entry : expectedFirstMatchesMap.entrySet()) {
        if (!entry.getValue().isEmpty()) {
            Assert.fail("Collection for key " + entry.getKey() + " is not empty");
        }
    }
    for (int i = 0; i < NUM_PROBES; i++) {
        pgen.reset();
        probeInput.reset();
        // prepare ..
        iterator.reopenProbe(probeInput);
        // .. and do second join
        while (iterator.callWithNextKey(nMatcher[i], collector)) ;
        // assert that each expected match was seen for the second input
        for (Entry<Integer, Collection<TupleMatch>> entry : expectedNMatchesMapList.get(i).entrySet()) {
            if (!entry.getValue().isEmpty()) {
                Assert.fail("Collection for key " + entry.getKey() + " is not empty");
            }
        }
    }
    iterator.close();
}

Example 5 with FlatJoinFunction

use of org.apache.flink.api.common.functions.FlatJoinFunction in project flink by apache.

the class NonReusingReOpenableHashTableITCase method doTest.

protected void doTest(TestData.TupleGeneratorIterator buildInput, TestData.TupleGeneratorIterator probeInput, TupleGenerator bgen, TupleGenerator pgen) throws Exception {
    // collect expected data
    final Map<Integer, Collection<TupleMatch>> expectedFirstMatchesMap = joinTuples(collectTupleData(buildInput), collectTupleData(probeInput));
    final List<Map<Integer, Collection<TupleMatch>>> expectedNMatchesMapList = new ArrayList<>(NUM_PROBES);
    final FlatJoinFunction[] nMatcher = new TupleMatchRemovingJoin[NUM_PROBES];
    for (int i = 0; i < NUM_PROBES; i++) {
        Map<Integer, Collection<TupleMatch>> tmp;
        expectedNMatchesMapList.add(tmp = deepCopy(expectedFirstMatchesMap));
        nMatcher[i] = new TupleMatchRemovingJoin(tmp);
    }
    final FlatJoinFunction firstMatcher = new TupleMatchRemovingJoin(expectedFirstMatchesMap);
    final Collector<Tuple2<Integer, String>> collector = new DiscardingOutputCollector<>();
    // reset the generators
    bgen.reset();
    pgen.reset();
    buildInput.reset();
    probeInput.reset();
    // compare with iterator values
    NonReusingBuildFirstReOpenableHashJoinIterator<Tuple2<Integer, String>, Tuple2<Integer, String>, Tuple2<Integer, String>> iterator = new NonReusingBuildFirstReOpenableHashJoinIterator<>(buildInput, probeInput, this.recordSerializer, this.record1Comparator, this.recordSerializer, this.record2Comparator, this.recordPairComparator, this.memoryManager, ioManager, this.parentTask, 1.0, false, false, true);
    iterator.open();
    // do first join with both inputs
    while (iterator.callWithNextKey(firstMatcher, collector)) ;
    // assert that each expected match was seen for the first input
    for (Entry<Integer, Collection<TupleMatch>> entry : expectedFirstMatchesMap.entrySet()) {
        if (!entry.getValue().isEmpty()) {
            Assert.fail("Collection for key " + entry.getKey() + " is not empty");
        }
    }
    for (int i = 0; i < NUM_PROBES; i++) {
        pgen.reset();
        probeInput.reset();
        // prepare ..
        iterator.reopenProbe(probeInput);
        // .. and do second join
        while (iterator.callWithNextKey(nMatcher[i], collector)) ;
        // assert that each expected match was seen for the second input
        for (Entry<Integer, Collection<TupleMatch>> entry : expectedNMatchesMapList.get(i).entrySet()) {
            if (!entry.getValue().isEmpty()) {
                Assert.fail("Collection for key " + entry.getKey() + " is not empty");
            }
        }
    }
    iterator.close();
}