Examples with ReplicatingInputFormat org.apache.flink.api.common.io.ReplicatingInputFormat used on opensource projects

Example 16 with ReplicatingInputFormat

use of org.apache.flink.api.common.io.ReplicatingInputFormat in project flink by apache.

the class ReplicatingDataSourceITCase method testReplicatedSourceToJoin.

@Test
public void testReplicatedSourceToJoin() throws Exception {
    /*
         * Test replicated source going into join
         */
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple1<Long>> source1 = env.createInput(new ReplicatingInputFormat<Long, GenericInputSplit>(new ParallelIteratorInputFormat<Long>(new NumberSequenceIterator(0L, 1000L))), BasicTypeInfo.LONG_TYPE_INFO).map(new ToTuple());
    DataSet<Tuple1<Long>> source2 = env.generateSequence(0L, 1000L).map(new ToTuple());
    DataSet<Tuple> pairs = source1.join(source2).where(0).equalTo(0).projectFirst(0).sum(0);
    List<Tuple> result = pairs.collect();
    String expectedResult = "(500500)";
    compareResultAsText(result, expectedResult);
}

Example 17 with ReplicatingInputFormat

use of org.apache.flink.api.common.io.ReplicatingInputFormat in project flink by apache.

the class ReplicatingDataSourceITCase method testReplicatedSourceToCross.

@Test
public void testReplicatedSourceToCross() throws Exception {
    /*
         * Test replicated source going into cross
         */
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple1<Long>> source1 = env.createInput(new ReplicatingInputFormat<Long, GenericInputSplit>(new ParallelIteratorInputFormat<Long>(new NumberSequenceIterator(0L, 1000L))), BasicTypeInfo.LONG_TYPE_INFO).map(new ToTuple());
    DataSet<Tuple1<Long>> source2 = env.generateSequence(0L, 1000L).map(new ToTuple());
    DataSet<Tuple1<Long>> pairs = source1.cross(source2).filter(new FilterFunction<Tuple2<Tuple1<Long>, Tuple1<Long>>>() {

        @Override
        public boolean filter(Tuple2<Tuple1<Long>, Tuple1<Long>> value) throws Exception {
            return value.f0.f0.equals(value.f1.f0);
        }
    }).map(new MapFunction<Tuple2<Tuple1<Long>, Tuple1<Long>>, Tuple1<Long>>() {

        @Override
        public Tuple1<Long> map(Tuple2<Tuple1<Long>, Tuple1<Long>> value) throws Exception {
            return value.f0;
        }
    }).sum(0);
    List<Tuple1<Long>> result = pairs.collect();
    String expectedResult = "(500500)";
    compareResultAsText(result, expectedResult);
}