Examples with CoMapFunction org.apache.flink.streaming.api.functions.co.CoMapFunction used on opensource projects

Example 1 with CoMapFunction

use of org.apache.flink.streaming.api.functions.co.CoMapFunction in project flink by apache.

the class IterateITCase method testCoIteration.

@Test
public void testCoIteration() throws Exception {
    int numRetries = 5;
    int timeoutScale = 1;
    for (int numRetry = 0; numRetry < numRetries; numRetry++) {
        try {
            TestSink.collected = new ArrayList<>();
            StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
            env.setParallelism(2);
            DataStream<String> otherSource = env.fromElements("1000", "2000").map(noOpStrMap).name("ParallelizeMap");
            ConnectedIterativeStreams<Integer, String> coIt = env.fromElements(0, 0).map(noOpIntMap).name("ParallelizeMap").iterate(2000 * timeoutScale).withFeedbackType(Types.STRING);
            try {
                coIt.keyBy(1, 2);
                fail();
            } catch (InvalidProgramException e) {
            // this is expected
            }
            DataStream<String> head = coIt.flatMap(new RichCoFlatMapFunction<Integer, String, String>() {

                private static final long serialVersionUID = 1L;

                boolean seenFromSource = false;

                @Override
                public void flatMap1(Integer value, Collector<String> out) throws Exception {
                    out.collect(((Integer) (value + 1)).toString());
                }

                @Override
                public void flatMap2(String value, Collector<String> out) throws Exception {
                    Integer intVal = Integer.valueOf(value);
                    if (intVal < 2) {
                        out.collect(((Integer) (intVal + 1)).toString());
                    }
                    if (intVal == 1000 || intVal == 2000) {
                        seenFromSource = true;
                    }
                }

                @Override
                public void close() {
                    assertTrue(seenFromSource);
                }
            });
            coIt.map(new CoMapFunction<Integer, String, String>() {

                @Override
                public String map1(Integer value) throws Exception {
                    return value.toString();
                }

                @Override
                public String map2(String value) throws Exception {
                    return value;
                }
            }).addSink(new ReceiveCheckNoOpSink<String>());
            coIt.closeWith(head.broadcast().union(otherSource));
            head.addSink(new TestSink()).setParallelism(1);
            assertEquals(1, env.getStreamGraph(false).getIterationSourceSinkPairs().size());
            env.execute();
            Collections.sort(TestSink.collected);
            assertEquals(Arrays.asList("1", "1", "2", "2", "2", "2"), TestSink.collected);
            // success
            break;
        } catch (Throwable t) {
            LOG.info("Run " + (numRetry + 1) + "/" + numRetries + " failed", t);
            if (numRetry >= numRetries - 1) {
                throw t;
            } else {
                timeoutScale *= 2;
            }
        }
    }
}

Example 2 with CoMapFunction

use of org.apache.flink.streaming.api.functions.co.CoMapFunction in project flink by apache.

the class SelfConnectionITCase method differentDataStreamSameChain.

/**
 * We connect two different data streams in a chain to a CoMap.
 */
@Test
public void differentDataStreamSameChain() throws Exception {
    TestListResultSink<String> resultSink = new TestListResultSink<>();
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(1);
    DataStream<Integer> src = env.fromElements(1, 3, 5);
    DataStream<String> stringMap = src.map(value -> "x " + value);
    stringMap.connect(src).map(new CoMapFunction<String, Integer, String>() {

        @Override
        public String map1(String value) {
            return value;
        }

        @Override
        public String map2(Integer value) {
            return String.valueOf(value + 1);
        }
    }).addSink(resultSink);
    env.execute();
    List<String> expected = Arrays.asList("x 1", "x 3", "x 5", "2", "4", "6");
    List<String> result = resultSink.getResult();
    Collections.sort(expected);
    Collections.sort(result);
    assertEquals(expected, result);
}

Example 3 with CoMapFunction

use of org.apache.flink.streaming.api.functions.co.CoMapFunction in project flink by apache.

the class SelfConnectionITCase method differentDataStreamDifferentChain.

/**
 * We connect two different data streams in different chains to a CoMap. (This is not actually
 * self-connect.)
 */
@Test
public void differentDataStreamDifferentChain() throws Exception {
    TestListResultSink<String> resultSink = new TestListResultSink<>();
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(3);
    DataStream<Integer> src = env.fromElements(1, 3, 5).disableChaining();
    DataStream<String> stringMap = src.flatMap(new FlatMapFunction<Integer, String>() {

        @Override
        public void flatMap(Integer value, Collector<String> out) throws Exception {
            out.collect("x " + value);
        }
    }).keyBy(String::length);
    DataStream<Long> longMap = src.map(value -> (long) (value + 1)).keyBy(Long::intValue);
    stringMap.connect(longMap).map(new CoMapFunction<String, Long, String>() {

        @Override
        public String map1(String value) {
            return value;
        }

        @Override
        public String map2(Long value) {
            return value.toString();
        }
    }).addSink(resultSink);
    env.execute();
    List<String> expected = Arrays.asList("x 1", "x 3", "x 5", "2", "4", "6");
    List<String> result = resultSink.getResult();
    Collections.sort(expected);
    Collections.sort(result);
    assertEquals(expected, result);
}

Example 4 with CoMapFunction

use of org.apache.flink.streaming.api.functions.co.CoMapFunction in project flink by apache.

the class DataStreamTest method operatorTest.

@Test
public void operatorTest() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStreamSource<Long> src = env.generateSequence(0, 0);
    MapFunction<Long, Integer> mapFunction = new MapFunction<Long, Integer>() {

        @Override
        public Integer map(Long value) throws Exception {
            return null;
        }
    };
    DataStream<Integer> map = src.map(mapFunction);
    map.addSink(new DiscardingSink<Integer>());
    assertEquals(mapFunction, getFunctionForDataStream(map));
    FlatMapFunction<Long, Integer> flatMapFunction = new FlatMapFunction<Long, Integer>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void flatMap(Long value, Collector<Integer> out) throws Exception {
        }
    };
    DataStream<Integer> flatMap = src.flatMap(flatMapFunction);
    flatMap.addSink(new DiscardingSink<Integer>());
    assertEquals(flatMapFunction, getFunctionForDataStream(flatMap));
    FilterFunction<Integer> filterFunction = new FilterFunction<Integer>() {

        @Override
        public boolean filter(Integer value) throws Exception {
            return false;
        }
    };
    DataStream<Integer> unionFilter = map.union(flatMap).filter(filterFunction);
    unionFilter.addSink(new DiscardingSink<Integer>());
    assertEquals(filterFunction, getFunctionForDataStream(unionFilter));
    try {
        getStreamGraph(env).getStreamEdgesOrThrow(map.getId(), unionFilter.getId());
    } catch (RuntimeException e) {
        fail(e.getMessage());
    }
    try {
        getStreamGraph(env).getStreamEdgesOrThrow(flatMap.getId(), unionFilter.getId());
    } catch (RuntimeException e) {
        fail(e.getMessage());
    }
    ConnectedStreams<Integer, Integer> connect = map.connect(flatMap);
    CoMapFunction<Integer, Integer, String> coMapper = new CoMapFunction<Integer, Integer, String>() {

        private static final long serialVersionUID = 1L;

        @Override
        public String map1(Integer value) {
            return null;
        }

        @Override
        public String map2(Integer value) {
            return null;
        }
    };
    DataStream<String> coMap = connect.map(coMapper);
    coMap.addSink(new DiscardingSink<String>());
    assertEquals(coMapper, getFunctionForDataStream(coMap));
    try {
        getStreamGraph(env).getStreamEdgesOrThrow(map.getId(), coMap.getId());
    } catch (RuntimeException e) {
        fail(e.getMessage());
    }
    try {
        getStreamGraph(env).getStreamEdgesOrThrow(flatMap.getId(), coMap.getId());
    } catch (RuntimeException e) {
        fail(e.getMessage());
    }
}

Example 5 with CoMapFunction

use of org.apache.flink.streaming.api.functions.co.CoMapFunction in project flink by apache.

the class SlotAllocationTest method testCoOperation.

@Test
public void testCoOperation() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    CoMapFunction<Long, Long, Long> dummyCoMap = new CoMapFunction<Long, Long, Long>() {

        @Override
        public Long map1(Long value) throws Exception {
            return null;
        }

        @Override
        public Long map2(Long value) throws Exception {
            return null;
        }
    };
    DataStream<Long> src1 = env.generateSequence(1, 10);
    DataStream<Long> src2 = env.generateSequence(1, 10).slotSharingGroup("src-1");
    // this should not inherit group "src-1"
    src1.connect(src2).map(dummyCoMap);
    DataStream<Long> src3 = env.generateSequence(1, 10).slotSharingGroup("group-1");
    DataStream<Long> src4 = env.generateSequence(1, 10).slotSharingGroup("group-1");
    // this should inherit "group-1" now
    src3.connect(src4).map(dummyCoMap);
    JobGraph jobGraph = env.getStreamGraph().getJobGraph();
    List<JobVertex> vertices = jobGraph.getVerticesSortedTopologicallyFromSources();
    // first pipeline
    assertEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(4).getSlotSharingGroup());
    assertNotEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(1).getSlotSharingGroup());
    assertNotEquals(vertices.get(1).getSlotSharingGroup(), vertices.get(4).getSlotSharingGroup());
    // second pipeline
    assertEquals(vertices.get(2).getSlotSharingGroup(), vertices.get(3).getSlotSharingGroup());
    assertEquals(vertices.get(2).getSlotSharingGroup(), vertices.get(5).getSlotSharingGroup());
    assertEquals(vertices.get(3).getSlotSharingGroup(), vertices.get(5).getSlotSharingGroup());
}