Examples with BroadcastProcessFunction org.apache.flink.streaming.api.functions.co.BroadcastProcessFunction used on opensource projects

Example 1 with BroadcastProcessFunction

use of org.apache.flink.streaming.api.functions.co.BroadcastProcessFunction in project flink by apache.

the class StreamGraphGeneratorTest method testUnalignedCheckpointDisabledOnPointwise.

@Test
public void testUnalignedCheckpointDisabledOnPointwise() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(42);
    DataStream<Long> source1 = env.fromSequence(1L, 10L);
    DataStream<Long> map1 = source1.forward().map(l -> l);
    DataStream<Long> source2 = env.fromSequence(2L, 11L);
    DataStream<Long> map2 = source2.shuffle().map(l -> l);
    final MapStateDescriptor<Long, Long> descriptor = new MapStateDescriptor<>("broadcast", BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO);
    final BroadcastStream<Long> broadcast = map1.broadcast(descriptor);
    final SingleOutputStreamOperator<Long> joined = map2.connect(broadcast).process(new BroadcastProcessFunction<Long, Long, Long>() {

        @Override
        public void processElement(Long value, ReadOnlyContext ctx, Collector<Long> out) {
        }

        @Override
        public void processBroadcastElement(Long value, Context ctx, Collector<Long> out) {
        }
    });
    DataStream<Long> map3 = joined.shuffle().map(l -> l);
    DataStream<Long> map4 = map3.rescale().map(l -> l).setParallelism(1337);
    StreamGraph streamGraph = env.getStreamGraph();
    assertEquals(7, streamGraph.getStreamNodes().size());
    // forward
    assertThat(edge(streamGraph, source1, map1), supportsUnalignedCheckpoints(false));
    // shuffle
    assertThat(edge(streamGraph, source2, map2), supportsUnalignedCheckpoints(true));
    // broadcast, but other channel is forwarded
    assertThat(edge(streamGraph, map1, joined), supportsUnalignedCheckpoints(false));
    // forward
    assertThat(edge(streamGraph, map2, joined), supportsUnalignedCheckpoints(false));
    // shuffle
    assertThat(edge(streamGraph, joined, map3), supportsUnalignedCheckpoints(true));
    // rescale
    assertThat(edge(streamGraph, map3, map4), supportsUnalignedCheckpoints(false));
}

Example 2 with BroadcastProcessFunction

use of org.apache.flink.streaming.api.functions.co.BroadcastProcessFunction in project flink by apache.

the class DataStreamTest method testFailedTranslationOnKeyed.

/**
 * Tests that with a {@link KeyedStream} we have to provide a {@link
 * KeyedBroadcastProcessFunction}.
 */
@Test
public void testFailedTranslationOnKeyed() {
    final MapStateDescriptor<Long, String> descriptor = new MapStateDescriptor<>("broadcast", BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    final DataStream<Long> srcOne = env.generateSequence(0L, 5L).assignTimestampsAndWatermarks(new CustomWmEmitter<Long>() {

        @Override
        public long extractTimestamp(Long element, long previousElementTimestamp) {
            return element;
        }
    }).keyBy((KeySelector<Long, Long>) value -> value);
    final DataStream<String> srcTwo = env.fromElements("Test:0", "Test:1", "Test:2", "Test:3", "Test:4", "Test:5").assignTimestampsAndWatermarks(new CustomWmEmitter<String>() {

        @Override
        public long extractTimestamp(String element, long previousElementTimestamp) {
            return Long.parseLong(element.split(":")[1]);
        }
    });
    BroadcastStream<String> broadcast = srcTwo.broadcast(descriptor);
    BroadcastConnectedStream<Long, String> bcStream = srcOne.connect(broadcast);
    expectedException.expect(IllegalArgumentException.class);
    bcStream.process(new BroadcastProcessFunction<Long, String, String>() {

        @Override
        public void processBroadcastElement(String value, Context ctx, Collector<String> out) throws Exception {
        // do nothing
        }

        @Override
        public void processElement(Long value, ReadOnlyContext ctx, Collector<String> out) throws Exception {
        // do nothing
        }
    });
}