Search in sources :

Example 1 with KeyedBroadcastProcessFunction

use of org.apache.flink.streaming.api.functions.co.KeyedBroadcastProcessFunction in project flink by apache.

the class DataStreamTest method testFailedTranslationOnKeyed.

/**
 * Tests that with a {@link KeyedStream} we have to provide a {@link
 * KeyedBroadcastProcessFunction}.
 */
@Test
public void testFailedTranslationOnKeyed() {
    final MapStateDescriptor<Long, String> descriptor = new MapStateDescriptor<>("broadcast", BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    final DataStream<Long> srcOne = env.generateSequence(0L, 5L).assignTimestampsAndWatermarks(new CustomWmEmitter<Long>() {

        @Override
        public long extractTimestamp(Long element, long previousElementTimestamp) {
            return element;
        }
    }).keyBy((KeySelector<Long, Long>) value -> value);
    final DataStream<String> srcTwo = env.fromElements("Test:0", "Test:1", "Test:2", "Test:3", "Test:4", "Test:5").assignTimestampsAndWatermarks(new CustomWmEmitter<String>() {

        @Override
        public long extractTimestamp(String element, long previousElementTimestamp) {
            return Long.parseLong(element.split(":")[1]);
        }
    });
    BroadcastStream<String> broadcast = srcTwo.broadcast(descriptor);
    BroadcastConnectedStream<Long, String> bcStream = srcOne.connect(broadcast);
    expectedException.expect(IllegalArgumentException.class);
    bcStream.process(new BroadcastProcessFunction<Long, String, String>() {

        @Override
        public void processBroadcastElement(String value, Context ctx, Collector<String> out) throws Exception {
        // do nothing
        }

        @Override
        public void processElement(Long value, ReadOnlyContext ctx, Collector<String> out) throws Exception {
        // do nothing
        }
    });
}
Also used : Tuple1(org.apache.flink.api.java.tuple.Tuple1) Tuple2(org.apache.flink.api.java.tuple.Tuple2) BasicArrayTypeInfo(org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo) PurgingTrigger(org.apache.flink.streaming.api.windowing.triggers.PurgingTrigger) BroadcastPartitioner(org.apache.flink.streaming.runtime.partitioner.BroadcastPartitioner) TupleTypeInfo(org.apache.flink.api.java.typeutils.TupleTypeInfo) CoFlatMapFunction(org.apache.flink.streaming.api.functions.co.CoFlatMapFunction) KeyedBroadcastProcessFunction(org.apache.flink.streaming.api.functions.co.KeyedBroadcastProcessFunction) MapFunction(org.apache.flink.api.common.functions.MapFunction) BasicTypeInfo(org.apache.flink.api.common.typeinfo.BasicTypeInfo) Assert.assertThat(org.junit.Assert.assertThat) AggregateFunction(org.apache.flink.api.common.functions.AggregateFunction) ShufflePartitioner(org.apache.flink.streaming.runtime.partitioner.ShufflePartitioner) InvalidProgramException(org.apache.flink.api.common.InvalidProgramException) ResourceSpec(org.apache.flink.api.common.operators.ResourceSpec) CustomPartitionerWrapper(org.apache.flink.streaming.runtime.partitioner.CustomPartitionerWrapper) Duration(java.time.Duration) StreamGraph(org.apache.flink.streaming.api.graph.StreamGraph) TestLogger(org.apache.flink.util.TestLogger) Function(org.apache.flink.api.common.functions.Function) Assert.fail(org.junit.Assert.fail) AssignerWithPunctuatedWatermarks(org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks) TypeInformation(org.apache.flink.api.common.typeinfo.TypeInformation) Method(java.lang.reflect.Method) GlobalWindow(org.apache.flink.streaming.api.windowing.windows.GlobalWindow) StringStartsWith(org.hamcrest.core.StringStartsWith) CoMapFunction(org.apache.flink.streaming.api.functions.co.CoMapFunction) KeySelector(org.apache.flink.api.java.functions.KeySelector) ForwardPartitioner(org.apache.flink.streaming.runtime.partitioner.ForwardPartitioner) SinkFunction(org.apache.flink.streaming.api.functions.sink.SinkFunction) WatermarkStrategy(org.apache.flink.api.common.eventtime.WatermarkStrategy) ConnectedStreams(org.apache.flink.streaming.api.datastream.ConnectedStreams) KeyedStream(org.apache.flink.streaming.api.datastream.KeyedStream) FlatMapFunction(org.apache.flink.api.common.functions.FlatMapFunction) PrimitiveArrayTypeInfo(org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo) GlobalPartitioner(org.apache.flink.streaming.runtime.partitioner.GlobalPartitioner) FilterFunction(org.apache.flink.api.common.functions.FilterFunction) List(java.util.List) TypeExtractor(org.apache.flink.api.java.typeutils.TypeExtractor) BroadcastConnectedStream(org.apache.flink.streaming.api.datastream.BroadcastConnectedStream) TumblingEventTimeWindows(org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows) Assert.assertFalse(org.junit.Assert.assertFalse) AllWindowFunction(org.apache.flink.streaming.api.functions.windowing.AllWindowFunction) StreamExecutionEnvironment(org.apache.flink.streaming.api.environment.StreamExecutionEnvironment) CountTrigger(org.apache.flink.streaming.api.windowing.triggers.CountTrigger) BroadcastStream(org.apache.flink.streaming.api.datastream.BroadcastStream) AbstractUdfStreamOperator(org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator) CoreMatchers.equalTo(org.hamcrest.CoreMatchers.equalTo) Watermark(org.apache.flink.streaming.api.watermark.Watermark) MapStateDescriptor(org.apache.flink.api.common.state.MapStateDescriptor) ObjectArrayTypeInfo(org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo) DataStreamSource(org.apache.flink.streaming.api.datastream.DataStreamSource) KeyedProcessFunction(org.apache.flink.streaming.api.functions.KeyedProcessFunction) Partitioner(org.apache.flink.api.common.functions.Partitioner) KeyedProcessOperator(org.apache.flink.streaming.api.operators.KeyedProcessOperator) StreamPartitioner(org.apache.flink.streaming.runtime.partitioner.StreamPartitioner) Collector(org.apache.flink.util.Collector) ProcessOperator(org.apache.flink.streaming.api.operators.ProcessOperator) ProcessFunction(org.apache.flink.streaming.api.functions.ProcessFunction) ReduceFunction(org.apache.flink.api.common.functions.ReduceFunction) ExpectedException(org.junit.rules.ExpectedException) Nullable(javax.annotation.Nullable) Types(org.apache.flink.api.common.typeinfo.Types) DataStreamSink(org.apache.flink.streaming.api.datastream.DataStreamSink) RebalancePartitioner(org.apache.flink.streaming.runtime.partitioner.RebalancePartitioner) StreamEdge(org.apache.flink.streaming.api.graph.StreamEdge) Time(org.apache.flink.streaming.api.windowing.time.Time) GenericTypeInfo(org.apache.flink.api.java.typeutils.GenericTypeInfo) DiscardingSink(org.apache.flink.streaming.api.functions.sink.DiscardingSink) Assert.assertNotNull(org.junit.Assert.assertNotNull) SingleOutputStreamOperator(org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) LegacyKeyedProcessOperator(org.apache.flink.streaming.api.operators.LegacyKeyedProcessOperator) DataStream(org.apache.flink.streaming.api.datastream.DataStream) StreamOperator(org.apache.flink.streaming.api.operators.StreamOperator) KeyGroupStreamPartitioner(org.apache.flink.streaming.runtime.partitioner.KeyGroupStreamPartitioner) Rule(org.junit.Rule) BroadcastProcessFunction(org.apache.flink.streaming.api.functions.co.BroadcastProcessFunction) EnumTypeInfo(org.apache.flink.api.java.typeutils.EnumTypeInfo) Assert(org.junit.Assert) GlobalWindows(org.apache.flink.streaming.api.windowing.assigners.GlobalWindows) Assert.assertEquals(org.junit.Assert.assertEquals) MapStateDescriptor(org.apache.flink.api.common.state.MapStateDescriptor) InvalidProgramException(org.apache.flink.api.common.InvalidProgramException) ExpectedException(org.junit.rules.ExpectedException) StreamExecutionEnvironment(org.apache.flink.streaming.api.environment.StreamExecutionEnvironment) Test(org.junit.Test)

Aggregations

Method (java.lang.reflect.Method)1 Duration (java.time.Duration)1 List (java.util.List)1 Nullable (javax.annotation.Nullable)1 InvalidProgramException (org.apache.flink.api.common.InvalidProgramException)1 WatermarkStrategy (org.apache.flink.api.common.eventtime.WatermarkStrategy)1 AggregateFunction (org.apache.flink.api.common.functions.AggregateFunction)1 FilterFunction (org.apache.flink.api.common.functions.FilterFunction)1 FlatMapFunction (org.apache.flink.api.common.functions.FlatMapFunction)1 Function (org.apache.flink.api.common.functions.Function)1 MapFunction (org.apache.flink.api.common.functions.MapFunction)1 Partitioner (org.apache.flink.api.common.functions.Partitioner)1 ReduceFunction (org.apache.flink.api.common.functions.ReduceFunction)1 ResourceSpec (org.apache.flink.api.common.operators.ResourceSpec)1 MapStateDescriptor (org.apache.flink.api.common.state.MapStateDescriptor)1 BasicArrayTypeInfo (org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo)1 BasicTypeInfo (org.apache.flink.api.common.typeinfo.BasicTypeInfo)1 PrimitiveArrayTypeInfo (org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo)1 TypeInformation (org.apache.flink.api.common.typeinfo.TypeInformation)1 Types (org.apache.flink.api.common.typeinfo.Types)1