Search in sources :

Example 1 with PurgingTrigger

use of org.apache.flink.streaming.api.windowing.triggers.PurgingTrigger in project flink by apache.

the class DataStreamTest method testWindowOperatorDescription.

/**
 * Tests that verifies window operator has different name and description.
 */
@Test
public void testWindowOperatorDescription() {
    // global window
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<Long> dataStream1 = env.generateSequence(0, 0).windowAll(GlobalWindows.create()).trigger(PurgingTrigger.of(CountTrigger.of(10))).reduce(new ReduceFunction<Long>() {

        private static final long serialVersionUID = 1L;

        @Override
        public Long reduce(Long value1, Long value2) throws Exception {
            return null;
        }
    });
    // name is simplified
    assertEquals("GlobalWindows", dataStream1.getTransformation().getName());
    // description contains detail of function:
    // TriggerWindow(GlobalWindows(), ReducingStateDescriptor{name=window-contents,
    // defaultValue=null,
    // serializer=org.apache.flink.api.common.typeutils.base.LongSerializer@6af9fcb2},
    // PurgingTrigger(CountTrigger(10)), AllWindowedStream.reduce(AllWindowedStream.java:229))
    assertTrue(dataStream1.getTransformation().getDescription().contains("PurgingTrigger"));
    // keyed window
    DataStream<Long> dataStream2 = env.generateSequence(0, 0).keyBy(value -> value).window(TumblingEventTimeWindows.of(Time.milliseconds(1000))).trigger(PurgingTrigger.of(CountTrigger.of(10))).reduce(new ReduceFunction<Long>() {

        private static final long serialVersionUID = 1L;

        @Override
        public Long reduce(Long value1, Long value2) throws Exception {
            return null;
        }
    });
    // name is simplified
    assertEquals("TumblingEventTimeWindows", dataStream2.getTransformation().getName());
    // description contains detail of function:
    // Window(TumblingEventTimeWindows(1000), PurgingTrigger, ReduceFunction$36,
    // PassThroughWindowFunction)
    assertTrue(dataStream2.getTransformation().getDescription().contains("PurgingTrigger"));
}
Also used : Tuple1(org.apache.flink.api.java.tuple.Tuple1) Tuple2(org.apache.flink.api.java.tuple.Tuple2) BasicArrayTypeInfo(org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo) PurgingTrigger(org.apache.flink.streaming.api.windowing.triggers.PurgingTrigger) BroadcastPartitioner(org.apache.flink.streaming.runtime.partitioner.BroadcastPartitioner) TupleTypeInfo(org.apache.flink.api.java.typeutils.TupleTypeInfo) CoFlatMapFunction(org.apache.flink.streaming.api.functions.co.CoFlatMapFunction) KeyedBroadcastProcessFunction(org.apache.flink.streaming.api.functions.co.KeyedBroadcastProcessFunction) MapFunction(org.apache.flink.api.common.functions.MapFunction) BasicTypeInfo(org.apache.flink.api.common.typeinfo.BasicTypeInfo) Assert.assertThat(org.junit.Assert.assertThat) AggregateFunction(org.apache.flink.api.common.functions.AggregateFunction) ShufflePartitioner(org.apache.flink.streaming.runtime.partitioner.ShufflePartitioner) InvalidProgramException(org.apache.flink.api.common.InvalidProgramException) ResourceSpec(org.apache.flink.api.common.operators.ResourceSpec) CustomPartitionerWrapper(org.apache.flink.streaming.runtime.partitioner.CustomPartitionerWrapper) Duration(java.time.Duration) StreamGraph(org.apache.flink.streaming.api.graph.StreamGraph) TestLogger(org.apache.flink.util.TestLogger) Function(org.apache.flink.api.common.functions.Function) Assert.fail(org.junit.Assert.fail) AssignerWithPunctuatedWatermarks(org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks) TypeInformation(org.apache.flink.api.common.typeinfo.TypeInformation) Method(java.lang.reflect.Method) GlobalWindow(org.apache.flink.streaming.api.windowing.windows.GlobalWindow) StringStartsWith(org.hamcrest.core.StringStartsWith) CoMapFunction(org.apache.flink.streaming.api.functions.co.CoMapFunction) KeySelector(org.apache.flink.api.java.functions.KeySelector) ForwardPartitioner(org.apache.flink.streaming.runtime.partitioner.ForwardPartitioner) SinkFunction(org.apache.flink.streaming.api.functions.sink.SinkFunction) WatermarkStrategy(org.apache.flink.api.common.eventtime.WatermarkStrategy) ConnectedStreams(org.apache.flink.streaming.api.datastream.ConnectedStreams) KeyedStream(org.apache.flink.streaming.api.datastream.KeyedStream) FlatMapFunction(org.apache.flink.api.common.functions.FlatMapFunction) PrimitiveArrayTypeInfo(org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo) GlobalPartitioner(org.apache.flink.streaming.runtime.partitioner.GlobalPartitioner) FilterFunction(org.apache.flink.api.common.functions.FilterFunction) List(java.util.List) TypeExtractor(org.apache.flink.api.java.typeutils.TypeExtractor) BroadcastConnectedStream(org.apache.flink.streaming.api.datastream.BroadcastConnectedStream) TumblingEventTimeWindows(org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows) Assert.assertFalse(org.junit.Assert.assertFalse) AllWindowFunction(org.apache.flink.streaming.api.functions.windowing.AllWindowFunction) StreamExecutionEnvironment(org.apache.flink.streaming.api.environment.StreamExecutionEnvironment) CountTrigger(org.apache.flink.streaming.api.windowing.triggers.CountTrigger) BroadcastStream(org.apache.flink.streaming.api.datastream.BroadcastStream) AbstractUdfStreamOperator(org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator) CoreMatchers.equalTo(org.hamcrest.CoreMatchers.equalTo) Watermark(org.apache.flink.streaming.api.watermark.Watermark) MapStateDescriptor(org.apache.flink.api.common.state.MapStateDescriptor) ObjectArrayTypeInfo(org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo) DataStreamSource(org.apache.flink.streaming.api.datastream.DataStreamSource) KeyedProcessFunction(org.apache.flink.streaming.api.functions.KeyedProcessFunction) Partitioner(org.apache.flink.api.common.functions.Partitioner) KeyedProcessOperator(org.apache.flink.streaming.api.operators.KeyedProcessOperator) StreamPartitioner(org.apache.flink.streaming.runtime.partitioner.StreamPartitioner) Collector(org.apache.flink.util.Collector) ProcessOperator(org.apache.flink.streaming.api.operators.ProcessOperator) ProcessFunction(org.apache.flink.streaming.api.functions.ProcessFunction) ReduceFunction(org.apache.flink.api.common.functions.ReduceFunction) ExpectedException(org.junit.rules.ExpectedException) Nullable(javax.annotation.Nullable) Types(org.apache.flink.api.common.typeinfo.Types) DataStreamSink(org.apache.flink.streaming.api.datastream.DataStreamSink) RebalancePartitioner(org.apache.flink.streaming.runtime.partitioner.RebalancePartitioner) StreamEdge(org.apache.flink.streaming.api.graph.StreamEdge) Time(org.apache.flink.streaming.api.windowing.time.Time) GenericTypeInfo(org.apache.flink.api.java.typeutils.GenericTypeInfo) DiscardingSink(org.apache.flink.streaming.api.functions.sink.DiscardingSink) Assert.assertNotNull(org.junit.Assert.assertNotNull) SingleOutputStreamOperator(org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) LegacyKeyedProcessOperator(org.apache.flink.streaming.api.operators.LegacyKeyedProcessOperator) DataStream(org.apache.flink.streaming.api.datastream.DataStream) StreamOperator(org.apache.flink.streaming.api.operators.StreamOperator) KeyGroupStreamPartitioner(org.apache.flink.streaming.runtime.partitioner.KeyGroupStreamPartitioner) Rule(org.junit.Rule) BroadcastProcessFunction(org.apache.flink.streaming.api.functions.co.BroadcastProcessFunction) EnumTypeInfo(org.apache.flink.api.java.typeutils.EnumTypeInfo) Assert(org.junit.Assert) GlobalWindows(org.apache.flink.streaming.api.windowing.assigners.GlobalWindows) Assert.assertEquals(org.junit.Assert.assertEquals) StreamExecutionEnvironment(org.apache.flink.streaming.api.environment.StreamExecutionEnvironment) InvalidProgramException(org.apache.flink.api.common.InvalidProgramException) ExpectedException(org.junit.rules.ExpectedException) Test(org.junit.Test)

Aggregations

Method (java.lang.reflect.Method)1 Duration (java.time.Duration)1 List (java.util.List)1 Nullable (javax.annotation.Nullable)1 InvalidProgramException (org.apache.flink.api.common.InvalidProgramException)1 WatermarkStrategy (org.apache.flink.api.common.eventtime.WatermarkStrategy)1 AggregateFunction (org.apache.flink.api.common.functions.AggregateFunction)1 FilterFunction (org.apache.flink.api.common.functions.FilterFunction)1 FlatMapFunction (org.apache.flink.api.common.functions.FlatMapFunction)1 Function (org.apache.flink.api.common.functions.Function)1 MapFunction (org.apache.flink.api.common.functions.MapFunction)1 Partitioner (org.apache.flink.api.common.functions.Partitioner)1 ReduceFunction (org.apache.flink.api.common.functions.ReduceFunction)1 ResourceSpec (org.apache.flink.api.common.operators.ResourceSpec)1 MapStateDescriptor (org.apache.flink.api.common.state.MapStateDescriptor)1 BasicArrayTypeInfo (org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo)1 BasicTypeInfo (org.apache.flink.api.common.typeinfo.BasicTypeInfo)1 PrimitiveArrayTypeInfo (org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo)1 TypeInformation (org.apache.flink.api.common.typeinfo.TypeInformation)1 Types (org.apache.flink.api.common.typeinfo.Types)1