Examples with AssignerWithPeriodicWatermarks org.apache.flink.streaming.api.functions.AssignerWithPeriodicWatermarks used on opensource projects

Example 1 with AssignerWithPeriodicWatermarks

use of org.apache.flink.streaming.api.functions.AssignerWithPeriodicWatermarks in project flink by apache.

the class TimestampITCase method testTimestampExtractorWithLongMaxWatermarkFromSource2.

/**
	 * This test verifies that the timestamp extractor forwards Long.MAX_VALUE watermarks.
	 * 
	 * Same test as before, but using a different timestamp extractor
	 */
@Test
public void testTimestampExtractorWithLongMaxWatermarkFromSource2() throws Exception {
    final int NUM_ELEMENTS = 10;
    StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
    env.getConfig().setAutoWatermarkInterval(10);
    env.setParallelism(2);
    env.getConfig().disableSysoutLogging();
    DataStream<Integer> source1 = env.addSource(new SourceFunction<Integer>() {

        @Override
        public void run(SourceContext<Integer> ctx) throws Exception {
            int index = 1;
            while (index <= NUM_ELEMENTS) {
                ctx.collectWithTimestamp(index, index);
                ctx.collectWithTimestamp(index - 1, index - 1);
                index++;
                ctx.emitWatermark(new Watermark(index - 2));
            }
            // emit the final Long.MAX_VALUE watermark, do it twice and verify that
            // we only see one in the result
            ctx.emitWatermark(new Watermark(Long.MAX_VALUE));
            ctx.emitWatermark(new Watermark(Long.MAX_VALUE));
        }

        @Override
        public void cancel() {
        }
    });
    source1.assignTimestampsAndWatermarks(new AssignerWithPeriodicWatermarks<Integer>() {

        @Override
        public long extractTimestamp(Integer element, long currentTimestamp) {
            return element;
        }

        @Override
        public Watermark getCurrentWatermark() {
            return null;
        }
    }).transform("Watermark Check", BasicTypeInfo.INT_TYPE_INFO, new CustomOperator(true));
    env.execute();
    Assert.assertTrue(CustomOperator.finalWatermarks[0].size() == 1);
    Assert.assertTrue(CustomOperator.finalWatermarks[0].get(0).getTimestamp() == Long.MAX_VALUE);
}

Example 2 with AssignerWithPeriodicWatermarks

use of org.apache.flink.streaming.api.functions.AssignerWithPeriodicWatermarks in project flink by apache.

the class AbstractFetcherTest method testSkipCorruptedRecordWithPeriodicWatermarks.

@Test
public void testSkipCorruptedRecordWithPeriodicWatermarks() throws Exception {
    final String testTopic = "test topic name";
    Map<KafkaTopicPartition, Long> originalPartitions = new HashMap<>();
    originalPartitions.put(new KafkaTopicPartition(testTopic, 1), KafkaTopicPartitionStateSentinel.LATEST_OFFSET);
    TestSourceContext<Long> sourceContext = new TestSourceContext<>();
    TestProcessingTimeService processingTimeProvider = new TestProcessingTimeService();
    TestFetcher<Long> fetcher = new TestFetcher<>(sourceContext, originalPartitions, new SerializedValue<AssignerWithPeriodicWatermarks<Long>>(new PeriodicTestExtractor()), /* periodic watermark assigner */
    null, /* punctuated watermark assigner */
    processingTimeProvider, 10);
    final KafkaTopicPartitionState<Object> partitionStateHolder = fetcher.subscribedPartitionStates()[0];
    // elements generate a watermark if the timestamp is a multiple of three
    fetcher.emitRecord(1L, partitionStateHolder, 1L);
    fetcher.emitRecord(2L, partitionStateHolder, 2L);
    fetcher.emitRecord(3L, partitionStateHolder, 3L);
    assertEquals(3L, sourceContext.getLatestElement().getValue().longValue());
    assertEquals(3L, sourceContext.getLatestElement().getTimestamp());
    assertEquals(3L, partitionStateHolder.getOffset());
    // advance timer for watermark emitting
    processingTimeProvider.setCurrentTime(10L);
    assertTrue(sourceContext.hasWatermark());
    assertEquals(3L, sourceContext.getLatestWatermark().getTimestamp());
    // emit null record
    fetcher.emitRecord(null, partitionStateHolder, 4L);
    // no elements should have been collected
    assertEquals(3L, sourceContext.getLatestElement().getValue().longValue());
    assertEquals(3L, sourceContext.getLatestElement().getTimestamp());
    // the offset in state still should have advanced
    assertEquals(4L, partitionStateHolder.getOffset());
    // no watermarks should be collected
    processingTimeProvider.setCurrentTime(20L);
    assertFalse(sourceContext.hasWatermark());
}

Example 3 with AssignerWithPeriodicWatermarks

use of org.apache.flink.streaming.api.functions.AssignerWithPeriodicWatermarks in project flink by apache.

the class AbstractFetcherTest method testPeriodicWatermarks.

@Test
public void testPeriodicWatermarks() throws Exception {
    final String testTopic = "test topic name";
    Map<KafkaTopicPartition, Long> originalPartitions = new HashMap<>();
    originalPartitions.put(new KafkaTopicPartition(testTopic, 7), KafkaTopicPartitionStateSentinel.LATEST_OFFSET);
    originalPartitions.put(new KafkaTopicPartition(testTopic, 13), KafkaTopicPartitionStateSentinel.LATEST_OFFSET);
    originalPartitions.put(new KafkaTopicPartition(testTopic, 21), KafkaTopicPartitionStateSentinel.LATEST_OFFSET);
    TestSourceContext<Long> sourceContext = new TestSourceContext<>();
    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
    TestFetcher<Long> fetcher = new TestFetcher<>(sourceContext, originalPartitions, new SerializedValue<AssignerWithPeriodicWatermarks<Long>>(new PeriodicTestExtractor()), null, /* punctuated watermarks assigner*/
    processingTimeService, 10);
    final KafkaTopicPartitionState<Object> part1 = fetcher.subscribedPartitionStates()[0];
    final KafkaTopicPartitionState<Object> part2 = fetcher.subscribedPartitionStates()[1];
    final KafkaTopicPartitionState<Object> part3 = fetcher.subscribedPartitionStates()[2];
    // elements generate a watermark if the timestamp is a multiple of three
    // elements for partition 1
    fetcher.emitRecord(1L, part1, 1L);
    fetcher.emitRecord(2L, part1, 2L);
    fetcher.emitRecord(3L, part1, 3L);
    assertEquals(3L, sourceContext.getLatestElement().getValue().longValue());
    assertEquals(3L, sourceContext.getLatestElement().getTimestamp());
    // elements for partition 2
    fetcher.emitRecord(12L, part2, 1L);
    assertEquals(12L, sourceContext.getLatestElement().getValue().longValue());
    assertEquals(12L, sourceContext.getLatestElement().getTimestamp());
    // elements for partition 3
    fetcher.emitRecord(101L, part3, 1L);
    fetcher.emitRecord(102L, part3, 2L);
    assertEquals(102L, sourceContext.getLatestElement().getValue().longValue());
    assertEquals(102L, sourceContext.getLatestElement().getTimestamp());
    processingTimeService.setCurrentTime(10);
    // now, we should have a watermark (this blocks until the periodic thread emitted the watermark)
    assertEquals(3L, sourceContext.getLatestWatermark().getTimestamp());
    // advance partition 3
    fetcher.emitRecord(1003L, part3, 3L);
    fetcher.emitRecord(1004L, part3, 4L);
    fetcher.emitRecord(1005L, part3, 5L);
    assertEquals(1005L, sourceContext.getLatestElement().getValue().longValue());
    assertEquals(1005L, sourceContext.getLatestElement().getTimestamp());
    // advance partition 1 beyond partition 2 - this bumps the watermark
    fetcher.emitRecord(30L, part1, 4L);
    assertEquals(30L, sourceContext.getLatestElement().getValue().longValue());
    assertEquals(30L, sourceContext.getLatestElement().getTimestamp());
    processingTimeService.setCurrentTime(20);
    // this blocks until the periodic thread emitted the watermark
    assertEquals(12L, sourceContext.getLatestWatermark().getTimestamp());
    // advance partition 2 again - this bumps the watermark
    fetcher.emitRecord(13L, part2, 2L);
    fetcher.emitRecord(14L, part2, 3L);
    fetcher.emitRecord(15L, part2, 3L);
    processingTimeService.setCurrentTime(30);
    // this blocks until the periodic thread emitted the watermark
    long watermarkTs = sourceContext.getLatestWatermark().getTimestamp();
    assertTrue(watermarkTs >= 13L && watermarkTs <= 15L);
}