Examples with TestProcessingTimeService org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService used on opensource projects

Example 6 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class AbstractFetcherTest method testSkipCorruptedRecordWithPunctuatedWatermarks.

@Test
public void testSkipCorruptedRecordWithPunctuatedWatermarks() throws Exception {
    final String testTopic = "test topic name";
    Map<KafkaTopicPartition, Long> originalPartitions = new HashMap<>();
    originalPartitions.put(new KafkaTopicPartition(testTopic, 1), KafkaTopicPartitionStateSentinel.LATEST_OFFSET);
    TestSourceContext<Long> sourceContext = new TestSourceContext<>();
    TestProcessingTimeService processingTimeProvider = new TestProcessingTimeService();
    TestFetcher<Long> fetcher = new TestFetcher<>(sourceContext, originalPartitions, null, /* periodic watermark assigner */
    new SerializedValue<AssignerWithPunctuatedWatermarks<Long>>(new PunctuatedTestExtractor()), /* punctuated watermark assigner */
    processingTimeProvider, 0);
    final KafkaTopicPartitionState<Object> partitionStateHolder = fetcher.subscribedPartitionStates()[0];
    // elements generate a watermark if the timestamp is a multiple of three
    fetcher.emitRecord(1L, partitionStateHolder, 1L);
    fetcher.emitRecord(2L, partitionStateHolder, 2L);
    fetcher.emitRecord(3L, partitionStateHolder, 3L);
    assertEquals(3L, sourceContext.getLatestElement().getValue().longValue());
    assertEquals(3L, sourceContext.getLatestElement().getTimestamp());
    assertTrue(sourceContext.hasWatermark());
    assertEquals(3L, sourceContext.getLatestWatermark().getTimestamp());
    assertEquals(3L, partitionStateHolder.getOffset());
    // emit null record
    fetcher.emitRecord(null, partitionStateHolder, 4L);
    // no elements or watermarks should have been collected
    assertEquals(3L, sourceContext.getLatestElement().getValue().longValue());
    assertEquals(3L, sourceContext.getLatestElement().getTimestamp());
    assertFalse(sourceContext.hasWatermark());
    // the offset in state still should have advanced
    assertEquals(4L, partitionStateHolder.getOffset());
}

Example 7 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class AbstractFetcherTest method testSkipCorruptedRecordWithPeriodicWatermarks.

@Test
public void testSkipCorruptedRecordWithPeriodicWatermarks() throws Exception {
    final String testTopic = "test topic name";
    Map<KafkaTopicPartition, Long> originalPartitions = new HashMap<>();
    originalPartitions.put(new KafkaTopicPartition(testTopic, 1), KafkaTopicPartitionStateSentinel.LATEST_OFFSET);
    TestSourceContext<Long> sourceContext = new TestSourceContext<>();
    TestProcessingTimeService processingTimeProvider = new TestProcessingTimeService();
    TestFetcher<Long> fetcher = new TestFetcher<>(sourceContext, originalPartitions, new SerializedValue<AssignerWithPeriodicWatermarks<Long>>(new PeriodicTestExtractor()), /* periodic watermark assigner */
    null, /* punctuated watermark assigner */
    processingTimeProvider, 10);
    final KafkaTopicPartitionState<Object> partitionStateHolder = fetcher.subscribedPartitionStates()[0];
    // elements generate a watermark if the timestamp is a multiple of three
    fetcher.emitRecord(1L, partitionStateHolder, 1L);
    fetcher.emitRecord(2L, partitionStateHolder, 2L);
    fetcher.emitRecord(3L, partitionStateHolder, 3L);
    assertEquals(3L, sourceContext.getLatestElement().getValue().longValue());
    assertEquals(3L, sourceContext.getLatestElement().getTimestamp());
    assertEquals(3L, partitionStateHolder.getOffset());
    // advance timer for watermark emitting
    processingTimeProvider.setCurrentTime(10L);
    assertTrue(sourceContext.hasWatermark());
    assertEquals(3L, sourceContext.getLatestWatermark().getTimestamp());
    // emit null record
    fetcher.emitRecord(null, partitionStateHolder, 4L);
    // no elements should have been collected
    assertEquals(3L, sourceContext.getLatestElement().getValue().longValue());
    assertEquals(3L, sourceContext.getLatestElement().getTimestamp());
    // the offset in state still should have advanced
    assertEquals(4L, partitionStateHolder.getOffset());
    // no watermarks should be collected
    processingTimeProvider.setCurrentTime(20L);
    assertFalse(sourceContext.hasWatermark());
}

Example 8 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class HeapInternalTimerServiceTest method testRegisteringProcessingTimeTimerInOnProcessingTimeDoesNotLeakPhysicalTimers.

/**
	 */
@Test
public void testRegisteringProcessingTimeTimerInOnProcessingTimeDoesNotLeakPhysicalTimers() throws Exception {
    @SuppressWarnings("unchecked") Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class);
    TestKeyContext keyContext = new TestKeyContext();
    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
    final HeapInternalTimerService<Integer, String> timerService = createTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, maxParallelism);
    int key = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
    keyContext.setCurrentKey(key);
    timerService.registerProcessingTimeTimer("ciao", 10);
    assertEquals(1, timerService.numProcessingTimeTimers());
    assertEquals(1, processingTimeService.getNumActiveTimers());
    assertThat(processingTimeService.getActiveTimerTimestamps(), containsInAnyOrder(10L));
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Exception {
            timerService.registerProcessingTimeTimer("ciao", 20);
            return null;
        }
    }).when(mockTriggerable).onProcessingTime(anyInternalTimer());
    processingTimeService.setCurrentTime(10);
    assertEquals(1, processingTimeService.getNumActiveTimers());
    assertThat(processingTimeService.getActiveTimerTimestamps(), containsInAnyOrder(20L));
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Exception {
            timerService.registerProcessingTimeTimer("ciao", 30);
            return null;
        }
    }).when(mockTriggerable).onProcessingTime(anyInternalTimer());
    processingTimeService.setCurrentTime(20);
    assertEquals(1, timerService.numProcessingTimeTimers());
    assertEquals(1, processingTimeService.getNumActiveTimers());
    assertThat(processingTimeService.getActiveTimerTimestamps(), containsInAnyOrder(30L));
}

Example 9 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class HeapInternalTimerServiceTest method testOnlySetsOnePhysicalProcessingTimeTimer.

/**
	 * Verify that we only ever have one processing-time task registered at the
	 * {@link ProcessingTimeService}.
	 */
@Test
public void testOnlySetsOnePhysicalProcessingTimeTimer() throws Exception {
    @SuppressWarnings("unchecked") Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class);
    TestKeyContext keyContext = new TestKeyContext();
    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
    HeapInternalTimerService<Integer, String> timerService = createTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, maxParallelism);
    int key = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
    keyContext.setCurrentKey(key);
    timerService.registerProcessingTimeTimer("ciao", 10);
    timerService.registerProcessingTimeTimer("ciao", 20);
    timerService.registerProcessingTimeTimer("ciao", 30);
    timerService.registerProcessingTimeTimer("hello", 10);
    timerService.registerProcessingTimeTimer("hello", 20);
    assertEquals(5, timerService.numProcessingTimeTimers());
    assertEquals(2, timerService.numProcessingTimeTimers("hello"));
    assertEquals(3, timerService.numProcessingTimeTimers("ciao"));
    assertEquals(1, processingTimeService.getNumActiveTimers());
    assertThat(processingTimeService.getActiveTimerTimestamps(), containsInAnyOrder(10L));
    processingTimeService.setCurrentTime(10);
    assertEquals(3, timerService.numProcessingTimeTimers());
    assertEquals(1, timerService.numProcessingTimeTimers("hello"));
    assertEquals(2, timerService.numProcessingTimeTimers("ciao"));
    assertEquals(1, processingTimeService.getNumActiveTimers());
    assertThat(processingTimeService.getActiveTimerTimestamps(), containsInAnyOrder(20L));
    processingTimeService.setCurrentTime(20);
    assertEquals(1, timerService.numProcessingTimeTimers());
    assertEquals(0, timerService.numProcessingTimeTimers("hello"));
    assertEquals(1, timerService.numProcessingTimeTimers("ciao"));
    assertEquals(1, processingTimeService.getNumActiveTimers());
    assertThat(processingTimeService.getActiveTimerTimestamps(), containsInAnyOrder(30L));
    processingTimeService.setCurrentTime(30);
    assertEquals(0, timerService.numProcessingTimeTimers());
    assertEquals(0, processingTimeService.getNumActiveTimers());
    timerService.registerProcessingTimeTimer("ciao", 40);
    assertEquals(1, processingTimeService.getNumActiveTimers());
}

Example 10 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class HeapInternalTimerServiceTest method testKeyGroupStartIndexSetting.

@Test
public void testKeyGroupStartIndexSetting() {
    int startKeyGroupIdx = 7;
    int endKeyGroupIdx = 21;
    KeyGroupsList testKeyGroupList = new KeyGroupRange(startKeyGroupIdx, endKeyGroupIdx);
    TestKeyContext keyContext = new TestKeyContext();
    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
    HeapInternalTimerService<Integer, String> service = new HeapInternalTimerService<>(testKeyGroupList.getNumberOfKeyGroups(), testKeyGroupList, keyContext, processingTimeService);
    Assert.assertEquals(startKeyGroupIdx, service.getLocalKeyGroupRangeStartIdx());
}