Examples with TestProcessingTimeService org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService used on opensource projects

Example 41 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class AbstractFetcherTest method testPeriodicWatermarks.

@Test
public void testPeriodicWatermarks() throws Exception {
    final String testTopic = "test topic name";
    Map<KafkaTopicPartition, Long> originalPartitions = new HashMap<>();
    originalPartitions.put(new KafkaTopicPartition(testTopic, 7), KafkaTopicPartitionStateSentinel.LATEST_OFFSET);
    originalPartitions.put(new KafkaTopicPartition(testTopic, 13), KafkaTopicPartitionStateSentinel.LATEST_OFFSET);
    originalPartitions.put(new KafkaTopicPartition(testTopic, 21), KafkaTopicPartitionStateSentinel.LATEST_OFFSET);
    TestSourceContext<Long> sourceContext = new TestSourceContext<>();
    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
    TestFetcher<Long> fetcher = new TestFetcher<>(sourceContext, originalPartitions, new SerializedValue<AssignerWithPeriodicWatermarks<Long>>(new PeriodicTestExtractor()), null, /* punctuated watermarks assigner*/
    processingTimeService, 10);
    final KafkaTopicPartitionState<Object> part1 = fetcher.subscribedPartitionStates()[0];
    final KafkaTopicPartitionState<Object> part2 = fetcher.subscribedPartitionStates()[1];
    final KafkaTopicPartitionState<Object> part3 = fetcher.subscribedPartitionStates()[2];
    // elements generate a watermark if the timestamp is a multiple of three
    // elements for partition 1
    fetcher.emitRecord(1L, part1, 1L);
    fetcher.emitRecord(2L, part1, 2L);
    fetcher.emitRecord(3L, part1, 3L);
    assertEquals(3L, sourceContext.getLatestElement().getValue().longValue());
    assertEquals(3L, sourceContext.getLatestElement().getTimestamp());
    // elements for partition 2
    fetcher.emitRecord(12L, part2, 1L);
    assertEquals(12L, sourceContext.getLatestElement().getValue().longValue());
    assertEquals(12L, sourceContext.getLatestElement().getTimestamp());
    // elements for partition 3
    fetcher.emitRecord(101L, part3, 1L);
    fetcher.emitRecord(102L, part3, 2L);
    assertEquals(102L, sourceContext.getLatestElement().getValue().longValue());
    assertEquals(102L, sourceContext.getLatestElement().getTimestamp());
    processingTimeService.setCurrentTime(10);
    // now, we should have a watermark (this blocks until the periodic thread emitted the watermark)
    assertEquals(3L, sourceContext.getLatestWatermark().getTimestamp());
    // advance partition 3
    fetcher.emitRecord(1003L, part3, 3L);
    fetcher.emitRecord(1004L, part3, 4L);
    fetcher.emitRecord(1005L, part3, 5L);
    assertEquals(1005L, sourceContext.getLatestElement().getValue().longValue());
    assertEquals(1005L, sourceContext.getLatestElement().getTimestamp());
    // advance partition 1 beyond partition 2 - this bumps the watermark
    fetcher.emitRecord(30L, part1, 4L);
    assertEquals(30L, sourceContext.getLatestElement().getValue().longValue());
    assertEquals(30L, sourceContext.getLatestElement().getTimestamp());
    processingTimeService.setCurrentTime(20);
    // this blocks until the periodic thread emitted the watermark
    assertEquals(12L, sourceContext.getLatestWatermark().getTimestamp());
    // advance partition 2 again - this bumps the watermark
    fetcher.emitRecord(13L, part2, 2L);
    fetcher.emitRecord(14L, part2, 3L);
    fetcher.emitRecord(15L, part2, 3L);
    processingTimeService.setCurrentTime(30);
    // this blocks until the periodic thread emitted the watermark
    long watermarkTs = sourceContext.getLatestWatermark().getTimestamp();
    assertTrue(watermarkTs >= 13L && watermarkTs <= 15L);
}

Example 42 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class HeapInternalTimerServiceTest method testDeleteEventTimeTimers.

/**
	 * This also verifies that we don't have leakage between keys/namespaces.
	 *
	 * <p>This also verifies that deleted timers don't fire.
	 */
@Test
public void testDeleteEventTimeTimers() throws Exception {
    @SuppressWarnings("unchecked") Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class);
    TestKeyContext keyContext = new TestKeyContext();
    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
    HeapInternalTimerService<Integer, String> timerService = createTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, maxParallelism);
    // get two different keys
    int key1 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
    int key2 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
    while (key2 == key1) {
        key2 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
    }
    keyContext.setCurrentKey(key1);
    timerService.registerEventTimeTimer("ciao", 10);
    timerService.registerEventTimeTimer("hello", 10);
    keyContext.setCurrentKey(key2);
    timerService.registerEventTimeTimer("ciao", 10);
    timerService.registerEventTimeTimer("hello", 10);
    assertEquals(4, timerService.numEventTimeTimers());
    assertEquals(2, timerService.numEventTimeTimers("hello"));
    assertEquals(2, timerService.numEventTimeTimers("ciao"));
    keyContext.setCurrentKey(key1);
    timerService.deleteEventTimeTimer("hello", 10);
    keyContext.setCurrentKey(key2);
    timerService.deleteEventTimeTimer("ciao", 10);
    assertEquals(2, timerService.numEventTimeTimers());
    assertEquals(1, timerService.numEventTimeTimers("hello"));
    assertEquals(1, timerService.numEventTimeTimers("ciao"));
    timerService.advanceWatermark(10);
    verify(mockTriggerable, times(2)).onEventTime(anyInternalTimer());
    verify(mockTriggerable, times(1)).onEventTime(eq(new InternalTimer<>(10, key1, "ciao")));
    verify(mockTriggerable, times(0)).onEventTime(eq(new InternalTimer<>(10, key1, "hello")));
    verify(mockTriggerable, times(0)).onEventTime(eq(new InternalTimer<>(10, key2, "ciao")));
    verify(mockTriggerable, times(1)).onEventTime(eq(new InternalTimer<>(10, key2, "hello")));
    assertEquals(0, timerService.numEventTimeTimers());
}

Example 43 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class AsyncWaitOperatorTest method testClosingWithBlockedEmitter.

/**
	 * Test case for FLINK-5638: Tests that the async wait operator can be closed even if the
	 * emitter is currently waiting on the checkpoint lock (e.g. in the case of two chained async
	 * wait operators where the latter operator's queue is currently full).
	 *
	 * Note that this test does not enforce the exact strict ordering because with the fix it is no
	 * longer possible. However, it provokes the described situation without the fix.
	 */
@Test(timeout = 10000L)
public void testClosingWithBlockedEmitter() throws Exception {
    final Object lock = new Object();
    ArgumentCaptor<Throwable> failureReason = ArgumentCaptor.forClass(Throwable.class);
    Environment environment = mock(Environment.class);
    when(environment.getMetricGroup()).thenReturn(new UnregisteredTaskMetricsGroup());
    when(environment.getTaskManagerInfo()).thenReturn(new TestingTaskManagerRuntimeInfo());
    when(environment.getUserClassLoader()).thenReturn(getClass().getClassLoader());
    when(environment.getTaskInfo()).thenReturn(new TaskInfo("testTask", 1, 0, 1, 0));
    doNothing().when(environment).failExternally(failureReason.capture());
    StreamTask<?, ?> containingTask = mock(StreamTask.class);
    when(containingTask.getEnvironment()).thenReturn(environment);
    when(containingTask.getCheckpointLock()).thenReturn(lock);
    when(containingTask.getProcessingTimeService()).thenReturn(new TestProcessingTimeService());
    StreamConfig streamConfig = mock(StreamConfig.class);
    doReturn(IntSerializer.INSTANCE).when(streamConfig).getTypeSerializerIn1(any(ClassLoader.class));
    final OneShotLatch closingLatch = new OneShotLatch();
    final OneShotLatch outputLatch = new OneShotLatch();
    Output<StreamRecord<Integer>> output = mock(Output.class);
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            assertTrue("Output should happen under the checkpoint lock.", Thread.currentThread().holdsLock(lock));
            outputLatch.trigger();
            // wait until we're in the closing method of the operator
            while (!closingLatch.isTriggered()) {
                lock.wait();
            }
            return null;
        }
    }).when(output).collect(any(StreamRecord.class));
    AsyncWaitOperator<Integer, Integer> operator = new TestAsyncWaitOperator<>(new MyAsyncFunction(), 1000L, 1, AsyncDataStream.OutputMode.ORDERED, closingLatch);
    operator.setup(containingTask, streamConfig, output);
    operator.open();
    synchronized (lock) {
        operator.processElement(new StreamRecord<>(42));
    }
    outputLatch.await();
    synchronized (lock) {
        operator.close();
    }
    // check that no concurrent exception has occurred
    try {
        verify(environment, never()).failExternally(any(Throwable.class));
    } catch (Error e) {
        // add the exception occurring in the emitter thread (root cause) as a suppressed
        // exception
        e.addSuppressed(failureReason.getValue());
        throw e;
    }
}

Example 44 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class HeapInternalTimerServiceTest method testSetAndFireProcessingTimeTimers.

/**
	 * This also verifies that we don't have leakage between keys/namespaces.
	 */
@Test
public void testSetAndFireProcessingTimeTimers() throws Exception {
    @SuppressWarnings("unchecked") Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class);
    TestKeyContext keyContext = new TestKeyContext();
    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
    HeapInternalTimerService<Integer, String> timerService = createTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, maxParallelism);
    // get two different keys
    int key1 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
    int key2 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
    while (key2 == key1) {
        key2 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
    }
    keyContext.setCurrentKey(key1);
    timerService.registerProcessingTimeTimer("ciao", 10);
    timerService.registerProcessingTimeTimer("hello", 10);
    keyContext.setCurrentKey(key2);
    timerService.registerProcessingTimeTimer("ciao", 10);
    timerService.registerProcessingTimeTimer("hello", 10);
    assertEquals(4, timerService.numProcessingTimeTimers());
    assertEquals(2, timerService.numProcessingTimeTimers("hello"));
    assertEquals(2, timerService.numProcessingTimeTimers("ciao"));
    processingTimeService.setCurrentTime(10);
    verify(mockTriggerable, times(4)).onProcessingTime(anyInternalTimer());
    verify(mockTriggerable, times(1)).onProcessingTime(eq(new InternalTimer<>(10, key1, "ciao")));
    verify(mockTriggerable, times(1)).onProcessingTime(eq(new InternalTimer<>(10, key1, "hello")));
    verify(mockTriggerable, times(1)).onProcessingTime(eq(new InternalTimer<>(10, key2, "ciao")));
    verify(mockTriggerable, times(1)).onProcessingTime(eq(new InternalTimer<>(10, key2, "hello")));
    assertEquals(0, timerService.numProcessingTimeTimers());
}

Example 45 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class HeapInternalTimerServiceTest method testCurrentEventTime.

@Test
public void testCurrentEventTime() throws Exception {
    @SuppressWarnings("unchecked") Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class);
    TestKeyContext keyContext = new TestKeyContext();
    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
    HeapInternalTimerService<Integer, String> timerService = createTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, maxParallelism);
    timerService.advanceWatermark(17);
    assertEquals(17, timerService.currentWatermark());
    timerService.advanceWatermark(42);
    assertEquals(42, timerService.currentWatermark());
}