Examples with TestProcessingTimeService org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService used on opensource projects

Example 71 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class StreamingRuntimeContextTest method createMapPlainMockOp.

@SuppressWarnings("unchecked")
private static AbstractStreamOperator<?> createMapPlainMockOp() throws Exception {
    AbstractStreamOperator<?> operatorMock = mock(AbstractStreamOperator.class);
    ExecutionConfig config = new ExecutionConfig();
    KeyedStateBackend keyedStateBackend = mock(KeyedStateBackend.class);
    DefaultKeyedStateStore keyedStateStore = new DefaultKeyedStateStore(keyedStateBackend, config);
    when(operatorMock.getExecutionConfig()).thenReturn(config);
    doAnswer(new Answer<MapState<Integer, String>>() {

        @Override
        public MapState<Integer, String> answer(InvocationOnMock invocationOnMock) throws Throwable {
            MapStateDescriptor<Integer, String> descr = (MapStateDescriptor<Integer, String>) invocationOnMock.getArguments()[2];
            AbstractKeyedStateBackend<Integer> backend = new MemoryStateBackend().createKeyedStateBackend(new DummyEnvironment("test_task", 1, 0), new JobID(), "test_op", IntSerializer.INSTANCE, 1, new KeyGroupRange(0, 0), new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID()), TtlTimeProvider.DEFAULT, new UnregisteredMetricsGroup(), Collections.emptyList(), new CloseableRegistry());
            backend.setCurrentKey(0);
            return backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, descr);
        }
    }).when(keyedStateBackend).getPartitionedState(Matchers.any(), any(TypeSerializer.class), any(MapStateDescriptor.class));
    when(operatorMock.getKeyedStateStore()).thenReturn(keyedStateStore);
    when(operatorMock.getOperatorID()).thenReturn(new OperatorID());
    when(operatorMock.getProcessingTimeService()).thenReturn(new TestProcessingTimeService());
    return operatorMock;
}

Example 72 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class InternalTimerServiceImplTest method testDeleteProcessingTimeTimers.

/**
 * This also verifies that we don't have leakage between keys/namespaces.
 *
 * <p>This also verifies that deleted timers don't fire.
 */
@Test
public void testDeleteProcessingTimeTimers() throws Exception {
    @SuppressWarnings("unchecked") Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class);
    TestKeyContext keyContext = new TestKeyContext();
    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
    InternalTimerServiceImpl<Integer, String> timerService = createAndStartInternalTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, createQueueFactory());
    // get two different keys
    int key1 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
    int key2 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
    while (key2 == key1) {
        key2 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
    }
    keyContext.setCurrentKey(key1);
    timerService.registerProcessingTimeTimer("ciao", 10);
    timerService.registerProcessingTimeTimer("hello", 10);
    keyContext.setCurrentKey(key2);
    timerService.registerProcessingTimeTimer("ciao", 10);
    timerService.registerProcessingTimeTimer("hello", 10);
    assertEquals(4, timerService.numProcessingTimeTimers());
    assertEquals(2, timerService.numProcessingTimeTimers("hello"));
    assertEquals(2, timerService.numProcessingTimeTimers("ciao"));
    keyContext.setCurrentKey(key1);
    timerService.deleteProcessingTimeTimer("hello", 10);
    keyContext.setCurrentKey(key2);
    timerService.deleteProcessingTimeTimer("ciao", 10);
    assertEquals(2, timerService.numProcessingTimeTimers());
    assertEquals(1, timerService.numProcessingTimeTimers("hello"));
    assertEquals(1, timerService.numProcessingTimeTimers("ciao"));
    processingTimeService.setCurrentTime(10);
    verify(mockTriggerable, times(2)).onProcessingTime(anyInternalTimer());
    verify(mockTriggerable, times(1)).onProcessingTime(eq(new TimerHeapInternalTimer<>(10, key1, "ciao")));
    verify(mockTriggerable, times(0)).onProcessingTime(eq(new TimerHeapInternalTimer<>(10, key1, "hello")));
    verify(mockTriggerable, times(0)).onProcessingTime(eq(new TimerHeapInternalTimer<>(10, key2, "ciao")));
    verify(mockTriggerable, times(1)).onProcessingTime(eq(new TimerHeapInternalTimer<>(10, key2, "hello")));
    assertEquals(0, timerService.numEventTimeTimers());
}

Example 73 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class AsyncSinkWriterTest method testThatABatchWithSizeSmallerThanMaxBatchSizeIsFlushedOnTimeoutExpiry.

@Test
public void testThatABatchWithSizeSmallerThanMaxBatchSizeIsFlushedOnTimeoutExpiry() throws Exception {
    AsyncSinkWriterImpl sink = new AsyncSinkWriterImplBuilder().context(sinkInitContext).maxBatchSize(10).maxInFlightRequests(20).maxBatchSizeInBytes(10_000).maxTimeInBufferMS(100).maxRecordSizeInBytes(10_000).simulateFailures(true).build();
    TestProcessingTimeService tpts = sinkInitContext.getTestProcessingTimeService();
    tpts.setCurrentTime(0L);
    for (int i = 0; i < 8; i++) {
        sink.write(String.valueOf(i));
    }
    tpts.setCurrentTime(99L);
    assertEquals(0, res.size());
    tpts.setCurrentTime(100L);
    assertEquals(8, res.size());
}

Example 74 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class AsyncSinkWriterTest method testThatOnExpiryOfAnOldTimeoutANewOneMayBeRegisteredImmediately.

@Test
public void testThatOnExpiryOfAnOldTimeoutANewOneMayBeRegisteredImmediately() throws Exception {
    AsyncSinkWriterImpl sink = new AsyncSinkWriterImplBuilder().context(sinkInitContext).maxBatchSize(10).maxInFlightRequests(20).maxBatchSizeInBytes(10_000).maxTimeInBufferMS(100).maxRecordSizeInBytes(10_000).simulateFailures(true).build();
    TestProcessingTimeService tpts = sinkInitContext.getTestProcessingTimeService();
    tpts.setCurrentTime(0L);
    sink.write("1");
    tpts.setCurrentTime(100L);
    assertEquals(1, res.size());
    sink.write("2");
    tpts.setCurrentTime(200L);
    assertEquals(2, res.size());
}

Example 75 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class AsyncSinkWriterTest method testThatOneAndOnlyOneCallbackIsEverRegistered.

@Test
public void testThatOneAndOnlyOneCallbackIsEverRegistered() throws Exception {
    AsyncSinkWriterImpl sink = new AsyncSinkWriterImplBuilder().context(sinkInitContext).maxBatchSize(10).maxInFlightRequests(20).maxBatchSizeInBytes(10_000).maxTimeInBufferMS(100).maxRecordSizeInBytes(10_000).simulateFailures(true).build();
    TestProcessingTimeService tpts = sinkInitContext.getTestProcessingTimeService();
    tpts.setCurrentTime(0L);
    // A timer is registered here to elapse at t=100
    sink.write("1");
    assertEquals(0, res.size());
    tpts.setCurrentTime(10L);
    sink.flush(true);
    assertEquals(1, res.size());
    // At t=20, we write a new element that should not trigger another
    tpts.setCurrentTime(20L);
    // timer to be registered. If it is, it should elapse at t=120s.
    sink.write("2");
    assertEquals(1, res.size());
    tpts.setCurrentTime(100L);
    assertEquals(2, res.size());
    sink.write("3");
    // At t=199s, our third element has not been written
    tpts.setCurrentTime(199L);
    // therefore, no timer fired at 120s.
    assertEquals(2, res.size());
    tpts.setCurrentTime(200L);
    assertEquals(3, res.size());
}