Examples with TestProcessingTimeService org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService used on opensource projects

Example 61 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class StreamTaskStateInitializerImplTest method testNoRestore.

@Test
public void testNoRestore() throws Exception {
    MemoryStateBackend stateBackend = spy(new MemoryStateBackend(1024));
    // No job manager provided state to restore
    StreamTaskStateInitializer streamTaskStateManager = streamTaskStateManager(stateBackend, null, true);
    OperatorID operatorID = new OperatorID(47L, 11L);
    AbstractStreamOperator<?> streamOperator = mock(AbstractStreamOperator.class);
    when(streamOperator.getOperatorID()).thenReturn(operatorID);
    TypeSerializer<?> typeSerializer = new IntSerializer();
    CloseableRegistry closeableRegistry = new CloseableRegistry();
    StreamOperatorStateContext stateContext = streamTaskStateManager.streamOperatorStateContext(streamOperator.getOperatorID(), streamOperator.getClass().getSimpleName(), new TestProcessingTimeService(), streamOperator, typeSerializer, closeableRegistry, new UnregisteredMetricsGroup(), 1.0, false);
    OperatorStateBackend operatorStateBackend = stateContext.operatorStateBackend();
    CheckpointableKeyedStateBackend<?> keyedStateBackend = stateContext.keyedStateBackend();
    InternalTimeServiceManager<?> timeServiceManager = stateContext.internalTimerServiceManager();
    CloseableIterable<KeyGroupStatePartitionStreamProvider> keyedStateInputs = stateContext.rawKeyedStateInputs();
    CloseableIterable<StatePartitionStreamProvider> operatorStateInputs = stateContext.rawOperatorStateInputs();
    Assert.assertFalse("Expected the context to NOT be restored", stateContext.isRestored());
    Assert.assertNotNull(operatorStateBackend);
    Assert.assertNotNull(keyedStateBackend);
    Assert.assertNotNull(timeServiceManager);
    Assert.assertNotNull(keyedStateInputs);
    Assert.assertNotNull(operatorStateInputs);
    checkCloseablesRegistered(closeableRegistry, operatorStateBackend, keyedStateBackend, keyedStateInputs, operatorStateInputs);
    Assert.assertFalse(keyedStateInputs.iterator().hasNext());
    Assert.assertFalse(operatorStateInputs.iterator().hasNext());
}

Example 62 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class InternalTimerServiceImplTest method testForEachProcessingTimeTimers.

/**
 * This also verifies that we iterate over all timers and set the key context on each element.
 */
@Test
public void testForEachProcessingTimeTimers() throws Exception {
    @SuppressWarnings("unchecked") Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class);
    TestKeyContext keyContext = new TestKeyContext();
    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
    InternalTimerServiceImpl<Integer, String> timerService = createAndStartInternalTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, createQueueFactory());
    // get two different keys
    int key1 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
    int key2 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
    while (key2 == key1) {
        key2 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
    }
    Set<Tuple3<Integer, String, Long>> timers = new HashSet<>();
    timers.add(Tuple3.of(key1, "ciao", 10L));
    timers.add(Tuple3.of(key1, "hello", 10L));
    timers.add(Tuple3.of(key2, "ciao", 10L));
    timers.add(Tuple3.of(key2, "hello", 10L));
    for (Tuple3<Integer, String, Long> timer : timers) {
        keyContext.setCurrentKey(timer.f0);
        timerService.registerProcessingTimeTimer(timer.f1, timer.f2);
    }
    Set<Tuple3<Integer, String, Long>> results = new HashSet<>();
    timerService.forEachProcessingTimeTimer((namespace, timer) -> {
        results.add(Tuple3.of((Integer) keyContext.getCurrentKey(), namespace, timer));
    });
    Assert.assertEquals(timers, results);
}

Example 63 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class InternalTimerServiceImplTest method testSetAndFireProcessingTimeTimers.

/**
 * This also verifies that we don't have leakage between keys/namespaces.
 */
@Test
public void testSetAndFireProcessingTimeTimers() throws Exception {
    @SuppressWarnings("unchecked") Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class);
    TestKeyContext keyContext = new TestKeyContext();
    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
    InternalTimerServiceImpl<Integer, String> timerService = createAndStartInternalTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, createQueueFactory());
    // get two different keys
    int key1 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
    int key2 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
    while (key2 == key1) {
        key2 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
    }
    keyContext.setCurrentKey(key1);
    timerService.registerProcessingTimeTimer("ciao", 10);
    timerService.registerProcessingTimeTimer("hello", 10);
    keyContext.setCurrentKey(key2);
    timerService.registerProcessingTimeTimer("ciao", 10);
    timerService.registerProcessingTimeTimer("hello", 10);
    assertEquals(4, timerService.numProcessingTimeTimers());
    assertEquals(2, timerService.numProcessingTimeTimers("hello"));
    assertEquals(2, timerService.numProcessingTimeTimers("ciao"));
    processingTimeService.setCurrentTime(10);
    verify(mockTriggerable, times(4)).onProcessingTime(anyInternalTimer());
    verify(mockTriggerable, times(1)).onProcessingTime(eq(new TimerHeapInternalTimer<>(10, key1, "ciao")));
    verify(mockTriggerable, times(1)).onProcessingTime(eq(new TimerHeapInternalTimer<>(10, key1, "hello")));
    verify(mockTriggerable, times(1)).onProcessingTime(eq(new TimerHeapInternalTimer<>(10, key2, "ciao")));
    verify(mockTriggerable, times(1)).onProcessingTime(eq(new TimerHeapInternalTimer<>(10, key2, "hello")));
    assertEquals(0, timerService.numProcessingTimeTimers());
}

Example 64 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class InternalTimerServiceImplTest method testCurrentEventTime.

@Test
public void testCurrentEventTime() throws Exception {
    @SuppressWarnings("unchecked") Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class);
    TestKeyContext keyContext = new TestKeyContext();
    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
    InternalTimerServiceImpl<Integer, String> timerService = createAndStartInternalTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, createQueueFactory());
    timerService.advanceWatermark(17);
    assertEquals(17, timerService.currentWatermark());
    timerService.advanceWatermark(42);
    assertEquals(42, timerService.currentWatermark());
}

Example 65 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class InternalTimerServiceImplTest method testSnapshotAndRestore.

private void testSnapshotAndRestore(int snapshotVersion) throws Exception {
    @SuppressWarnings("unchecked") Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class);
    TestKeyContext keyContext = new TestKeyContext();
    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
    InternalTimerServiceImpl<Integer, String> timerService = createAndStartInternalTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, createQueueFactory());
    // get two different keys
    int key1 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
    int key2 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
    while (key2 == key1) {
        key2 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
    }
    keyContext.setCurrentKey(key1);
    timerService.registerProcessingTimeTimer("ciao", 10);
    timerService.registerEventTimeTimer("hello", 10);
    keyContext.setCurrentKey(key2);
    timerService.registerEventTimeTimer("ciao", 10);
    timerService.registerProcessingTimeTimer("hello", 10);
    assertEquals(2, timerService.numProcessingTimeTimers());
    assertEquals(1, timerService.numProcessingTimeTimers("hello"));
    assertEquals(1, timerService.numProcessingTimeTimers("ciao"));
    assertEquals(2, timerService.numEventTimeTimers());
    assertEquals(1, timerService.numEventTimeTimers("hello"));
    assertEquals(1, timerService.numEventTimeTimers("ciao"));
    Map<Integer, byte[]> snapshot = new HashMap<>();
    for (Integer keyGroupIndex : testKeyGroupRange) {
        try (ByteArrayOutputStream outStream = new ByteArrayOutputStream()) {
            InternalTimersSnapshot<Integer, String> timersSnapshot = timerService.snapshotTimersForKeyGroup(keyGroupIndex);
            InternalTimersSnapshotReaderWriters.getWriterForVersion(snapshotVersion, timersSnapshot, timerService.getKeySerializer(), timerService.getNamespaceSerializer()).writeTimersSnapshot(new DataOutputViewStreamWrapper(outStream));
            snapshot.put(keyGroupIndex, outStream.toByteArray());
        }
    }
    @SuppressWarnings("unchecked") Triggerable<Integer, String> mockTriggerable2 = mock(Triggerable.class);
    keyContext = new TestKeyContext();
    processingTimeService = new TestProcessingTimeService();
    timerService = restoreTimerService(snapshot, snapshotVersion, mockTriggerable2, keyContext, processingTimeService, testKeyGroupRange, createQueueFactory());
    processingTimeService.setCurrentTime(10);
    timerService.advanceWatermark(10);
    verify(mockTriggerable2, times(2)).onProcessingTime(anyInternalTimer());
    verify(mockTriggerable2, times(1)).onProcessingTime(eq(new TimerHeapInternalTimer<>(10, key1, "ciao")));
    verify(mockTriggerable2, times(1)).onProcessingTime(eq(new TimerHeapInternalTimer<>(10, key2, "hello")));
    verify(mockTriggerable2, times(2)).onEventTime(anyInternalTimer());
    verify(mockTriggerable2, times(1)).onEventTime(eq(new TimerHeapInternalTimer<>(10, key1, "hello")));
    verify(mockTriggerable2, times(1)).onEventTime(eq(new TimerHeapInternalTimer<>(10, key2, "ciao")));
    assertEquals(0, timerService.numEventTimeTimers());
}