Examples with TestProcessingTimeService org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService used on opensource projects

Example 56 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class BatchExecutionInternalTimeServiceTest method testSettingSameKeyDoesNotFireTimers.

@Test
public void testSettingSameKeyDoesNotFireTimers() {
    BatchExecutionKeyedStateBackend<Integer> keyedStatedBackend = new BatchExecutionKeyedStateBackend<>(KEY_SERIALIZER, new KeyGroupRange(0, 1));
    InternalTimeServiceManager<Integer> timeServiceManager = BatchExecutionInternalTimeServiceManager.create(keyedStatedBackend, this.getClass().getClassLoader(), new DummyKeyContext(), new TestProcessingTimeService(), Collections.emptyList());
    List<Long> timers = new ArrayList<>();
    InternalTimerService<VoidNamespace> timerService = timeServiceManager.getInternalTimerService("test", KEY_SERIALIZER, new VoidNamespaceSerializer(), LambdaTrigger.eventTimeTrigger(timer -> timers.add(timer.getTimestamp())));
    keyedStatedBackend.setCurrentKey(1);
    timerService.registerEventTimeTimer(VoidNamespace.INSTANCE, 123);
    keyedStatedBackend.setCurrentKey(1);
    assertThat(timers, equalTo(Collections.emptyList()));
}

Example 57 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class BatchExecutionInternalTimeServiceTest method testFiringEventTimeTimers.

@Test
public void testFiringEventTimeTimers() throws Exception {
    BatchExecutionKeyedStateBackend<Integer> keyedStatedBackend = new BatchExecutionKeyedStateBackend<>(KEY_SERIALIZER, new KeyGroupRange(0, 1));
    InternalTimeServiceManager<Integer> timeServiceManager = BatchExecutionInternalTimeServiceManager.create(keyedStatedBackend, this.getClass().getClassLoader(), new DummyKeyContext(), new TestProcessingTimeService(), Collections.emptyList());
    List<Long> timers = new ArrayList<>();
    InternalTimerService<VoidNamespace> timerService = timeServiceManager.getInternalTimerService("test", KEY_SERIALIZER, new VoidNamespaceSerializer(), LambdaTrigger.eventTimeTrigger(timer -> timers.add(timer.getTimestamp())));
    keyedStatedBackend.setCurrentKey(1);
    timerService.registerEventTimeTimer(VoidNamespace.INSTANCE, 123);
    // advancing the watermark should not fire timers
    timeServiceManager.advanceWatermark(new Watermark(1000));
    timerService.deleteEventTimeTimer(VoidNamespace.INSTANCE, 123);
    timerService.registerEventTimeTimer(VoidNamespace.INSTANCE, 150);
    // changing the current key fires all timers
    keyedStatedBackend.setCurrentKey(2);
    assertThat(timers, equalTo(Collections.singletonList(150L)));
}

Example 58 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class SourceOperatorEventTimeTest method testSequenceOfEvents.

@SuppressWarnings("FinalPrivateMethod")
@SafeVarargs
private final List<Object> testSequenceOfEvents(final boolean emitProgressiveWatermarks, final WatermarkStrategy<Integer> watermarkStrategy, final Consumer<ReaderOutput<Integer>>... actions) throws Exception {
    final CollectingDataOutput<Integer> out = new CollectingDataOutput<>();
    final TestProcessingTimeService timeService = new TestProcessingTimeService();
    // start somewhere that is not zero
    timeService.setCurrentTime(Integer.MAX_VALUE);
    final SourceReader<Integer, MockSourceSplit> reader = new InterpretingSourceReader(actions);
    final SourceOperator<Integer, MockSourceSplit> sourceOperator = createTestOperator(reader, watermarkStrategy, timeService, emitProgressiveWatermarks);
    while (sourceOperator.emitNext(out) != DataInputStatus.END_OF_INPUT) {
        timeService.setCurrentTime(timeService.getCurrentProcessingTime() + 100);
    }
    return out.events;
}

Example 59 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class StreamSourceOperatorWatermarksTest method testAutomaticWatermarkContext.

@Test
public void testAutomaticWatermarkContext() throws Exception {
    // regular stream source operator
    final StreamSource<String, InfiniteSource<String>> operator = new StreamSource<>(new InfiniteSource<>());
    long watermarkInterval = 10;
    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
    processingTimeService.setCurrentTime(0);
    MockStreamTask<?, ?> task = setupSourceOperator(operator, TimeCharacteristic.IngestionTime, watermarkInterval, processingTimeService);
    final List<StreamElement> output = new ArrayList<>();
    StreamSourceContexts.getSourceContext(TimeCharacteristic.IngestionTime, processingTimeService, task.getCheckpointLock(), new CollectorOutput<String>(output), operator.getExecutionConfig().getAutoWatermarkInterval(), -1, true);
    for (long i = 1; i < 100; i += watermarkInterval) {
        processingTimeService.setCurrentTime(i);
    }
    assertEquals(9, output.size());
    long nextWatermark = 0;
    for (StreamElement el : output) {
        nextWatermark += watermarkInterval;
        Watermark wm = (Watermark) el;
        assertEquals(wm.getTimestamp(), nextWatermark);
    }
}

Example 60 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class StreamTaskStateInitializerImplTest method testWithRestore.

@SuppressWarnings("unchecked")
@Test
public void testWithRestore() throws Exception {
    StateBackend mockingBackend = spy(new StateBackend() {

        @Override
        public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(Environment env, JobID jobID, String operatorIdentifier, TypeSerializer<K> keySerializer, int numberOfKeyGroups, KeyGroupRange keyGroupRange, TaskKvStateRegistry kvStateRegistry, TtlTimeProvider ttlTimeProvider, MetricGroup metricGroup, @Nonnull Collection<KeyedStateHandle> stateHandles, CloseableRegistry cancelStreamRegistry) throws Exception {
            return mock(AbstractKeyedStateBackend.class);
        }

        @Override
        public OperatorStateBackend createOperatorStateBackend(Environment env, String operatorIdentifier, @Nonnull Collection<OperatorStateHandle> stateHandles, CloseableRegistry cancelStreamRegistry) throws Exception {
            return mock(OperatorStateBackend.class);
        }
    });
    OperatorID operatorID = new OperatorID(47L, 11L);
    TaskStateSnapshot taskStateSnapshot = new TaskStateSnapshot();
    Random random = new Random(0x42);
    OperatorSubtaskState operatorSubtaskState = OperatorSubtaskState.builder().setManagedOperatorState(new OperatorStreamStateHandle(Collections.singletonMap("a", new OperatorStateHandle.StateMetaInfo(new long[] { 0, 10 }, SPLIT_DISTRIBUTE)), CheckpointTestUtils.createDummyStreamStateHandle(random, null))).setRawOperatorState(new OperatorStreamStateHandle(Collections.singletonMap("_default_", new OperatorStateHandle.StateMetaInfo(new long[] { 0, 20, 30 }, SPLIT_DISTRIBUTE)), CheckpointTestUtils.createDummyStreamStateHandle(random, null))).setManagedKeyedState(CheckpointTestUtils.createDummyKeyGroupStateHandle(random, null)).setRawKeyedState(CheckpointTestUtils.createDummyKeyGroupStateHandle(random, null)).setInputChannelState(singleton(createNewInputChannelStateHandle(10, random))).setResultSubpartitionState(singleton(createNewResultSubpartitionStateHandle(10, random))).build();
    taskStateSnapshot.putSubtaskStateByOperatorID(operatorID, operatorSubtaskState);
    JobManagerTaskRestore jobManagerTaskRestore = new JobManagerTaskRestore(42L, taskStateSnapshot);
    StreamTaskStateInitializer streamTaskStateManager = streamTaskStateManager(mockingBackend, jobManagerTaskRestore, false);
    AbstractStreamOperator<?> streamOperator = mock(AbstractStreamOperator.class);
    when(streamOperator.getOperatorID()).thenReturn(operatorID);
    TypeSerializer<?> typeSerializer = new IntSerializer();
    CloseableRegistry closeableRegistry = new CloseableRegistry();
    StreamOperatorStateContext stateContext = streamTaskStateManager.streamOperatorStateContext(streamOperator.getOperatorID(), streamOperator.getClass().getSimpleName(), new TestProcessingTimeService(), streamOperator, typeSerializer, closeableRegistry, new UnregisteredMetricsGroup(), 1.0, false);
    OperatorStateBackend operatorStateBackend = stateContext.operatorStateBackend();
    CheckpointableKeyedStateBackend<?> keyedStateBackend = stateContext.keyedStateBackend();
    InternalTimeServiceManager<?> timeServiceManager = stateContext.internalTimerServiceManager();
    CloseableIterable<KeyGroupStatePartitionStreamProvider> keyedStateInputs = stateContext.rawKeyedStateInputs();
    CloseableIterable<StatePartitionStreamProvider> operatorStateInputs = stateContext.rawOperatorStateInputs();
    Assert.assertTrue("Expected the context to be restored", stateContext.isRestored());
    Assert.assertEquals(OptionalLong.of(42L), stateContext.getRestoredCheckpointId());
    Assert.assertNotNull(operatorStateBackend);
    Assert.assertNotNull(keyedStateBackend);
    // this is deactivated on purpose so that it does not attempt to consume the raw keyed
    // state.
    Assert.assertNull(timeServiceManager);
    Assert.assertNotNull(keyedStateInputs);
    Assert.assertNotNull(operatorStateInputs);
    int count = 0;
    for (KeyGroupStatePartitionStreamProvider keyedStateInput : keyedStateInputs) {
        ++count;
    }
    Assert.assertEquals(1, count);
    count = 0;
    for (StatePartitionStreamProvider operatorStateInput : operatorStateInputs) {
        ++count;
    }
    Assert.assertEquals(3, count);
    checkCloseablesRegistered(closeableRegistry, operatorStateBackend, keyedStateBackend, keyedStateInputs, operatorStateInputs);
}