Examples with InternalTimeServiceManager org.apache.flink.streaming.api.operators.InternalTimeServiceManager used on opensource projects

Example 1 with InternalTimeServiceManager

use of org.apache.flink.streaming.api.operators.InternalTimeServiceManager in project flink by apache.

the class KeyedStateInputFormat method open.

@Override
@SuppressWarnings("unchecked")
public void open(KeyGroupRangeInputSplit split) throws IOException {
    registry = new CloseableRegistry();
    final StreamOperatorStateContext context = new StreamOperatorContextBuilder(getRuntimeContext(), configuration, operatorState, split, registry, stateBackend).withMaxParallelism(split.getNumKeyGroups()).withKey(operator, operator.getKeyType().createSerializer(getRuntimeContext().getExecutionConfig())).build(LOG);
    AbstractKeyedStateBackend<K> keyedStateBackend = (AbstractKeyedStateBackend<K>) context.keyedStateBackend();
    final DefaultKeyedStateStore keyedStateStore = new DefaultKeyedStateStore(keyedStateBackend, getRuntimeContext().getExecutionConfig());
    SavepointRuntimeContext ctx = new SavepointRuntimeContext(getRuntimeContext(), keyedStateStore);
    InternalTimeServiceManager<K> timeServiceManager = (InternalTimeServiceManager<K>) context.internalTimerServiceManager();
    try {
        operator.setup(getRuntimeContext().getExecutionConfig(), keyedStateBackend, timeServiceManager, ctx);
        operator.open();
        keysAndNamespaces = operator.getKeysAndNamespaces(ctx);
    } catch (Exception e) {
        throw new IOException("Failed to restore timer state", e);
    }
}

Example 2 with InternalTimeServiceManager

use of org.apache.flink.streaming.api.operators.InternalTimeServiceManager in project flink by apache.

the class BatchExecutionInternalTimeServiceTest method testProcessingTimeTimers.

@Test
public void testProcessingTimeTimers() {
    BatchExecutionKeyedStateBackend<Integer> keyedStatedBackend = new BatchExecutionKeyedStateBackend<>(KEY_SERIALIZER, new KeyGroupRange(0, 1));
    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
    InternalTimeServiceManager<Integer> timeServiceManager = BatchExecutionInternalTimeServiceManager.create(keyedStatedBackend, this.getClass().getClassLoader(), new DummyKeyContext(), processingTimeService, Collections.emptyList());
    List<Long> timers = new ArrayList<>();
    InternalTimerService<VoidNamespace> timerService = timeServiceManager.getInternalTimerService("test", KEY_SERIALIZER, new VoidNamespaceSerializer(), LambdaTrigger.processingTimeTrigger(timer -> timers.add(timer.getTimestamp())));
    keyedStatedBackend.setCurrentKey(1);
    timerService.registerProcessingTimeTimer(VoidNamespace.INSTANCE, 150);
    // we should never register physical timers
    assertThat(processingTimeService.getNumActiveTimers(), equalTo(0));
    // changing the current key fires all timers
    keyedStatedBackend.setCurrentKey(2);
    assertThat(timers, equalTo(Collections.singletonList(150L)));
}

Example 3 with InternalTimeServiceManager

use of org.apache.flink.streaming.api.operators.InternalTimeServiceManager in project flink by apache.

the class BatchExecutionInternalTimeServiceTest method testSettingSameKeyDoesNotFireTimers.

@Test
public void testSettingSameKeyDoesNotFireTimers() {
    BatchExecutionKeyedStateBackend<Integer> keyedStatedBackend = new BatchExecutionKeyedStateBackend<>(KEY_SERIALIZER, new KeyGroupRange(0, 1));
    InternalTimeServiceManager<Integer> timeServiceManager = BatchExecutionInternalTimeServiceManager.create(keyedStatedBackend, this.getClass().getClassLoader(), new DummyKeyContext(), new TestProcessingTimeService(), Collections.emptyList());
    List<Long> timers = new ArrayList<>();
    InternalTimerService<VoidNamespace> timerService = timeServiceManager.getInternalTimerService("test", KEY_SERIALIZER, new VoidNamespaceSerializer(), LambdaTrigger.eventTimeTrigger(timer -> timers.add(timer.getTimestamp())));
    keyedStatedBackend.setCurrentKey(1);
    timerService.registerEventTimeTimer(VoidNamespace.INSTANCE, 123);
    keyedStatedBackend.setCurrentKey(1);
    assertThat(timers, equalTo(Collections.emptyList()));
}

Example 4 with InternalTimeServiceManager

use of org.apache.flink.streaming.api.operators.InternalTimeServiceManager in project flink by apache.

the class BatchExecutionInternalTimeServiceTest method testFiringEventTimeTimers.

@Test
public void testFiringEventTimeTimers() throws Exception {
    BatchExecutionKeyedStateBackend<Integer> keyedStatedBackend = new BatchExecutionKeyedStateBackend<>(KEY_SERIALIZER, new KeyGroupRange(0, 1));
    InternalTimeServiceManager<Integer> timeServiceManager = BatchExecutionInternalTimeServiceManager.create(keyedStatedBackend, this.getClass().getClassLoader(), new DummyKeyContext(), new TestProcessingTimeService(), Collections.emptyList());
    List<Long> timers = new ArrayList<>();
    InternalTimerService<VoidNamespace> timerService = timeServiceManager.getInternalTimerService("test", KEY_SERIALIZER, new VoidNamespaceSerializer(), LambdaTrigger.eventTimeTrigger(timer -> timers.add(timer.getTimestamp())));
    keyedStatedBackend.setCurrentKey(1);
    timerService.registerEventTimeTimer(VoidNamespace.INSTANCE, 123);
    // advancing the watermark should not fire timers
    timeServiceManager.advanceWatermark(new Watermark(1000));
    timerService.deleteEventTimeTimer(VoidNamespace.INSTANCE, 123);
    timerService.registerEventTimeTimer(VoidNamespace.INSTANCE, 150);
    // changing the current key fires all timers
    keyedStatedBackend.setCurrentKey(2);
    assertThat(timers, equalTo(Collections.singletonList(150L)));
}

Example 5 with InternalTimeServiceManager

use of org.apache.flink.streaming.api.operators.InternalTimeServiceManager in project flink by apache.

the class StreamOperatorSnapshotRestoreTest method testOperatorStatesSnapshotRestoreInternal.

private void testOperatorStatesSnapshotRestoreInternal(final int mode) throws Exception {
    // -------------------------------------------------------------------------- snapshot
    StateBackend stateBackend;
    FsStateBackend fsstateBackend = createStateBackendInternal();
    switch(stateBackendEnum) {
        case FILE:
            stateBackend = fsstateBackend;
            break;
        case ROCKSDB_FULLY_ASYNC:
            stateBackend = new RocksDBStateBackend(fsstateBackend, TernaryBoolean.FALSE);
            break;
        case ROCKSDB_INCREMENTAL:
            stateBackend = new RocksDBStateBackend(fsstateBackend, TernaryBoolean.TRUE);
            break;
        default:
            throw new IllegalStateException(String.format("Do not support statebackend type %s", stateBackendEnum));
    }
    TestOneInputStreamOperator op = new TestOneInputStreamOperator(false);
    JobID jobID = new JobID();
    JobVertexID jobVertexID = new JobVertexID();
    int subtaskIdx = 0;
    LocalRecoveryDirectoryProvider directoryProvider = mode == ONLY_JM_RECOVERY ? null : new LocalRecoveryDirectoryProviderImpl(temporaryFolder.newFolder(), jobID, jobVertexID, subtaskIdx);
    LocalRecoveryConfig localRecoveryConfig = new LocalRecoveryConfig(directoryProvider);
    MockEnvironment mockEnvironment = new MockEnvironmentBuilder().setJobID(jobID).setJobVertexID(jobVertexID).setTaskName("test").setManagedMemorySize(1024L * 1024L).setInputSplitProvider(new MockInputSplitProvider()).setBufferSize(1024 * 1024).setTaskStateManager(new TestTaskStateManager(localRecoveryConfig)).setMaxParallelism(MAX_PARALLELISM).setSubtaskIndex(subtaskIdx).setUserCodeClassLoader(getClass().getClassLoader()).build();
    KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Integer> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(op, (KeySelector<Integer, Integer>) value -> value, TypeInformation.of(Integer.class), mockEnvironment);
    testHarness.setStateBackend(stateBackend);
    testHarness.open();
    for (int i = 0; i < 10; ++i) {
        testHarness.processElement(new StreamRecord<>(i));
    }
    OperatorSnapshotFinalizer snapshotWithLocalState = testHarness.snapshotWithLocalState(1L, 1L);
    testHarness.close();
    // -------------------------------------------------------------------------- restore
    op = new TestOneInputStreamOperator(true);
    testHarness = new KeyedOneInputStreamOperatorTestHarness<>(op, (KeySelector<Integer, Integer>) value -> value, TypeInformation.of(Integer.class), MAX_PARALLELISM, 1, /* num subtasks */
    0);
    testHarness.setTimeServiceManagerProvider(new InternalTimeServiceManager.Provider() {

        @Override
        public <K> InternalTimeServiceManager<K> create(CheckpointableKeyedStateBackend<K> keyedStatedBackend, ClassLoader userClassloader, KeyContext keyContext, ProcessingTimeService processingTimeService, Iterable<KeyGroupStatePartitionStreamProvider> rawKeyedStates) throws IOException {
            return null;
        }
    });
    testHarness.setStateBackend(stateBackend);
    OperatorSubtaskState jobManagerOwnedState = snapshotWithLocalState.getJobManagerOwnedState();
    OperatorSubtaskState taskLocalState = snapshotWithLocalState.getTaskLocalState();
    // We check if local state was created when we enabled local recovery
    Assert.assertTrue(mode > ONLY_JM_RECOVERY == (taskLocalState != null && taskLocalState.hasState()));
    if (mode == TM_REMOVE_JM_RECOVERY) {
        jobManagerOwnedState.getManagedKeyedState().discardState();
    } else if (mode == JM_REMOVE_TM_RECOVERY) {
        taskLocalState.getManagedKeyedState().discardState();
    }
    testHarness.initializeState(jobManagerOwnedState, taskLocalState);
    testHarness.open();
    for (int i = 0; i < 10; ++i) {
        testHarness.processElement(new StreamRecord<>(i));
    }
    testHarness.close();
}