Examples with ProcessingTimeService org.apache.flink.streaming.runtime.tasks.ProcessingTimeService used on opensource projects

Example 1 with ProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.ProcessingTimeService in project flink by apache.

the class AsyncWaitOperatorTest method testTimeoutCleanup.

/**
	 * FLINK-5652
	 * Tests that registered timers are properly canceled upon completion of a
	 * {@link StreamRecordQueueEntry} in order to avoid resource leaks because TriggerTasks hold
	 * a reference on the StreamRecordQueueEntry.
	 */
@Test
public void testTimeoutCleanup() throws Exception {
    final Object lock = new Object();
    final long timeout = 100000L;
    final long timestamp = 1L;
    Environment environment = mock(Environment.class);
    when(environment.getMetricGroup()).thenReturn(new UnregisteredTaskMetricsGroup());
    when(environment.getTaskManagerInfo()).thenReturn(new TestingTaskManagerRuntimeInfo());
    when(environment.getUserClassLoader()).thenReturn(getClass().getClassLoader());
    when(environment.getTaskInfo()).thenReturn(new TaskInfo("testTask", 1, 0, 1, 0));
    ScheduledFuture<?> scheduledFuture = mock(ScheduledFuture.class);
    ProcessingTimeService processingTimeService = mock(ProcessingTimeService.class);
    when(processingTimeService.getCurrentProcessingTime()).thenReturn(timestamp);
    doReturn(scheduledFuture).when(processingTimeService).registerTimer(anyLong(), any(ProcessingTimeCallback.class));
    StreamTask<?, ?> containingTask = mock(StreamTask.class);
    when(containingTask.getEnvironment()).thenReturn(environment);
    when(containingTask.getCheckpointLock()).thenReturn(lock);
    when(containingTask.getProcessingTimeService()).thenReturn(processingTimeService);
    StreamConfig streamConfig = mock(StreamConfig.class);
    doReturn(IntSerializer.INSTANCE).when(streamConfig).getTypeSerializerIn1(any(ClassLoader.class));
    Output<StreamRecord<Integer>> output = mock(Output.class);
    AsyncWaitOperator<Integer, Integer> operator = new AsyncWaitOperator<>(new AsyncFunction<Integer, Integer>() {

        private static final long serialVersionUID = -3718276118074877073L;

        @Override
        public void asyncInvoke(Integer input, AsyncCollector<Integer> collector) throws Exception {
            collector.collect(Collections.singletonList(input));
        }
    }, timeout, 1, AsyncDataStream.OutputMode.UNORDERED);
    operator.setup(containingTask, streamConfig, output);
    operator.open();
    final StreamRecord<Integer> streamRecord = new StreamRecord<>(42, timestamp);
    synchronized (lock) {
        // processing an element will register a timeout
        operator.processElement(streamRecord);
    }
    synchronized (lock) {
        // closing the operator waits until all inputs have been processed
        operator.close();
    }
    // check that we actually outputted the result of the single input
    verify(output).collect(eq(streamRecord));
    verify(processingTimeService).registerTimer(eq(processingTimeService.getCurrentProcessingTime() + timeout), any(ProcessingTimeCallback.class));
    // check that we have cancelled our registered timeout
    verify(scheduledFuture).cancel(eq(true));
}

Example 2 with ProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.ProcessingTimeService in project flink by apache.

the class StreamSourceOperatorTest method setupSourceOperator.

@SuppressWarnings("unchecked")
private static <T> void setupSourceOperator(StreamSource<T, ?> operator, TimeCharacteristic timeChar, long watermarkInterval, long latencyMarkInterval, final ProcessingTimeService timeProvider) {
    ExecutionConfig executionConfig = new ExecutionConfig();
    executionConfig.setAutoWatermarkInterval(watermarkInterval);
    executionConfig.setLatencyTrackingInterval(latencyMarkInterval);
    StreamConfig cfg = new StreamConfig(new Configuration());
    cfg.setStateBackend(new MemoryStateBackend());
    cfg.setTimeCharacteristic(timeChar);
    Environment env = new DummyEnvironment("MockTwoInputTask", 1, 0);
    StreamStatusMaintainer streamStatusMaintainer = mock(StreamStatusMaintainer.class);
    when(streamStatusMaintainer.getStreamStatus()).thenReturn(StreamStatus.ACTIVE);
    StreamTask<?, ?> mockTask = mock(StreamTask.class);
    when(mockTask.getName()).thenReturn("Mock Task");
    when(mockTask.getCheckpointLock()).thenReturn(new Object());
    when(mockTask.getConfiguration()).thenReturn(cfg);
    when(mockTask.getEnvironment()).thenReturn(env);
    when(mockTask.getExecutionConfig()).thenReturn(executionConfig);
    when(mockTask.getAccumulatorMap()).thenReturn(Collections.<String, Accumulator<?, ?>>emptyMap());
    when(mockTask.getStreamStatusMaintainer()).thenReturn(streamStatusMaintainer);
    doAnswer(new Answer<ProcessingTimeService>() {

        @Override
        public ProcessingTimeService answer(InvocationOnMock invocation) throws Throwable {
            if (timeProvider == null) {
                throw new RuntimeException("The time provider is null.");
            }
            return timeProvider;
        }
    }).when(mockTask).getProcessingTimeService();
    operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class));
}

Example 3 with ProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.ProcessingTimeService in project flink by apache.

the class StateInitializationContextImplTest method setUp.

@Before
public void setUp() throws Exception {
    this.writtenKeyGroups = 0;
    this.writtenOperatorStates = new HashSet<>();
    this.closableRegistry = new CloseableRegistry();
    ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos(64);
    List<KeyedStateHandle> keyedStateHandles = new ArrayList<>(NUM_HANDLES);
    int prev = 0;
    for (int i = 0; i < NUM_HANDLES; ++i) {
        out.reset();
        int size = i % 4;
        int end = prev + size;
        DataOutputView dov = new DataOutputViewStreamWrapper(out);
        KeyGroupRangeOffsets offsets = new KeyGroupRangeOffsets(i == 9 ? KeyGroupRange.EMPTY_KEY_GROUP_RANGE : new KeyGroupRange(prev, end));
        prev = end + 1;
        for (int kg : offsets.getKeyGroupRange()) {
            offsets.setKeyGroupOffset(kg, out.getPosition());
            dov.writeInt(kg);
            ++writtenKeyGroups;
        }
        KeyedStateHandle handle = new KeyGroupsStateHandle(offsets, new ByteStateHandleCloseChecking("kg-" + i, out.toByteArray()));
        keyedStateHandles.add(handle);
    }
    List<OperatorStateHandle> operatorStateHandles = new ArrayList<>(NUM_HANDLES);
    for (int i = 0; i < NUM_HANDLES; ++i) {
        int size = i % 4;
        out.reset();
        DataOutputView dov = new DataOutputViewStreamWrapper(out);
        LongArrayList offsets = new LongArrayList(size);
        for (int s = 0; s < size; ++s) {
            offsets.add(out.getPosition());
            int val = i * NUM_HANDLES + s;
            dov.writeInt(val);
            writtenOperatorStates.add(val);
        }
        Map<String, OperatorStateHandle.StateMetaInfo> offsetsMap = new HashMap<>();
        offsetsMap.put(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME, new OperatorStateHandle.StateMetaInfo(offsets.toArray(), OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
        OperatorStateHandle operatorStateHandle = new OperatorStreamStateHandle(offsetsMap, new ByteStateHandleCloseChecking("os-" + i, out.toByteArray()));
        operatorStateHandles.add(operatorStateHandle);
    }
    OperatorSubtaskState operatorSubtaskState = OperatorSubtaskState.builder().setRawOperatorState(new StateObjectCollection<>(operatorStateHandles)).setRawKeyedState(new StateObjectCollection<>(keyedStateHandles)).build();
    OperatorID operatorID = new OperatorID();
    TaskStateSnapshot taskStateSnapshot = new TaskStateSnapshot();
    taskStateSnapshot.putSubtaskStateByOperatorID(operatorID, operatorSubtaskState);
    JobManagerTaskRestore jobManagerTaskRestore = new JobManagerTaskRestore(0L, taskStateSnapshot);
    TaskStateManager manager = new TaskStateManagerImpl(new JobID(), new ExecutionAttemptID(), new TestTaskLocalStateStore(), new InMemoryStateChangelogStorage(), jobManagerTaskRestore, mock(CheckpointResponder.class));
    DummyEnvironment environment = new DummyEnvironment("test", 1, 0, prev);
    environment.setTaskStateManager(manager);
    StateBackend stateBackend = new MemoryStateBackend(1024);
    StreamTaskStateInitializer streamTaskStateManager = new StreamTaskStateInitializerImpl(environment, stateBackend, TtlTimeProvider.DEFAULT, new InternalTimeServiceManager.Provider() {

        @Override
        public <K> InternalTimeServiceManager<K> create(CheckpointableKeyedStateBackend<K> keyedStatedBackend, ClassLoader userClassloader, KeyContext keyContext, ProcessingTimeService processingTimeService, Iterable<KeyGroupStatePartitionStreamProvider> rawKeyedStates) throws Exception {
            // stream.
            return null;
        }
    });
    AbstractStreamOperator<?> mockOperator = mock(AbstractStreamOperator.class);
    when(mockOperator.getOperatorID()).thenReturn(operatorID);
    StreamOperatorStateContext stateContext = streamTaskStateManager.streamOperatorStateContext(operatorID, "TestOperatorClass", mock(ProcessingTimeService.class), mockOperator, // consumed by the timer service.
    IntSerializer.INSTANCE, closableRegistry, new UnregisteredMetricsGroup(), 1.0, false);
    OptionalLong restoredCheckpointId = stateContext.getRestoredCheckpointId();
    this.initializationContext = new StateInitializationContextImpl(restoredCheckpointId.isPresent() ? restoredCheckpointId.getAsLong() : null, stateContext.operatorStateBackend(), mock(KeyedStateStore.class), stateContext.rawKeyedStateInputs(), stateContext.rawOperatorStateInputs());
}

Example 4 with ProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.ProcessingTimeService in project flink by apache.

the class StreamTaskStateInitializerImplTest method streamTaskStateManager.

private StreamTaskStateInitializer streamTaskStateManager(StateBackend stateBackend, JobManagerTaskRestore jobManagerTaskRestore, boolean createTimerServiceManager) {
    JobID jobID = new JobID(42L, 43L);
    ExecutionAttemptID executionAttemptID = new ExecutionAttemptID();
    TestCheckpointResponder checkpointResponderMock = new TestCheckpointResponder();
    TaskLocalStateStore taskLocalStateStore = new TestTaskLocalStateStore();
    InMemoryStateChangelogStorage changelogStorage = new InMemoryStateChangelogStorage();
    TaskStateManager taskStateManager = TaskStateManagerImplTest.taskStateManager(jobID, executionAttemptID, checkpointResponderMock, jobManagerTaskRestore, taskLocalStateStore, changelogStorage);
    DummyEnvironment dummyEnvironment = new DummyEnvironment("test-task", 1, 0);
    dummyEnvironment.setTaskStateManager(taskStateManager);
    if (createTimerServiceManager) {
        return new StreamTaskStateInitializerImpl(dummyEnvironment, stateBackend);
    } else {
        return new StreamTaskStateInitializerImpl(dummyEnvironment, stateBackend, TtlTimeProvider.DEFAULT, new InternalTimeServiceManager.Provider() {

            @Override
            public <K> InternalTimeServiceManager<K> create(CheckpointableKeyedStateBackend<K> keyedStatedBackend, ClassLoader userClassloader, KeyContext keyContext, ProcessingTimeService processingTimeService, Iterable<KeyGroupStatePartitionStreamProvider> rawKeyedStates) throws Exception {
                return null;
            }
        });
    }
}

Example 5 with ProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.ProcessingTimeService in project flink by apache.

the class TestProcessingTimeServiceTest method testCustomTimeServiceProvider.

@Test
public void testCustomTimeServiceProvider() throws Throwable {
    final TestProcessingTimeService tp = new TestProcessingTimeService();
    final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>((env) -> new OneInputStreamTask<>(env, tp), BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    StreamMap<String, String> mapOperator = new StreamMap<>(new StreamTaskTimerTest.DummyMapFunction<>());
    streamConfig.setStreamOperator(mapOperator);
    streamConfig.setOperatorID(new OperatorID());
    testHarness.invoke();
    testHarness.waitForTaskRunning();
    ProcessingTimeService processingTimeService = ((StreamMap<?, ?>) testHarness.getHeadOperator()).getProcessingTimeService();
    assertEquals(Long.MIN_VALUE, processingTimeService.getCurrentProcessingTime());
    tp.setCurrentTime(11);
    assertEquals(processingTimeService.getCurrentProcessingTime(), 11);
    tp.setCurrentTime(15);
    tp.setCurrentTime(16);
    assertEquals(processingTimeService.getCurrentProcessingTime(), 16);
    // register 2 tasks
    processingTimeService.registerTimer(30, timestamp -> {
    });
    processingTimeService.registerTimer(40, timestamp -> {
    });
    assertEquals(2, tp.getNumActiveTimers());
    tp.setCurrentTime(35);
    assertEquals(1, tp.getNumActiveTimers());
    tp.setCurrentTime(40);
    assertEquals(0, tp.getNumActiveTimers());
    tp.shutdownService();
}