Examples with TestTaskStateManager org.apache.flink.runtime.state.TestTaskStateManager used on opensource projects

Example 11 with TestTaskStateManager

use of org.apache.flink.runtime.state.TestTaskStateManager in project flink by apache.

the class StreamTaskTerminationTest method testConcurrentAsyncCheckpointCannotFailFinishedStreamTask.

/**
 * FLINK-6833
 *
 * <p>Tests that a finished stream task cannot be failed by an asynchronous checkpointing
 * operation after the stream task has stopped running.
 */
@Test
public void testConcurrentAsyncCheckpointCannotFailFinishedStreamTask() throws Exception {
    final Configuration taskConfiguration = new Configuration();
    final StreamConfig streamConfig = new StreamConfig(taskConfiguration);
    final NoOpStreamOperator<Long> noOpStreamOperator = new NoOpStreamOperator<>();
    final StateBackend blockingStateBackend = new BlockingStateBackend();
    streamConfig.setStreamOperator(noOpStreamOperator);
    streamConfig.setOperatorID(new OperatorID());
    streamConfig.setStateBackend(blockingStateBackend);
    final long checkpointId = 0L;
    final long checkpointTimestamp = 0L;
    final JobInformation jobInformation = new JobInformation(new JobID(), "Test Job", new SerializedValue<>(new ExecutionConfig()), new Configuration(), Collections.emptyList(), Collections.emptyList());
    final TaskInformation taskInformation = new TaskInformation(new JobVertexID(), "Test Task", 1, 1, BlockingStreamTask.class.getName(), taskConfiguration);
    final TaskManagerRuntimeInfo taskManagerRuntimeInfo = new TestingTaskManagerRuntimeInfo();
    final ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();
    final Task task = new Task(jobInformation, taskInformation, new ExecutionAttemptID(), new AllocationID(), 0, 0, Collections.<ResultPartitionDeploymentDescriptor>emptyList(), Collections.<InputGateDeploymentDescriptor>emptyList(), MemoryManagerBuilder.newBuilder().setMemorySize(32L * 1024L).build(), new IOManagerAsync(), shuffleEnvironment, new KvStateService(new KvStateRegistry(), null, null), mock(BroadcastVariableManager.class), new TaskEventDispatcher(), ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES, new TestTaskStateManager(), mock(TaskManagerActions.class), mock(InputSplitProvider.class), mock(CheckpointResponder.class), new NoOpTaskOperatorEventGateway(), new TestGlobalAggregateManager(), TestingClassLoaderLease.newBuilder().build(), mock(FileCache.class), taskManagerRuntimeInfo, UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(), new NoOpResultPartitionConsumableNotifier(), mock(PartitionProducerStateChecker.class), Executors.directExecutor());
    CompletableFuture<Void> taskRun = CompletableFuture.runAsync(() -> task.run(), TestingUtils.defaultExecutor());
    // wait until the stream task started running
    RUN_LATCH.await();
    // trigger a checkpoint
    task.triggerCheckpointBarrier(checkpointId, checkpointTimestamp, CheckpointOptions.forCheckpointWithDefaultLocation());
    // wait until the task has completed execution
    taskRun.get();
    // check that no failure occurred
    if (task.getFailureCause() != null) {
        throw new Exception("Task failed", task.getFailureCause());
    }
    // check that we have entered the finished state
    assertEquals(ExecutionState.FINISHED, task.getExecutionState());
}

Example 12 with TestTaskStateManager

use of org.apache.flink.runtime.state.TestTaskStateManager in project flink by apache.

the class SubtaskCheckpointCoordinatorTest method testNotifyCheckpointAbortedAfterAsyncPhase.

@Test
public void testNotifyCheckpointAbortedAfterAsyncPhase() throws Exception {
    TestTaskStateManager stateManager = new TestTaskStateManager();
    MockEnvironment mockEnvironment = MockEnvironment.builder().setTaskStateManager(stateManager).build();
    try (SubtaskCheckpointCoordinatorImpl subtaskCheckpointCoordinator = (SubtaskCheckpointCoordinatorImpl) new MockSubtaskCheckpointCoordinatorBuilder().setEnvironment(mockEnvironment).build()) {
        final OperatorChain<?, ?> operatorChain = getOperatorChain(mockEnvironment);
        long checkpointId = 42L;
        subtaskCheckpointCoordinator.checkpointState(new CheckpointMetaData(checkpointId, System.currentTimeMillis()), CheckpointOptions.forCheckpointWithDefaultLocation(), new CheckpointMetricsBuilder(), operatorChain, false, () -> false);
        subtaskCheckpointCoordinator.notifyCheckpointAborted(checkpointId, operatorChain, () -> true);
        assertEquals(0, subtaskCheckpointCoordinator.getAbortedCheckpointSize());
        assertEquals(checkpointId, stateManager.getNotifiedAbortedCheckpointId());
    }
}

Example 13 with TestTaskStateManager

use of org.apache.flink.runtime.state.TestTaskStateManager in project flink by apache.

the class TwoInputStreamTaskTest method testWatermarkMetrics.

@Test
@SuppressWarnings("unchecked")
public void testWatermarkMetrics() throws Exception {
    final TwoInputStreamTaskTestHarness<String, Integer, String> testHarness = new TwoInputStreamTaskTestHarness<>(TwoInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    CoStreamMap<String, Integer, String> headOperator = new CoStreamMap<>(new IdentityMap());
    final OperatorID headOperatorId = new OperatorID();
    OneInputStreamTaskTest.WatermarkMetricOperator chainedOperator = new OneInputStreamTaskTest.WatermarkMetricOperator();
    OperatorID chainedOperatorId = new OperatorID();
    testHarness.setupOperatorChain(headOperatorId, headOperator).chain(chainedOperatorId, chainedOperator, BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig())).finish();
    InterceptingOperatorMetricGroup headOperatorMetricGroup = new InterceptingOperatorMetricGroup();
    InterceptingOperatorMetricGroup chainedOperatorMetricGroup = new InterceptingOperatorMetricGroup();
    InterceptingTaskMetricGroup taskMetricGroup = new InterceptingTaskMetricGroup() {

        @Override
        public InternalOperatorMetricGroup getOrAddOperator(OperatorID id, String name) {
            if (id.equals(headOperatorId)) {
                return headOperatorMetricGroup;
            } else if (id.equals(chainedOperatorId)) {
                return chainedOperatorMetricGroup;
            } else {
                return super.getOrAddOperator(id, name);
            }
        }
    };
    StreamMockEnvironment env = new StreamMockEnvironment(testHarness.jobConfig, testHarness.taskConfig, testHarness.memorySize, new MockInputSplitProvider(), testHarness.bufferSize, new TestTaskStateManager()) {

        @Override
        public TaskMetricGroup getMetricGroup() {
            return taskMetricGroup;
        }
    };
    testHarness.invoke(env);
    testHarness.waitForTaskRunning();
    Gauge<Long> taskInputWatermarkGauge = (Gauge<Long>) taskMetricGroup.get(MetricNames.IO_CURRENT_INPUT_WATERMARK);
    Gauge<Long> headInput1WatermarkGauge = (Gauge<Long>) headOperatorMetricGroup.get(MetricNames.IO_CURRENT_INPUT_1_WATERMARK);
    Gauge<Long> headInput2WatermarkGauge = (Gauge<Long>) headOperatorMetricGroup.get(MetricNames.IO_CURRENT_INPUT_2_WATERMARK);
    Gauge<Long> headInputWatermarkGauge = (Gauge<Long>) headOperatorMetricGroup.get(MetricNames.IO_CURRENT_INPUT_WATERMARK);
    Gauge<Long> headOutputWatermarkGauge = (Gauge<Long>) headOperatorMetricGroup.get(MetricNames.IO_CURRENT_OUTPUT_WATERMARK);
    Gauge<Long> chainedInputWatermarkGauge = (Gauge<Long>) chainedOperatorMetricGroup.get(MetricNames.IO_CURRENT_INPUT_WATERMARK);
    Gauge<Long> chainedOutputWatermarkGauge = (Gauge<Long>) chainedOperatorMetricGroup.get(MetricNames.IO_CURRENT_OUTPUT_WATERMARK);
    Assert.assertEquals("A metric was registered multiple times.", 7, new HashSet<>(Arrays.asList(taskInputWatermarkGauge, headInput1WatermarkGauge, headInput2WatermarkGauge, headInputWatermarkGauge, headOutputWatermarkGauge, chainedInputWatermarkGauge, chainedOutputWatermarkGauge)).size());
    Assert.assertEquals(Long.MIN_VALUE, taskInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, headInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, headInput1WatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, headInput2WatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, headOutputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, chainedInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, chainedOutputWatermarkGauge.getValue().longValue());
    testHarness.processElement(new Watermark(1L), 0, 0);
    testHarness.waitForInputProcessing();
    Assert.assertEquals(Long.MIN_VALUE, taskInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, headInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(1L, headInput1WatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, headInput2WatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, headOutputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, chainedInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, chainedOutputWatermarkGauge.getValue().longValue());
    testHarness.processElement(new Watermark(2L), 1, 0);
    testHarness.waitForInputProcessing();
    Assert.assertEquals(1L, taskInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(1L, headInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(1L, headInput1WatermarkGauge.getValue().longValue());
    Assert.assertEquals(2L, headInput2WatermarkGauge.getValue().longValue());
    Assert.assertEquals(1L, headOutputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(1L, chainedInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(2L, chainedOutputWatermarkGauge.getValue().longValue());
    testHarness.processElement(new Watermark(3L), 0, 0);
    testHarness.waitForInputProcessing();
    Assert.assertEquals(2L, taskInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(2L, headInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(3L, headInput1WatermarkGauge.getValue().longValue());
    Assert.assertEquals(2L, headInput2WatermarkGauge.getValue().longValue());
    Assert.assertEquals(2L, headOutputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(2L, chainedInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(4L, chainedOutputWatermarkGauge.getValue().longValue());
    testHarness.endInput();
    testHarness.waitForTaskCompletion();
}

Example 14 with TestTaskStateManager

use of org.apache.flink.runtime.state.TestTaskStateManager in project flink by apache.

the class SubtaskCheckpointCoordinatorTest method testNotifyCheckpointSubsumed.

@Test
public void testNotifyCheckpointSubsumed() throws Exception {
    TestTaskStateManager stateManager = new TestTaskStateManager();
    MockEnvironment mockEnvironment = MockEnvironment.builder().setTaskStateManager(stateManager).build();
    try (SubtaskCheckpointCoordinatorImpl subtaskCheckpointCoordinator = (SubtaskCheckpointCoordinatorImpl) new MockSubtaskCheckpointCoordinatorBuilder().setEnvironment(mockEnvironment).setUnalignedCheckpointEnabled(true).build()) {
        StreamMap<String, String> streamMap = new StreamMap<>((MapFunction<String, String>) value -> value);
        streamMap.setProcessingTimeService(new TestProcessingTimeService());
        final OperatorChain<String, AbstractStreamOperator<String>> operatorChain = operatorChain(streamMap);
        StreamTaskStateInitializerImpl stateInitializer = new StreamTaskStateInitializerImpl(mockEnvironment, new TestStateBackend());
        operatorChain.initializeStateAndOpenOperators(stateInitializer);
        long checkpointId = 42L;
        subtaskCheckpointCoordinator.getChannelStateWriter().start(checkpointId, CheckpointOptions.forCheckpointWithDefaultLocation());
        subtaskCheckpointCoordinator.checkpointState(new CheckpointMetaData(checkpointId, System.currentTimeMillis()), CheckpointOptions.forCheckpointWithDefaultLocation(), new CheckpointMetricsBuilder(), operatorChain, false, () -> false);
        long notifySubsumeCheckpointId = checkpointId + 1L;
        // notify checkpoint aborted before execution.
        subtaskCheckpointCoordinator.notifyCheckpointSubsumed(notifySubsumeCheckpointId, operatorChain, () -> true);
        assertEquals(notifySubsumeCheckpointId, ((TestStateBackend.TestKeyedStateBackend<?>) streamMap.getKeyedStateBackend()).getSubsumeCheckpointId());
    }
}

Example 15 with TestTaskStateManager

use of org.apache.flink.runtime.state.TestTaskStateManager in project flink by apache.

the class SubtaskCheckpointCoordinatorTest method testNotifyCheckpointAbortedBeforeAsyncPhase.

@Test
public void testNotifyCheckpointAbortedBeforeAsyncPhase() throws Exception {
    TestTaskStateManager stateManager = new TestTaskStateManager();
    MockEnvironment mockEnvironment = MockEnvironment.builder().setTaskStateManager(stateManager).build();
    try (SubtaskCheckpointCoordinatorImpl subtaskCheckpointCoordinator = (SubtaskCheckpointCoordinatorImpl) new MockSubtaskCheckpointCoordinatorBuilder().setEnvironment(mockEnvironment).setUnalignedCheckpointEnabled(true).build()) {
        CheckpointOperator checkpointOperator = new CheckpointOperator(new OperatorSnapshotFutures());
        final OperatorChain<String, AbstractStreamOperator<String>> operatorChain = operatorChain(checkpointOperator);
        long checkpointId = 42L;
        // notify checkpoint aborted before execution.
        subtaskCheckpointCoordinator.notifyCheckpointAborted(checkpointId, operatorChain, () -> true);
        assertEquals(1, subtaskCheckpointCoordinator.getAbortedCheckpointSize());
        subtaskCheckpointCoordinator.getChannelStateWriter().start(checkpointId, CheckpointOptions.forCheckpointWithDefaultLocation());
        subtaskCheckpointCoordinator.checkpointState(new CheckpointMetaData(checkpointId, System.currentTimeMillis()), CheckpointOptions.forCheckpointWithDefaultLocation(), new CheckpointMetricsBuilder(), operatorChain, false, () -> false);
        assertFalse(checkpointOperator.isCheckpointed());
        assertEquals(-1, stateManager.getReportedCheckpointId());
        assertEquals(0, subtaskCheckpointCoordinator.getAbortedCheckpointSize());
        assertEquals(0, subtaskCheckpointCoordinator.getAsyncCheckpointRunnableSize());
    }
}