Examples with TestCheckpointResponder org.apache.flink.runtime.taskmanager.TestCheckpointResponder used on opensource projects

Example 1 with TestCheckpointResponder

use of org.apache.flink.runtime.taskmanager.TestCheckpointResponder in project flink by apache.

the class StreamTaskStateInitializerImplTest method streamTaskStateManager.

private StreamTaskStateInitializer streamTaskStateManager(StateBackend stateBackend, JobManagerTaskRestore jobManagerTaskRestore, boolean createTimerServiceManager) {
    JobID jobID = new JobID(42L, 43L);
    ExecutionAttemptID executionAttemptID = new ExecutionAttemptID();
    TestCheckpointResponder checkpointResponderMock = new TestCheckpointResponder();
    TaskLocalStateStore taskLocalStateStore = new TestTaskLocalStateStore();
    InMemoryStateChangelogStorage changelogStorage = new InMemoryStateChangelogStorage();
    TaskStateManager taskStateManager = TaskStateManagerImplTest.taskStateManager(jobID, executionAttemptID, checkpointResponderMock, jobManagerTaskRestore, taskLocalStateStore, changelogStorage);
    DummyEnvironment dummyEnvironment = new DummyEnvironment("test-task", 1, 0);
    dummyEnvironment.setTaskStateManager(taskStateManager);
    if (createTimerServiceManager) {
        return new StreamTaskStateInitializerImpl(dummyEnvironment, stateBackend);
    } else {
        return new StreamTaskStateInitializerImpl(dummyEnvironment, stateBackend, TtlTimeProvider.DEFAULT, new InternalTimeServiceManager.Provider() {

            @Override
            public <K> InternalTimeServiceManager<K> create(CheckpointableKeyedStateBackend<K> keyedStatedBackend, ClassLoader userClassloader, KeyContext keyContext, ProcessingTimeService processingTimeService, Iterable<KeyGroupStatePartitionStreamProvider> rawKeyedStates) throws Exception {
                return null;
            }
        });
    }
}

Example 2 with TestCheckpointResponder

use of org.apache.flink.runtime.taskmanager.TestCheckpointResponder in project flink by apache.

the class TaskStateManagerImplTest method testForwardingSubtaskLocalStateBaseDirFromLocalStateStore.

/**
 * This tests if the {@link TaskStateManager} properly returns the subtask local state dir from
 * the corresponding {@link TaskLocalStateStoreImpl}.
 */
@Test
public void testForwardingSubtaskLocalStateBaseDirFromLocalStateStore() throws IOException {
    JobID jobID = new JobID(42L, 43L);
    AllocationID allocationID = new AllocationID(4711L, 23L);
    JobVertexID jobVertexID = new JobVertexID(12L, 34L);
    ExecutionAttemptID executionAttemptID = new ExecutionAttemptID();
    TestCheckpointResponder checkpointResponderMock = new TestCheckpointResponder();
    Executor directExecutor = Executors.directExecutor();
    TemporaryFolder tmpFolder = new TemporaryFolder();
    try {
        tmpFolder.create();
        File[] allocBaseDirs = new File[] { tmpFolder.newFolder(), tmpFolder.newFolder(), tmpFolder.newFolder() };
        LocalRecoveryDirectoryProviderImpl directoryProvider = new LocalRecoveryDirectoryProviderImpl(allocBaseDirs, jobID, jobVertexID, 0);
        LocalRecoveryConfig localRecoveryConfig = new LocalRecoveryConfig(directoryProvider);
        TaskLocalStateStore taskLocalStateStore = new TaskLocalStateStoreImpl(jobID, allocationID, jobVertexID, 13, localRecoveryConfig, directExecutor);
        InMemoryStateChangelogStorage changelogStorage = new InMemoryStateChangelogStorage();
        TaskStateManager taskStateManager = taskStateManager(jobID, executionAttemptID, checkpointResponderMock, null, taskLocalStateStore, changelogStorage);
        LocalRecoveryConfig localRecoveryConfFromTaskLocalStateStore = taskLocalStateStore.getLocalRecoveryConfig();
        LocalRecoveryConfig localRecoveryConfFromTaskStateManager = taskStateManager.createLocalRecoveryConfig();
        for (int i = 0; i < 10; ++i) {
            Assert.assertEquals(allocBaseDirs[i % allocBaseDirs.length], localRecoveryConfFromTaskLocalStateStore.getLocalStateDirectoryProvider().get().allocationBaseDirectory(i));
            Assert.assertEquals(allocBaseDirs[i % allocBaseDirs.length], localRecoveryConfFromTaskStateManager.getLocalStateDirectoryProvider().get().allocationBaseDirectory(i));
        }
        Assert.assertEquals(localRecoveryConfFromTaskLocalStateStore.isLocalRecoveryEnabled(), localRecoveryConfFromTaskStateManager.isLocalRecoveryEnabled());
    } finally {
        tmpFolder.delete();
    }
}

Example 3 with TestCheckpointResponder

use of org.apache.flink.runtime.taskmanager.TestCheckpointResponder in project flink by apache.

the class TaskStateManagerImplTest method testAcquringRestoreCheckpointId.

public void testAcquringRestoreCheckpointId() {
    TaskStateManagerImpl emptyStateManager = new TaskStateManagerImpl(new JobID(), new ExecutionAttemptID(), new TestTaskLocalStateStore(), null, null, new TestCheckpointResponder());
    Assert.assertFalse(emptyStateManager.getRestoreCheckpointId().isPresent());
    TaskStateManagerImpl nonEmptyStateManager = new TaskStateManagerImpl(new JobID(), new ExecutionAttemptID(), new TestTaskLocalStateStore(), null, new JobManagerTaskRestore(2, new TaskStateSnapshot()), new TestCheckpointResponder());
    Assert.assertEquals(2L, (long) nonEmptyStateManager.getRestoreCheckpointId().get());
}

Example 4 with TestCheckpointResponder

use of org.apache.flink.runtime.taskmanager.TestCheckpointResponder in project flink by apache.

the class MultipleInputStreamTaskTest method testTriggeringStopWithSavepointWithDrain.

@Test
public void testTriggeringStopWithSavepointWithDrain() throws Exception {
    SourceOperatorFactory<Integer> sourceOperatorFactory = new SourceOperatorFactory<>(new MockSource(Boundedness.CONTINUOUS_UNBOUNDED, 2), WatermarkStrategy.noWatermarks());
    CompletableFuture<Boolean> checkpointCompleted = new CompletableFuture<>();
    CheckpointResponder checkpointResponder = new TestCheckpointResponder() {

        @Override
        public void acknowledgeCheckpoint(JobID jobID, ExecutionAttemptID executionAttemptID, long checkpointId, CheckpointMetrics checkpointMetrics, TaskStateSnapshot subtaskState) {
            super.acknowledgeCheckpoint(jobID, executionAttemptID, checkpointId, checkpointMetrics, subtaskState);
            checkpointCompleted.complete(null);
        }
    };
    try (StreamTaskMailboxTestHarness<String> testHarness = new StreamTaskMailboxTestHarnessBuilder<>(MultipleInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO).setCollectNetworkEvents().modifyStreamConfig(config -> config.setCheckpointingEnabled(true)).modifyExecutionConfig(applyObjectReuse(objectReuse)).addInput(BasicTypeInfo.INT_TYPE_INFO).addInput(BasicTypeInfo.INT_TYPE_INFO).addInput(BasicTypeInfo.INT_TYPE_INFO).setTaskStateSnapshot(1, TaskStateSnapshot.FINISHED_ON_RESTORE).setupOperatorChain(new LifeCycleMonitorMultipleInputOperatorFactory()).finishForSingletonOperatorChain(StringSerializer.INSTANCE).setCheckpointResponder(checkpointResponder).build()) {
        CompletableFuture<Boolean> triggerResult = testHarness.streamTask.triggerCheckpointAsync(new CheckpointMetaData(2, 2), CheckpointOptions.alignedNoTimeout(SavepointType.terminate(SavepointFormatType.CANONICAL), CheckpointStorageLocationReference.getDefault()));
        checkpointCompleted.whenComplete((ignored, exception) -> testHarness.streamTask.notifyCheckpointCompleteAsync(2));
        testHarness.waitForTaskCompletion();
        testHarness.finishProcessing();
        assertTrue(triggerResult.isDone());
        assertTrue(triggerResult.get());
        assertTrue(checkpointCompleted.isDone());
    }
}

Example 5 with TestCheckpointResponder

use of org.apache.flink.runtime.taskmanager.TestCheckpointResponder in project flink by apache.

the class LocalStateForwardingTest method testReportingFromTaskStateManagerToResponderAndTaskLocalStateStore.

/**
 * This tests that state that was reported to the {@link
 * org.apache.flink.runtime.state.TaskStateManager} is also reported to {@link
 * org.apache.flink.runtime.taskmanager.CheckpointResponder} and {@link
 * TaskLocalStateStoreImpl}.
 */
@Test
public void testReportingFromTaskStateManagerToResponderAndTaskLocalStateStore() throws Exception {
    final JobID jobID = new JobID();
    final AllocationID allocationID = new AllocationID();
    final ExecutionAttemptID executionAttemptID = new ExecutionAttemptID();
    final CheckpointMetaData checkpointMetaData = new CheckpointMetaData(42L, 4711L);
    final CheckpointMetrics checkpointMetrics = new CheckpointMetrics();
    final int subtaskIdx = 42;
    JobVertexID jobVertexID = new JobVertexID();
    TaskStateSnapshot jmSnapshot = new TaskStateSnapshot();
    TaskStateSnapshot tmSnapshot = new TaskStateSnapshot();
    final AtomicBoolean jmReported = new AtomicBoolean(false);
    final AtomicBoolean tmReported = new AtomicBoolean(false);
    TestCheckpointResponder checkpointResponder = new TestCheckpointResponder() {

        @Override
        public void acknowledgeCheckpoint(JobID lJobID, ExecutionAttemptID lExecutionAttemptID, long lCheckpointId, CheckpointMetrics lCheckpointMetrics, TaskStateSnapshot lSubtaskState) {
            Assert.assertEquals(jobID, lJobID);
            Assert.assertEquals(executionAttemptID, lExecutionAttemptID);
            Assert.assertEquals(checkpointMetaData.getCheckpointId(), lCheckpointId);
            Assert.assertEquals(checkpointMetrics, lCheckpointMetrics);
            jmReported.set(true);
        }
    };
    Executor executor = Executors.directExecutor();
    LocalRecoveryDirectoryProviderImpl directoryProvider = new LocalRecoveryDirectoryProviderImpl(temporaryFolder.newFolder(), jobID, jobVertexID, subtaskIdx);
    LocalRecoveryConfig localRecoveryConfig = new LocalRecoveryConfig(directoryProvider);
    TaskLocalStateStore taskLocalStateStore = new TaskLocalStateStoreImpl(jobID, allocationID, jobVertexID, subtaskIdx, localRecoveryConfig, executor) {

        @Override
        public void storeLocalState(@Nonnegative long checkpointId, @Nullable TaskStateSnapshot localState) {
            Assert.assertEquals(tmSnapshot, localState);
            tmReported.set(true);
        }
    };
    StateChangelogStorage<?> stateChangelogStorage = new InMemoryStateChangelogStorage();
    TaskStateManagerImpl taskStateManager = new TaskStateManagerImpl(jobID, executionAttemptID, taskLocalStateStore, stateChangelogStorage, null, checkpointResponder);
    taskStateManager.reportTaskStateSnapshots(checkpointMetaData, checkpointMetrics, jmSnapshot, tmSnapshot);
    Assert.assertTrue("Reporting for JM state was not called.", jmReported.get());
    Assert.assertTrue("Reporting for TM state was not called.", tmReported.get());
}