Examples with CheckpointMetaData org.apache.flink.runtime.checkpoint.CheckpointMetaData used on opensource projects

Example 21 with CheckpointMetaData

use of org.apache.flink.runtime.checkpoint.CheckpointMetaData in project flink by apache.

the class SourceOperatorStreamTaskTest method triggerCheckpointWaitForFinish.

private void triggerCheckpointWaitForFinish(StreamTaskMailboxTestHarness<Integer> testHarness, long checkpointId, CheckpointOptions checkpointOptions) throws Exception {
    // Trigger a checkpoint.
    testHarness.taskStateManager.getWaitForReportLatch().reset();
    CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, checkpointId);
    Future<Boolean> checkpointFuture = testHarness.getStreamTask().triggerCheckpointAsync(checkpointMetaData, checkpointOptions);
    // Wait until the checkpoint finishes.
    // We have to mark the source reader as available here, otherwise the runMailboxStep() call
    // after
    // checkpoint is completed will block.
    getSourceReaderFromTask(testHarness).markAvailable();
    processUntil(testHarness, checkpointFuture::isDone);
    Future<Void> checkpointNotified = testHarness.getStreamTask().notifyCheckpointCompleteAsync(checkpointId);
    processUntil(testHarness, checkpointNotified::isDone);
    testHarness.taskStateManager.getWaitForReportLatch().await();
}

Example 22 with CheckpointMetaData

use of org.apache.flink.runtime.checkpoint.CheckpointMetaData in project flink by apache.

the class StreamTaskFinalCheckpointsTest method testWaitingForPendingCheckpointsOnFinished.

@Test
public void testWaitingForPendingCheckpointsOnFinished() throws Exception {
    long delayedCheckpointId = 2;
    CompletingCheckpointResponder responder = new CompletingCheckpointResponder() {

        @Override
        public void acknowledgeCheckpoint(JobID jobID, ExecutionAttemptID executionAttemptID, long checkpointId, CheckpointMetrics checkpointMetrics, TaskStateSnapshot subtaskState) {
            if (delayedCheckpointId == checkpointId) {
                try {
                    // Give some potential time for the task to finish before the
                    // checkpoint is acknowledged, also do not notify its completion
                    Thread.sleep(CONCURRENT_EVENT_WAIT_PERIOD_MS);
                } catch (InterruptedException e) {
                    throw new FlinkRuntimeException(e);
                }
            } else {
                super.acknowledgeCheckpoint(jobID, executionAttemptID, checkpointId, checkpointMetrics, subtaskState);
            }
        }
    };
    try (StreamTaskMailboxTestHarness<String> harness = createTestHarness(responder)) {
        // finish all data
        harness.waitForTaskCompletion();
        // trigger the final checkpoint
        harness.streamTask.triggerCheckpointOnBarrier(new CheckpointMetaData(1, 101), CheckpointOptions.forCheckpointWithDefaultLocation(), new CheckpointMetricsBuilder().setBytesProcessedDuringAlignment(0L).setAlignmentDurationNanos(0L));
        // trigger another checkpoint that we want to complete before finishing the task
        harness.streamTask.triggerCheckpointOnBarrier(new CheckpointMetaData(delayedCheckpointId, 101), CheckpointOptions.forCheckpointWithDefaultLocation(), new CheckpointMetricsBuilder().setBytesProcessedDuringAlignment(0L).setAlignmentDurationNanos(0L));
        harness.processAll();
        harness.finishProcessing();
        assertEquals(delayedCheckpointId, harness.getTaskStateManager().getReportedCheckpointId());
    }
}

Example 23 with CheckpointMetaData

use of org.apache.flink.runtime.checkpoint.CheckpointMetaData in project flink by apache.

the class SubtaskCheckpointCoordinatorTest method testNotifyCheckpointAbortedDuringAsyncPhase.

@Test
public void testNotifyCheckpointAbortedDuringAsyncPhase() throws Exception {
    MockEnvironment mockEnvironment = MockEnvironment.builder().build();
    try (SubtaskCheckpointCoordinatorImpl subtaskCheckpointCoordinator = (SubtaskCheckpointCoordinatorImpl) new MockSubtaskCheckpointCoordinatorBuilder().setEnvironment(mockEnvironment).setExecutor(Executors.newSingleThreadExecutor()).setUnalignedCheckpointEnabled(true).build()) {
        final BlockingRunnableFuture rawKeyedStateHandleFuture = new BlockingRunnableFuture();
        OperatorSnapshotFutures operatorSnapshotResult = new OperatorSnapshotFutures(DoneFuture.of(SnapshotResult.empty()), rawKeyedStateHandleFuture, DoneFuture.of(SnapshotResult.empty()), DoneFuture.of(SnapshotResult.empty()), DoneFuture.of(SnapshotResult.empty()), DoneFuture.of(SnapshotResult.empty()));
        final OperatorChain<String, AbstractStreamOperator<String>> operatorChain = operatorChain(new CheckpointOperator(operatorSnapshotResult));
        long checkpointId = 42L;
        subtaskCheckpointCoordinator.getChannelStateWriter().start(checkpointId, CheckpointOptions.forCheckpointWithDefaultLocation());
        subtaskCheckpointCoordinator.checkpointState(new CheckpointMetaData(checkpointId, System.currentTimeMillis()), CheckpointOptions.forCheckpointWithDefaultLocation(), new CheckpointMetricsBuilder(), operatorChain, false, () -> false);
        rawKeyedStateHandleFuture.awaitRun();
        assertEquals(1, subtaskCheckpointCoordinator.getAsyncCheckpointRunnableSize());
        assertFalse(rawKeyedStateHandleFuture.isCancelled());
        subtaskCheckpointCoordinator.notifyCheckpointAborted(checkpointId, operatorChain, () -> true);
        assertTrue(rawKeyedStateHandleFuture.isCancelled());
        assertEquals(0, subtaskCheckpointCoordinator.getAsyncCheckpointRunnableSize());
    }
}

Example 24 with CheckpointMetaData

use of org.apache.flink.runtime.checkpoint.CheckpointMetaData in project flink by apache.

the class SubtaskCheckpointCoordinatorTest method testNotifyCheckpointAbortedAfterAsyncPhase.

@Test
public void testNotifyCheckpointAbortedAfterAsyncPhase() throws Exception {
    TestTaskStateManager stateManager = new TestTaskStateManager();
    MockEnvironment mockEnvironment = MockEnvironment.builder().setTaskStateManager(stateManager).build();
    try (SubtaskCheckpointCoordinatorImpl subtaskCheckpointCoordinator = (SubtaskCheckpointCoordinatorImpl) new MockSubtaskCheckpointCoordinatorBuilder().setEnvironment(mockEnvironment).build()) {
        final OperatorChain<?, ?> operatorChain = getOperatorChain(mockEnvironment);
        long checkpointId = 42L;
        subtaskCheckpointCoordinator.checkpointState(new CheckpointMetaData(checkpointId, System.currentTimeMillis()), CheckpointOptions.forCheckpointWithDefaultLocation(), new CheckpointMetricsBuilder(), operatorChain, false, () -> false);
        subtaskCheckpointCoordinator.notifyCheckpointAborted(checkpointId, operatorChain, () -> true);
        assertEquals(0, subtaskCheckpointCoordinator.getAbortedCheckpointSize());
        assertEquals(checkpointId, stateManager.getNotifiedAbortedCheckpointId());
    }
}

Example 25 with CheckpointMetaData

use of org.apache.flink.runtime.checkpoint.CheckpointMetaData in project flink by apache.

the class SubtaskCheckpointCoordinatorTest method testSavepointNotResultingInPriorityEvents.

@Test
public void testSavepointNotResultingInPriorityEvents() throws Exception {
    MockEnvironment mockEnvironment = MockEnvironment.builder().build();
    try (SubtaskCheckpointCoordinator coordinator = new MockSubtaskCheckpointCoordinatorBuilder().setUnalignedCheckpointEnabled(true).setEnvironment(mockEnvironment).build()) {
        AtomicReference<Boolean> broadcastedPriorityEvent = new AtomicReference<>(null);
        final OperatorChain<?, ?> operatorChain = new RegularOperatorChain(new MockStreamTaskBuilder(mockEnvironment).build(), new NonRecordWriter<>()) {

            @Override
            public void broadcastEvent(AbstractEvent event, boolean isPriorityEvent) throws IOException {
                super.broadcastEvent(event, isPriorityEvent);
                broadcastedPriorityEvent.set(isPriorityEvent);
            }
        };
        coordinator.checkpointState(new CheckpointMetaData(0, 0), new CheckpointOptions(SavepointType.savepoint(SavepointFormatType.CANONICAL), CheckpointStorageLocationReference.getDefault()), new CheckpointMetricsBuilder(), operatorChain, false, () -> true);
        assertEquals(false, broadcastedPriorityEvent.get());
    }
}