Examples with CheckpointMetaData org.apache.flink.runtime.checkpoint.CheckpointMetaData used on opensource projects

Example 36 with CheckpointMetaData

use of org.apache.flink.runtime.checkpoint.CheckpointMetaData in project flink by apache.

the class StreamTaskTest method testSyncSavepointWithEndInput.

/**
 * Test for SyncSavepoint and EndInput interactions. Targets following scenarios scenarios:
 *
 * <ol>
 *   <li>Thread1: notify sync savepoint
 *   <li>Thread2: endInput
 *   <li>Thread1: confirm/abort/abortAsync
 *   <li>assert inputEnded: confirmed - no, abort/abortAsync - yes
 * </ol>
 */
private void testSyncSavepointWithEndInput(BiConsumerWithException<StreamTask<?, ?>, Long, IOException> savepointResult, SnapshotType checkpointType, boolean expectEndInput) throws Exception {
    StreamTaskMailboxTestHarness<String> harness = new StreamTaskMailboxTestHarnessBuilder<>(OneInputStreamTask::new, STRING_TYPE_INFO).addInput(STRING_TYPE_INFO).setupOutputForSingletonOperatorChain(new TestBoundedOneInputStreamOperator()).build();
    final long checkpointId = 1L;
    CountDownLatch savepointTriggeredLatch = new CountDownLatch(1);
    CountDownLatch inputEndedLatch = new CountDownLatch(1);
    MailboxExecutor executor = harness.streamTask.getMailboxExecutorFactory().createExecutor(MAX_PRIORITY);
    executor.execute(() -> {
        try {
            harness.streamTask.triggerCheckpointOnBarrier(new CheckpointMetaData(checkpointId, checkpointId), new CheckpointOptions(checkpointType, getDefault()), new CheckpointMetricsBuilder());
        } catch (IOException e) {
            fail(e.getMessage());
        }
    }, "triggerCheckpointOnBarrier");
    new Thread(() -> {
        try {
            savepointTriggeredLatch.await();
            harness.endInput(expectEndInput);
            inputEndedLatch.countDown();
        } catch (InterruptedException e) {
            fail(e.getMessage());
        }
    }).start();
    // this mails should be executed from the one above (from triggerCheckpointOnBarrier)
    executor.execute(savepointTriggeredLatch::countDown, "savepointTriggeredLatch");
    executor.execute(() -> {
        inputEndedLatch.await();
        savepointResult.accept(harness.streamTask, checkpointId);
    }, "savepointResult");
    harness.processAll();
    Assert.assertEquals(expectEndInput, TestBoundedOneInputStreamOperator.isInputEnded());
}

Example 37 with CheckpointMetaData

use of org.apache.flink.runtime.checkpoint.CheckpointMetaData in project flink by apache.

the class StreamTaskTest method testEmptySubtaskStateLeadsToStatelessAcknowledgment.

/**
 * FLINK-5985
 *
 * <p>This test ensures that empty snapshots (no op/keyed stated whatsoever) will be reported as
 * stateless tasks. This happens by translating an empty {@link SubtaskState} into reporting
 * 'null' to #acknowledgeCheckpoint.
 */
@Test
public void testEmptySubtaskStateLeadsToStatelessAcknowledgment() throws Exception {
    // latch blocks until the async checkpoint thread acknowledges
    final OneShotLatch checkpointCompletedLatch = new OneShotLatch();
    final List<SubtaskState> checkpointResult = new ArrayList<>(1);
    CheckpointResponder checkpointResponder = mock(CheckpointResponder.class);
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            SubtaskState subtaskState = invocation.getArgument(4);
            checkpointResult.add(subtaskState);
            checkpointCompletedLatch.trigger();
            return null;
        }
    }).when(checkpointResponder).acknowledgeCheckpoint(any(JobID.class), any(ExecutionAttemptID.class), anyLong(), any(CheckpointMetrics.class), nullable(TaskStateSnapshot.class));
    TaskStateManager taskStateManager = new TaskStateManagerImpl(new JobID(1L, 2L), new ExecutionAttemptID(), mock(TaskLocalStateStoreImpl.class), new InMemoryStateChangelogStorage(), null, checkpointResponder);
    // mock the operator with empty snapshot result (all state handles are null)
    OneInputStreamOperator<String, String> statelessOperator = streamOperatorWithSnapshot(new OperatorSnapshotFutures());
    try (MockEnvironment mockEnvironment = new MockEnvironmentBuilder().setTaskStateManager(taskStateManager).build()) {
        RunningTask<MockStreamTask> task = runTask(() -> createMockStreamTask(mockEnvironment, operatorChain(statelessOperator)));
        waitTaskIsRunning(task.streamTask, task.invocationFuture);
        task.streamTask.triggerCheckpointAsync(new CheckpointMetaData(42L, 1L), CheckpointOptions.forCheckpointWithDefaultLocation());
        checkpointCompletedLatch.await(30, TimeUnit.SECONDS);
        // ensure that 'null' was acknowledged as subtask state
        Assert.assertNull(checkpointResult.get(0));
        task.streamTask.cancel();
        task.waitForTaskCompletion(true);
    }
}

Example 38 with CheckpointMetaData

use of org.apache.flink.runtime.checkpoint.CheckpointMetaData in project flink by apache.

the class StreamTaskTest method testCheckpointFailueOnClosedOperator.

/**
 * Tests exeptions is thrown by triggering checkpoint if operators are closed. This was
 * initially implemented for FLINK-16383. However after FLINK-2491 operators lifecycle has
 * changed and now we: (1) redefined close() to dispose(). After closing operators, there should
 * be no opportunity to invoke anything on the task. close() mentioned in FLINK-16383 is now
 * more like finish(). (2) We support triggering and performing checkpoints if operators are
 * finished.
 */
@Test
public void testCheckpointFailueOnClosedOperator() throws Throwable {
    ClosingOperator<Integer> operator = new ClosingOperator<>();
    StreamTaskMailboxTestHarnessBuilder<Integer> builder = new StreamTaskMailboxTestHarnessBuilder<>(OneInputStreamTask::new, BasicTypeInfo.INT_TYPE_INFO).addInput(BasicTypeInfo.INT_TYPE_INFO);
    try (StreamTaskMailboxTestHarness<Integer> harness = builder.setupOutputForSingletonOperatorChain(operator).build()) {
        // keeps the mailbox from suspending
        harness.setAutoProcess(false);
        harness.processElement(new StreamRecord<>(1));
        harness.streamTask.operatorChain.finishOperators(harness.streamTask.getActionExecutor(), StopMode.DRAIN);
        harness.streamTask.operatorChain.closeAllOperators();
        assertTrue(ClosingOperator.closed.get());
        harness.streamTask.triggerCheckpointOnBarrier(new CheckpointMetaData(1, 0), CheckpointOptions.forCheckpointWithDefaultLocation(), new CheckpointMetricsBuilder());
    } catch (Exception ex) {
        ExceptionUtils.assertThrowableWithMessage(ex, "OperatorChain and Task should never be closed at this point");
    }
}

Example 39 with CheckpointMetaData

use of org.apache.flink.runtime.checkpoint.CheckpointMetaData in project flink by apache.

the class SubtaskCheckpointCoordinatorTest method testBroadcastCancelCheckpointMarkerOnAbortingFromCoordinator.

@Test
public void testBroadcastCancelCheckpointMarkerOnAbortingFromCoordinator() throws Exception {
    OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(OneInputStreamTask::new, 1, 1, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    streamConfig.setStreamOperator(new MapOperator());
    testHarness.invoke();
    testHarness.waitForTaskRunning();
    MockEnvironment mockEnvironment = MockEnvironment.builder().build();
    try (SubtaskCheckpointCoordinator subtaskCheckpointCoordinator = new MockSubtaskCheckpointCoordinatorBuilder().setEnvironment(mockEnvironment).build()) {
        ArrayList<Object> recordOrEvents = new ArrayList<>();
        StreamElementSerializer<String> stringStreamElementSerializer = new StreamElementSerializer<>(StringSerializer.INSTANCE);
        ResultPartitionWriter resultPartitionWriter = new RecordOrEventCollectingResultPartitionWriter<>(recordOrEvents, stringStreamElementSerializer);
        mockEnvironment.addOutputs(Collections.singletonList(resultPartitionWriter));
        OneInputStreamTask<String, String> task = testHarness.getTask();
        OperatorChain<String, OneInputStreamOperator<String, String>> operatorChain = new RegularOperatorChain<>(task, StreamTask.createRecordWriterDelegate(streamConfig, mockEnvironment));
        long checkpointId = 42L;
        // notify checkpoint aborted before execution.
        subtaskCheckpointCoordinator.notifyCheckpointAborted(checkpointId, operatorChain, () -> true);
        subtaskCheckpointCoordinator.checkpointState(new CheckpointMetaData(checkpointId, System.currentTimeMillis()), CheckpointOptions.forCheckpointWithDefaultLocation(), new CheckpointMetricsBuilder(), operatorChain, false, () -> false);
        assertEquals(1, recordOrEvents.size());
        Object recordOrEvent = recordOrEvents.get(0);
        // ensure CancelCheckpointMarker is broadcast downstream.
        assertTrue(recordOrEvent instanceof CancelCheckpointMarker);
        assertEquals(checkpointId, ((CancelCheckpointMarker) recordOrEvent).getCheckpointId());
        testHarness.endInput();
        testHarness.waitForTaskCompletion();
    }
}

Example 40 with CheckpointMetaData

use of org.apache.flink.runtime.checkpoint.CheckpointMetaData in project flink by apache.

the class SubtaskCheckpointCoordinatorTest method testNotifyCheckpointSubsumed.

@Test
public void testNotifyCheckpointSubsumed() throws Exception {
    TestTaskStateManager stateManager = new TestTaskStateManager();
    MockEnvironment mockEnvironment = MockEnvironment.builder().setTaskStateManager(stateManager).build();
    try (SubtaskCheckpointCoordinatorImpl subtaskCheckpointCoordinator = (SubtaskCheckpointCoordinatorImpl) new MockSubtaskCheckpointCoordinatorBuilder().setEnvironment(mockEnvironment).setUnalignedCheckpointEnabled(true).build()) {
        StreamMap<String, String> streamMap = new StreamMap<>((MapFunction<String, String>) value -> value);
        streamMap.setProcessingTimeService(new TestProcessingTimeService());
        final OperatorChain<String, AbstractStreamOperator<String>> operatorChain = operatorChain(streamMap);
        StreamTaskStateInitializerImpl stateInitializer = new StreamTaskStateInitializerImpl(mockEnvironment, new TestStateBackend());
        operatorChain.initializeStateAndOpenOperators(stateInitializer);
        long checkpointId = 42L;
        subtaskCheckpointCoordinator.getChannelStateWriter().start(checkpointId, CheckpointOptions.forCheckpointWithDefaultLocation());
        subtaskCheckpointCoordinator.checkpointState(new CheckpointMetaData(checkpointId, System.currentTimeMillis()), CheckpointOptions.forCheckpointWithDefaultLocation(), new CheckpointMetricsBuilder(), operatorChain, false, () -> false);
        long notifySubsumeCheckpointId = checkpointId + 1L;
        // notify checkpoint aborted before execution.
        subtaskCheckpointCoordinator.notifyCheckpointSubsumed(notifySubsumeCheckpointId, operatorChain, () -> true);
        assertEquals(notifySubsumeCheckpointId, ((TestStateBackend.TestKeyedStateBackend<?>) streamMap.getKeyedStateBackend()).getSubsumeCheckpointId());
    }
}