Examples with CheckpointMetricsBuilder org.apache.flink.runtime.checkpoint.CheckpointMetricsBuilder used on opensource projects

Example 11 with CheckpointMetricsBuilder

use of org.apache.flink.runtime.checkpoint.CheckpointMetricsBuilder in project flink by apache.

the class SubtaskCheckpointCoordinatorTest method testBroadcastCancelCheckpointMarkerOnAbortingFromCoordinator.

@Test
public void testBroadcastCancelCheckpointMarkerOnAbortingFromCoordinator() throws Exception {
    OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(OneInputStreamTask::new, 1, 1, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    streamConfig.setStreamOperator(new MapOperator());
    testHarness.invoke();
    testHarness.waitForTaskRunning();
    MockEnvironment mockEnvironment = MockEnvironment.builder().build();
    try (SubtaskCheckpointCoordinator subtaskCheckpointCoordinator = new MockSubtaskCheckpointCoordinatorBuilder().setEnvironment(mockEnvironment).build()) {
        ArrayList<Object> recordOrEvents = new ArrayList<>();
        StreamElementSerializer<String> stringStreamElementSerializer = new StreamElementSerializer<>(StringSerializer.INSTANCE);
        ResultPartitionWriter resultPartitionWriter = new RecordOrEventCollectingResultPartitionWriter<>(recordOrEvents, stringStreamElementSerializer);
        mockEnvironment.addOutputs(Collections.singletonList(resultPartitionWriter));
        OneInputStreamTask<String, String> task = testHarness.getTask();
        OperatorChain<String, OneInputStreamOperator<String, String>> operatorChain = new RegularOperatorChain<>(task, StreamTask.createRecordWriterDelegate(streamConfig, mockEnvironment));
        long checkpointId = 42L;
        // notify checkpoint aborted before execution.
        subtaskCheckpointCoordinator.notifyCheckpointAborted(checkpointId, operatorChain, () -> true);
        subtaskCheckpointCoordinator.checkpointState(new CheckpointMetaData(checkpointId, System.currentTimeMillis()), CheckpointOptions.forCheckpointWithDefaultLocation(), new CheckpointMetricsBuilder(), operatorChain, false, () -> false);
        assertEquals(1, recordOrEvents.size());
        Object recordOrEvent = recordOrEvents.get(0);
        // ensure CancelCheckpointMarker is broadcast downstream.
        assertTrue(recordOrEvent instanceof CancelCheckpointMarker);
        assertEquals(checkpointId, ((CancelCheckpointMarker) recordOrEvent).getCheckpointId());
        testHarness.endInput();
        testHarness.waitForTaskCompletion();
    }
}

Example 12 with CheckpointMetricsBuilder

use of org.apache.flink.runtime.checkpoint.CheckpointMetricsBuilder in project flink by apache.

the class SubtaskCheckpointCoordinatorTest method testNotifyCheckpointSubsumed.

@Test
public void testNotifyCheckpointSubsumed() throws Exception {
    TestTaskStateManager stateManager = new TestTaskStateManager();
    MockEnvironment mockEnvironment = MockEnvironment.builder().setTaskStateManager(stateManager).build();
    try (SubtaskCheckpointCoordinatorImpl subtaskCheckpointCoordinator = (SubtaskCheckpointCoordinatorImpl) new MockSubtaskCheckpointCoordinatorBuilder().setEnvironment(mockEnvironment).setUnalignedCheckpointEnabled(true).build()) {
        StreamMap<String, String> streamMap = new StreamMap<>((MapFunction<String, String>) value -> value);
        streamMap.setProcessingTimeService(new TestProcessingTimeService());
        final OperatorChain<String, AbstractStreamOperator<String>> operatorChain = operatorChain(streamMap);
        StreamTaskStateInitializerImpl stateInitializer = new StreamTaskStateInitializerImpl(mockEnvironment, new TestStateBackend());
        operatorChain.initializeStateAndOpenOperators(stateInitializer);
        long checkpointId = 42L;
        subtaskCheckpointCoordinator.getChannelStateWriter().start(checkpointId, CheckpointOptions.forCheckpointWithDefaultLocation());
        subtaskCheckpointCoordinator.checkpointState(new CheckpointMetaData(checkpointId, System.currentTimeMillis()), CheckpointOptions.forCheckpointWithDefaultLocation(), new CheckpointMetricsBuilder(), operatorChain, false, () -> false);
        long notifySubsumeCheckpointId = checkpointId + 1L;
        // notify checkpoint aborted before execution.
        subtaskCheckpointCoordinator.notifyCheckpointSubsumed(notifySubsumeCheckpointId, operatorChain, () -> true);
        assertEquals(notifySubsumeCheckpointId, ((TestStateBackend.TestKeyedStateBackend<?>) streamMap.getKeyedStateBackend()).getSubsumeCheckpointId());
    }
}

Example 13 with CheckpointMetricsBuilder

use of org.apache.flink.runtime.checkpoint.CheckpointMetricsBuilder in project flink by apache.

the class SubtaskCheckpointCoordinatorTest method testNotifyCheckpointAbortedBeforeAsyncPhase.

@Test
public void testNotifyCheckpointAbortedBeforeAsyncPhase() throws Exception {
    TestTaskStateManager stateManager = new TestTaskStateManager();
    MockEnvironment mockEnvironment = MockEnvironment.builder().setTaskStateManager(stateManager).build();
    try (SubtaskCheckpointCoordinatorImpl subtaskCheckpointCoordinator = (SubtaskCheckpointCoordinatorImpl) new MockSubtaskCheckpointCoordinatorBuilder().setEnvironment(mockEnvironment).setUnalignedCheckpointEnabled(true).build()) {
        CheckpointOperator checkpointOperator = new CheckpointOperator(new OperatorSnapshotFutures());
        final OperatorChain<String, AbstractStreamOperator<String>> operatorChain = operatorChain(checkpointOperator);
        long checkpointId = 42L;
        // notify checkpoint aborted before execution.
        subtaskCheckpointCoordinator.notifyCheckpointAborted(checkpointId, operatorChain, () -> true);
        assertEquals(1, subtaskCheckpointCoordinator.getAbortedCheckpointSize());
        subtaskCheckpointCoordinator.getChannelStateWriter().start(checkpointId, CheckpointOptions.forCheckpointWithDefaultLocation());
        subtaskCheckpointCoordinator.checkpointState(new CheckpointMetaData(checkpointId, System.currentTimeMillis()), CheckpointOptions.forCheckpointWithDefaultLocation(), new CheckpointMetricsBuilder(), operatorChain, false, () -> false);
        assertFalse(checkpointOperator.isCheckpointed());
        assertEquals(-1, stateManager.getReportedCheckpointId());
        assertEquals(0, subtaskCheckpointCoordinator.getAbortedCheckpointSize());
        assertEquals(0, subtaskCheckpointCoordinator.getAsyncCheckpointRunnableSize());
    }
}

Example 14 with CheckpointMetricsBuilder

use of org.apache.flink.runtime.checkpoint.CheckpointMetricsBuilder in project flink by apache.

the class StreamTaskFinalCheckpointsTest method testCheckpointDoneOnFinishedOperator.

@Test
public void testCheckpointDoneOnFinishedOperator() throws Exception {
    FinishingOperator finishingOperator = new FinishingOperator();
    StreamTaskMailboxTestHarnessBuilder<Integer> builder = new StreamTaskMailboxTestHarnessBuilder<>(OneInputStreamTask::new, BasicTypeInfo.INT_TYPE_INFO).addInput(BasicTypeInfo.INT_TYPE_INFO);
    StreamTaskMailboxTestHarness<Integer> harness = builder.setupOutputForSingletonOperatorChain(finishingOperator).build();
    // keeps the mailbox from suspending
    harness.setAutoProcess(false);
    harness.processElement(new StreamRecord<>(1));
    harness.streamTask.operatorChain.finishOperators(harness.streamTask.getActionExecutor(), StopMode.DRAIN);
    assertTrue(FinishingOperator.finished);
    harness.getTaskStateManager().getWaitForReportLatch().reset();
    harness.streamTask.triggerCheckpointOnBarrier(new CheckpointMetaData(2, 0), CheckpointOptions.forCheckpointWithDefaultLocation(), new CheckpointMetricsBuilder().setBytesProcessedDuringAlignment(0L).setAlignmentDurationNanos(0L));
    harness.getTaskStateManager().getWaitForReportLatch().await();
    assertEquals(2, harness.getTaskStateManager().getReportedCheckpointId());
}

Example 15 with CheckpointMetricsBuilder

use of org.apache.flink.runtime.checkpoint.CheckpointMetricsBuilder in project flink by apache.

the class CheckpointBarrierHandler method notifyCheckpoint.

protected void notifyCheckpoint(CheckpointBarrier checkpointBarrier) throws IOException {
    CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointBarrier.getId(), checkpointBarrier.getTimestamp(), System.currentTimeMillis());
    CheckpointMetricsBuilder checkpointMetrics;
    if (checkpointBarrier.getId() == startAlignmentCheckpointId) {
        checkpointMetrics = new CheckpointMetricsBuilder().setAlignmentDurationNanos(latestAlignmentDurationNanos).setBytesProcessedDuringAlignment(latestBytesProcessedDuringAlignment).setCheckpointStartDelayNanos(latestCheckpointStartDelayNanos);
    } else {
        checkpointMetrics = new CheckpointMetricsBuilder().setAlignmentDurationNanos(0L).setBytesProcessedDuringAlignment(0L).setCheckpointStartDelayNanos(0);
    }
    toNotifyOnCheckpoint.triggerCheckpointOnBarrier(checkpointMetaData, checkpointBarrier.getCheckpointOptions(), checkpointMetrics);
}