Examples with CheckpointMetaData org.apache.flink.runtime.checkpoint.CheckpointMetaData used on opensource projects

Example 26 with CheckpointMetaData

use of org.apache.flink.runtime.checkpoint.CheckpointMetaData in project flink by apache.

the class SubtaskCheckpointCoordinatorTest method testForceAlignedCheckpointResultingInPriorityEvents.

@Test
public void testForceAlignedCheckpointResultingInPriorityEvents() throws Exception {
    final long checkpointId = 42L;
    MockEnvironment mockEnvironment = MockEnvironment.builder().build();
    try (SubtaskCheckpointCoordinator coordinator = new MockSubtaskCheckpointCoordinatorBuilder().setUnalignedCheckpointEnabled(true).setEnvironment(mockEnvironment).build()) {
        AtomicReference<Boolean> broadcastedPriorityEvent = new AtomicReference<>(null);
        final OperatorChain<?, ?> operatorChain = new RegularOperatorChain(new MockStreamTaskBuilder(mockEnvironment).build(), new NonRecordWriter<>()) {

            @Override
            public void broadcastEvent(AbstractEvent event, boolean isPriorityEvent) throws IOException {
                super.broadcastEvent(event, isPriorityEvent);
                broadcastedPriorityEvent.set(isPriorityEvent);
                // test if we can write output data
                coordinator.getChannelStateWriter().addOutputData(checkpointId, new ResultSubpartitionInfo(0, 0), 0, BufferBuilderTestUtils.buildSomeBuffer(500));
            }
        };
        CheckpointOptions forcedAlignedOptions = CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, CheckpointStorageLocationReference.getDefault()).withUnalignedUnsupported();
        coordinator.checkpointState(new CheckpointMetaData(checkpointId, 0), forcedAlignedOptions, new CheckpointMetricsBuilder(), operatorChain, false, () -> true);
        assertEquals(true, broadcastedPriorityEvent.get());
    }
}

Example 27 with CheckpointMetaData

use of org.apache.flink.runtime.checkpoint.CheckpointMetaData in project flink by apache.

the class SynchronousCheckpointTest method launchSynchronousSavepointAndWaitForSyncSavepointIdToBeSet.

private void launchSynchronousSavepointAndWaitForSyncSavepointIdToBeSet() throws InterruptedException {
    streamTaskUnderTest.triggerCheckpointAsync(new CheckpointMetaData(42, System.currentTimeMillis()), new CheckpointOptions(SavepointType.suspend(SavepointFormatType.CANONICAL), CheckpointStorageLocationReference.getDefault()));
    waitForSyncSavepointIdToBeSet(streamTaskUnderTest);
}

Example 28 with CheckpointMetaData

use of org.apache.flink.runtime.checkpoint.CheckpointMetaData in project flink by apache.

the class BarrierTracker method notifyCheckpoint.

private void notifyCheckpoint(long checkpointId, long timestamp, CheckpointOptions checkpointOptions) throws Exception {
    if (toNotifyOnCheckpoint != null) {
        CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, timestamp);
        CheckpointMetrics checkpointMetrics = new CheckpointMetrics().setBytesBufferedInAlignment(0L).setAlignmentDurationNanos(0L);
        toNotifyOnCheckpoint.triggerCheckpointOnBarrier(checkpointMetaData, checkpointOptions, checkpointMetrics);
    }
}

Example 29 with CheckpointMetaData

use of org.apache.flink.runtime.checkpoint.CheckpointMetaData in project flink by apache.

the class StreamTaskTest method testFailingCheckpointStreamOperator.

@Test
public void testFailingCheckpointStreamOperator() throws Exception {
    final long checkpointId = 42L;
    final long timestamp = 1L;
    TaskInfo mockTaskInfo = mock(TaskInfo.class);
    when(mockTaskInfo.getTaskNameWithSubtasks()).thenReturn("foobar");
    when(mockTaskInfo.getIndexOfThisSubtask()).thenReturn(0);
    Environment mockEnvironment = mock(Environment.class);
    when(mockEnvironment.getTaskInfo()).thenReturn(mockTaskInfo);
    StreamTask<?, AbstractStreamOperator<?>> streamTask = mock(StreamTask.class, Mockito.CALLS_REAL_METHODS);
    CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, timestamp);
    streamTask.setEnvironment(mockEnvironment);
    // mock the operators
    StreamOperator<?> streamOperator1 = mock(StreamOperator.class, withSettings().extraInterfaces(StreamCheckpointedOperator.class));
    StreamOperator<?> streamOperator2 = mock(StreamOperator.class, withSettings().extraInterfaces(StreamCheckpointedOperator.class));
    StreamOperator<?> streamOperator3 = mock(StreamOperator.class, withSettings().extraInterfaces(StreamCheckpointedOperator.class));
    // mock the returned snapshots
    OperatorSnapshotResult operatorSnapshotResult1 = mock(OperatorSnapshotResult.class);
    OperatorSnapshotResult operatorSnapshotResult2 = mock(OperatorSnapshotResult.class);
    final Exception testException = new Exception("Test exception");
    when(streamOperator1.snapshotState(anyLong(), anyLong(), any(CheckpointOptions.class))).thenReturn(operatorSnapshotResult1);
    when(streamOperator2.snapshotState(anyLong(), anyLong(), any(CheckpointOptions.class))).thenReturn(operatorSnapshotResult2);
    when(streamOperator3.snapshotState(anyLong(), anyLong(), any(CheckpointOptions.class))).thenThrow(testException);
    // mock the returned legacy snapshots
    StreamStateHandle streamStateHandle1 = mock(StreamStateHandle.class);
    StreamStateHandle streamStateHandle2 = mock(StreamStateHandle.class);
    StreamStateHandle streamStateHandle3 = mock(StreamStateHandle.class);
    when(streamOperator1.snapshotLegacyOperatorState(anyLong(), anyLong(), any(CheckpointOptions.class))).thenReturn(streamStateHandle1);
    when(streamOperator2.snapshotLegacyOperatorState(anyLong(), anyLong(), any(CheckpointOptions.class))).thenReturn(streamStateHandle2);
    when(streamOperator3.snapshotLegacyOperatorState(anyLong(), anyLong(), any(CheckpointOptions.class))).thenReturn(streamStateHandle3);
    // set up the task
    StreamOperator<?>[] streamOperators = { streamOperator1, streamOperator2, streamOperator3 };
    OperatorChain<Void, AbstractStreamOperator<Void>> operatorChain = mock(OperatorChain.class);
    when(operatorChain.getAllOperators()).thenReturn(streamOperators);
    Whitebox.setInternalState(streamTask, "isRunning", true);
    Whitebox.setInternalState(streamTask, "lock", new Object());
    Whitebox.setInternalState(streamTask, "operatorChain", operatorChain);
    Whitebox.setInternalState(streamTask, "cancelables", new CloseableRegistry());
    Whitebox.setInternalState(streamTask, "configuration", new StreamConfig(new Configuration()));
    try {
        streamTask.triggerCheckpoint(checkpointMetaData, CheckpointOptions.forFullCheckpoint());
        fail("Expected test exception here.");
    } catch (Exception e) {
        assertEquals(testException, e.getCause());
    }
    verify(operatorSnapshotResult1).cancel();
    verify(operatorSnapshotResult2).cancel();
    verify(streamStateHandle1).discardState();
    verify(streamStateHandle2).discardState();
    verify(streamStateHandle3).discardState();
}

Example 30 with CheckpointMetaData

use of org.apache.flink.runtime.checkpoint.CheckpointMetaData in project flink by apache.

the class StreamTaskCancellationBarrierTest method testEmitCancellationBarrierWhenNotReady.

/**
	 * This test checks that tasks emit a proper cancel checkpoint barrier, if a "trigger checkpoint" message
	 * comes before they are ready.
	 */
@Test
public void testEmitCancellationBarrierWhenNotReady() throws Exception {
    StreamTask<String, ?> task = new InitBlockingTask();
    StreamTaskTestHarness<String> testHarness = new StreamTaskTestHarness<>(task, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    // start the test - this cannot succeed across the 'init()' method
    testHarness.invoke();
    // tell the task to commence a checkpoint
    boolean result = task.triggerCheckpoint(new CheckpointMetaData(41L, System.currentTimeMillis()), CheckpointOptions.forFullCheckpoint());
    assertFalse("task triggered checkpoint though not ready", result);
    // a cancellation barrier should be downstream
    Object emitted = testHarness.getOutput().poll();
    assertNotNull("nothing emitted", emitted);
    assertTrue("wrong type emitted", emitted instanceof CancelCheckpointMarker);
    assertEquals("wrong checkpoint id", 41L, ((CancelCheckpointMarker) emitted).getCheckpointId());
}