Examples with CheckpointOptions org.apache.flink.runtime.checkpoint.CheckpointOptions used on opensource projects

Example 36 with CheckpointOptions

use of org.apache.flink.runtime.checkpoint.CheckpointOptions in project flink by apache.

the class AlternatingCheckpointsTest method testBarrierHandling.

private void testBarrierHandling(SnapshotType checkpointType) throws Exception {
    final long barrierId = 123L;
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    SingleInputGate gate = new SingleInputGateBuilder().setNumberOfChannels(2).build();
    TestInputChannel fast = new TestInputChannel(gate, 0, false, true);
    TestInputChannel slow = new TestInputChannel(gate, 1, false, true);
    gate.setInputChannels(fast, slow);
    SingleCheckpointBarrierHandler barrierHandler = getTestBarrierHandlerFactory(target).create(gate);
    CheckpointedInputGate checkpointedGate = new CheckpointedInputGate(gate, barrierHandler, new SyncMailboxExecutor());
    if (checkpointType.isSavepoint()) {
        fast.setBlocked(true);
        slow.setBlocked(true);
    }
    CheckpointOptions options = checkpointType.isSavepoint() ? alignedNoTimeout(checkpointType, getDefault()) : unaligned(CheckpointType.CHECKPOINT, getDefault());
    Buffer barrier = barrier(barrierId, 1, options);
    send(barrier.retainBuffer(), fast, checkpointedGate);
    assertEquals(checkpointType.isSavepoint(), target.triggeredCheckpoints.isEmpty());
    send(barrier.retainBuffer(), slow, checkpointedGate);
    assertEquals(singletonList(barrierId), target.triggeredCheckpoints);
    if (checkpointType.isSavepoint()) {
        for (InputChannel channel : gate.getInputChannels().values()) {
            assertFalse(String.format("channel %d should be resumed", channel.getChannelIndex()), ((TestInputChannel) channel).isBlocked());
        }
    }
}

Example 37 with CheckpointOptions

use of org.apache.flink.runtime.checkpoint.CheckpointOptions in project flink by apache.

the class AlternatingCheckpointsTest method testTimeoutAlignmentWhenLocalBarrierFirst.

@Test
public void testTimeoutAlignmentWhenLocalBarrierFirst() throws Exception {
    // given: Gate with remote and local channels.
    int numChannels = 3;
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(numChannels, getTestBarrierHandlerFactory(target)).withMixedChannels(0).withMailboxExecutor().build();
    long alignmentTimeout = 100;
    Buffer checkpointBarrier = withTimeout(1, alignmentTimeout);
    // when: Execute the first checkpoint when announcement received first.
    ((TestInputChannel) gate.getChannel(0)).read(checkpointBarrier.retainBuffer());
    (getChannel(gate, 1)).onBuffer(checkpointBarrier.retainBuffer(), 0, 0);
    (getChannel(gate, 2)).onBuffer(checkpointBarrier.retainBuffer(), 0, 0);
    assertAnnouncement(gate);
    assertAnnouncement(gate);
    assertBarrier(gate);
    assertBarrier(gate);
    assertBarrier(gate);
    // then: The checkpoint executed successfully.
    assertEquals(1, target.getTriggeredCheckpointCounter());
    // given: The time in the future.
    clock.advanceTime(alignmentTimeout + 1, TimeUnit.MILLISECONDS);
    checkpointBarrier = withTimeout(2, alignmentTimeout);
    // when: Execute the second checkpoint when barrier from local channel without announcement
    // received first.
    ((TestInputChannel) gate.getChannel(0)).read(checkpointBarrier.retainBuffer());
    assertBarrier(gate);
    // then: Nothing happens because the alignment timeout should only start after this barrier.
    assertEquals(1, target.getTriggeredCheckpointCounter());
    // when: Receiving the barrier from second channel(with/without) announcement after time
    // more than alignment timeout.
    clock.advanceTime(alignmentTimeout + 1, TimeUnit.MILLISECONDS);
    (getChannel(gate, 1)).onBuffer(checkpointBarrier.retainBuffer(), 1, 0);
    assertAnnouncement(gate);
    assertBarrier(gate);
    // then: The checkpoint should started as unaligned.
    assertEquals(2, target.getTriggeredCheckpointCounter());
    List<CheckpointOptions> checkpointOptions = target.getTriggeredCheckpointOptions();
    assertEquals(AlignmentType.UNALIGNED, checkpointOptions.get(checkpointOptions.size() - 1).getAlignment());
}

Example 38 with CheckpointOptions

use of org.apache.flink.runtime.checkpoint.CheckpointOptions in project flink by apache.

the class MultipleInputStreamTaskChainedSourcesCheckpointingTest method createBarrier.

private CheckpointBarrier createBarrier(StreamTaskMailboxTestHarness<String> testHarness) {
    StreamConfig config = testHarness.getStreamTask().getConfiguration();
    CheckpointOptions checkpointOptions = CheckpointOptions.forConfig(CheckpointType.CHECKPOINT, CheckpointStorageLocationReference.getDefault(), config.isExactlyOnceCheckpointMode(), config.isUnalignedCheckpointsEnabled(), config.getAlignedCheckpointTimeout().toMillis());
    return new CheckpointBarrier(metaData.getCheckpointId(), metaData.getTimestamp(), checkpointOptions);
}

Example 39 with CheckpointOptions

use of org.apache.flink.runtime.checkpoint.CheckpointOptions in project flink by apache.

the class MultipleInputStreamTaskTest method testTriggeringCheckpointWithFinishedChannels.

private void testTriggeringCheckpointWithFinishedChannels(CheckpointOptions checkpointOptions) throws Exception {
    ResultPartition[] partitionWriters = new ResultPartition[2];
    try {
        for (int i = 0; i < partitionWriters.length; ++i) {
            partitionWriters[i] = PartitionTestUtils.createPartition(ResultPartitionType.PIPELINED_BOUNDED);
            partitionWriters[i].setup();
        }
        CompletingCheckpointResponder checkpointResponder = new CompletingCheckpointResponder();
        try (StreamTaskMailboxTestHarness<String> testHarness = new StreamTaskMailboxTestHarnessBuilder<>(MultipleInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO).addInput(BasicTypeInfo.STRING_TYPE_INFO).addInput(BasicTypeInfo.INT_TYPE_INFO).addInput(BasicTypeInfo.DOUBLE_TYPE_INFO).addAdditionalOutput(partitionWriters).setCheckpointResponder(checkpointResponder).modifyStreamConfig(config -> {
            config.setCheckpointingEnabled(true);
            config.setUnalignedCheckpointsEnabled(checkpointOptions.isUnalignedCheckpoint() || checkpointOptions.isTimeoutable());
        }).setupOperatorChain(new MapToStringMultipleInputOperatorFactory(3)).finishForSingletonOperatorChain(StringSerializer.INSTANCE).build()) {
            checkpointResponder.setHandlers(testHarness.streamTask::notifyCheckpointCompleteAsync, testHarness.streamTask::notifyCheckpointAbortAsync);
            testHarness.getStreamTask().getCheckpointBarrierHandler().get();
            // Tests triggering checkpoint when all the inputs are alive.
            CompletableFuture<Boolean> checkpointFuture = triggerCheckpoint(testHarness, 2, checkpointOptions);
            processMailTillCheckpointSucceeds(testHarness, checkpointFuture);
            assertEquals(2, testHarness.getTaskStateManager().getReportedCheckpointId());
            // Tests triggering checkpoint after some inputs have received EndOfPartition.
            testHarness.processEvent(new EndOfData(StopMode.DRAIN), 0, 0);
            testHarness.processEvent(EndOfPartitionEvent.INSTANCE, 0, 0);
            checkpointFuture = triggerCheckpoint(testHarness, 4, checkpointOptions);
            processMailTillCheckpointSucceeds(testHarness, checkpointFuture);
            assertEquals(4, testHarness.getTaskStateManager().getReportedCheckpointId());
            // Tests triggering checkpoint after all the inputs have received EndOfPartition.
            testHarness.processEvent(new EndOfData(StopMode.DRAIN), 1, 0);
            testHarness.processEvent(new EndOfData(StopMode.DRAIN), 2, 0);
            testHarness.processEvent(EndOfPartitionEvent.INSTANCE, 1, 0);
            testHarness.processEvent(EndOfPartitionEvent.INSTANCE, 2, 0);
            checkpointFuture = triggerCheckpoint(testHarness, 6, checkpointOptions);
            // Notifies the result partition that all records are processed after the
            // last checkpoint is triggered.
            checkpointFuture.thenAccept((ignored) -> {
                for (ResultPartition resultPartition : partitionWriters) {
                    resultPartition.onSubpartitionAllDataProcessed(0);
                }
            });
            // The checkpoint 6 would be triggered successfully.
            testHarness.processAll();
            testHarness.finishProcessing();
            assertTrue(checkpointFuture.isDone());
            testHarness.getTaskStateManager().getWaitForReportLatch().await();
            assertEquals(6, testHarness.getTaskStateManager().getReportedCheckpointId());
            // Each result partition should have emitted 3 barriers and 1 EndOfUserRecordsEvent.
            for (ResultPartition resultPartition : partitionWriters) {
                assertEquals(4, resultPartition.getNumberOfQueuedBuffers());
            }
        }
    } finally {
        for (ResultPartitionWriter writer : partitionWriters) {
            if (writer != null) {
                writer.close();
            }
        }
    }
}

Example 40 with CheckpointOptions

use of org.apache.flink.runtime.checkpoint.CheckpointOptions in project flink by apache.

the class CheckpointBarrierTest method testSerialization.

/**
 * Test serialization of the checkpoint barrier. The checkpoint barrier does not support its own
 * serialization, in order to be immutable.
 */
@Test
public void testSerialization() throws Exception {
    long id = Integer.MAX_VALUE + 123123L;
    long timestamp = Integer.MAX_VALUE + 1228L;
    CheckpointOptions options = CheckpointOptions.forCheckpointWithDefaultLocation();
    CheckpointBarrier barrier = new CheckpointBarrier(id, timestamp, options);
    try {
        barrier.write(new DataOutputSerializer(1024));
        fail("should throw an exception");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        barrier.read(new DataInputDeserializer(new byte[32]));
        fail("should throw an exception");
    } catch (UnsupportedOperationException e) {
    // expected
    }
}