Examples with CheckpointOptions org.apache.flink.runtime.checkpoint.CheckpointOptions used on opensource projects

Example 26 with CheckpointOptions

use of org.apache.flink.runtime.checkpoint.CheckpointOptions in project flink by apache.

the class SnapshotUtils method snapshot.

public static <OUT, OP extends StreamOperator<OUT>> TaggedOperatorSubtaskState snapshot(OP operator, int index, long timestamp, boolean isExactlyOnceMode, boolean isUnalignedCheckpoint, Configuration configuration, Path savepointPath) throws Exception {
    CheckpointOptions options = CheckpointOptions.forConfig(SavepointType.savepoint(SavepointFormatType.CANONICAL), AbstractFsCheckpointStorageAccess.encodePathAsReference(savepointPath), isExactlyOnceMode, isUnalignedCheckpoint, CheckpointOptions.NO_ALIGNED_CHECKPOINT_TIME_OUT);
    operator.prepareSnapshotPreBarrier(CHECKPOINT_ID);
    CheckpointStreamFactory storage = createStreamFactory(configuration, options);
    OperatorSnapshotFutures snapshotInProgress = operator.snapshotState(CHECKPOINT_ID, timestamp, options, storage);
    OperatorSubtaskState state = new OperatorSnapshotFinalizer(snapshotInProgress).getJobManagerOwnedState();
    operator.notifyCheckpointComplete(CHECKPOINT_ID);
    return new TaggedOperatorSubtaskState(index, state);
}

Example 27 with CheckpointOptions

use of org.apache.flink.runtime.checkpoint.CheckpointOptions in project flink by apache.

the class MultipleInputStreamTask method triggerStopWithSavepointAsync.

private CompletableFuture<Boolean> triggerStopWithSavepointAsync(CheckpointMetaData checkpointMetaData, CheckpointOptions checkpointOptions) {
    CompletableFuture<Void> sourcesStopped = new CompletableFuture<>();
    final StopMode stopMode = ((SavepointType) checkpointOptions.getCheckpointType()).shouldDrain() ? StopMode.DRAIN : StopMode.NO_DRAIN;
    mainMailboxExecutor.execute(() -> {
        setSynchronousSavepoint(checkpointMetaData.getCheckpointId());
        FutureUtils.forward(FutureUtils.waitForAll(operatorChain.getSourceTaskInputs().stream().map(s -> s.getOperator().stop(stopMode)).collect(Collectors.toList())), sourcesStopped);
    }, "stop chained Flip-27 source for stop-with-savepoint --drain");
    return sourcesStopped.thenCompose(ignore -> triggerSourcesCheckpointAsync(checkpointMetaData, checkpointOptions));
}

Example 28 with CheckpointOptions

use of org.apache.flink.runtime.checkpoint.CheckpointOptions in project flink by apache.

the class SourceStreamTask method init.

@Override
protected void init() {
    // we check if the source is actually inducing the checkpoints, rather
    // than the trigger
    SourceFunction<?> source = mainOperator.getUserFunction();
    if (source instanceof ExternallyInducedSource) {
        externallyInducedCheckpoints = true;
        ExternallyInducedSource.CheckpointTrigger triggerHook = new ExternallyInducedSource.CheckpointTrigger() {

            @Override
            public void triggerCheckpoint(long checkpointId) throws FlinkException {
                // TODO - we need to see how to derive those. We should probably not
                // encode this in the
                // TODO -   source's trigger message, but do a handshake in this task
                // between the trigger
                // TODO -   message from the master, and the source's trigger
                // notification
                final CheckpointOptions checkpointOptions = CheckpointOptions.forConfig(CheckpointType.CHECKPOINT, CheckpointStorageLocationReference.getDefault(), configuration.isExactlyOnceCheckpointMode(), configuration.isUnalignedCheckpointsEnabled(), configuration.getAlignedCheckpointTimeout().toMillis());
                final long timestamp = System.currentTimeMillis();
                final CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, timestamp, timestamp);
                try {
                    SourceStreamTask.super.triggerCheckpointAsync(checkpointMetaData, checkpointOptions).get();
                } catch (RuntimeException e) {
                    throw e;
                } catch (Exception e) {
                    throw new FlinkException(e.getMessage(), e);
                }
            }
        };
        ((ExternallyInducedSource<?, ?>) source).setCheckpointTrigger(triggerHook);
    }
    getEnvironment().getMetricGroup().getIOMetricGroup().gauge(MetricNames.CHECKPOINT_START_DELAY_TIME, this::getAsyncCheckpointStartDelayNanos);
}

Example 29 with CheckpointOptions

use of org.apache.flink.runtime.checkpoint.CheckpointOptions in project flink by apache.

the class StreamTaskTest method testSyncSavepointWithEndInput.

/**
 * Test for SyncSavepoint and EndInput interactions. Targets following scenarios scenarios:
 *
 * <ol>
 *   <li>Thread1: notify sync savepoint
 *   <li>Thread2: endInput
 *   <li>Thread1: confirm/abort/abortAsync
 *   <li>assert inputEnded: confirmed - no, abort/abortAsync - yes
 * </ol>
 */
private void testSyncSavepointWithEndInput(BiConsumerWithException<StreamTask<?, ?>, Long, IOException> savepointResult, SnapshotType checkpointType, boolean expectEndInput) throws Exception {
    StreamTaskMailboxTestHarness<String> harness = new StreamTaskMailboxTestHarnessBuilder<>(OneInputStreamTask::new, STRING_TYPE_INFO).addInput(STRING_TYPE_INFO).setupOutputForSingletonOperatorChain(new TestBoundedOneInputStreamOperator()).build();
    final long checkpointId = 1L;
    CountDownLatch savepointTriggeredLatch = new CountDownLatch(1);
    CountDownLatch inputEndedLatch = new CountDownLatch(1);
    MailboxExecutor executor = harness.streamTask.getMailboxExecutorFactory().createExecutor(MAX_PRIORITY);
    executor.execute(() -> {
        try {
            harness.streamTask.triggerCheckpointOnBarrier(new CheckpointMetaData(checkpointId, checkpointId), new CheckpointOptions(checkpointType, getDefault()), new CheckpointMetricsBuilder());
        } catch (IOException e) {
            fail(e.getMessage());
        }
    }, "triggerCheckpointOnBarrier");
    new Thread(() -> {
        try {
            savepointTriggeredLatch.await();
            harness.endInput(expectEndInput);
            inputEndedLatch.countDown();
        } catch (InterruptedException e) {
            fail(e.getMessage());
        }
    }).start();
    // this mails should be executed from the one above (from triggerCheckpointOnBarrier)
    executor.execute(savepointTriggeredLatch::countDown, "savepointTriggeredLatch");
    executor.execute(() -> {
        inputEndedLatch.await();
        savepointResult.accept(harness.streamTask, checkpointId);
    }, "savepointResult");
    harness.processAll();
    Assert.assertEquals(expectEndInput, TestBoundedOneInputStreamOperator.isInputEnded());
}

Example 30 with CheckpointOptions

use of org.apache.flink.runtime.checkpoint.CheckpointOptions in project flink by apache.

the class SubtaskCheckpointCoordinatorTest method initCheckpoint.

private boolean initCheckpoint(boolean unalignedCheckpointEnabled, SnapshotType checkpointType) throws IOException, CheckpointException {
    class MockWriter extends ChannelStateWriterImpl.NoOpChannelStateWriter {

        private boolean started;

        @Override
        public void start(long checkpointId, CheckpointOptions checkpointOptions) {
            started = true;
        }
    }
    MockWriter writer = new MockWriter();
    SubtaskCheckpointCoordinator coordinator = coordinator(unalignedCheckpointEnabled, writer);
    CheckpointStorageLocationReference locationReference = CheckpointStorageLocationReference.getDefault();
    coordinator.initInputsCheckpoint(1L, unalignedCheckpointEnabled ? CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, locationReference) : CheckpointOptions.alignedNoTimeout(checkpointType, locationReference));
    return writer.started;
}