Examples with CheckpointStorage org.apache.flink.runtime.state.CheckpointStorage used on opensource projects

Example 6 with CheckpointStorage

use of org.apache.flink.runtime.state.CheckpointStorage in project flink by apache.

the class ChangelogStateBackendLoadingTest method testApplicationDefinedHasPrecedence.

@Test
public void testApplicationDefinedHasPrecedence() throws Exception {
    final StateBackend appBackend = new MockStateBackend();
    // "rocksdb" should not take effect
    final StateBackend backend = StateBackendLoader.fromApplicationOrConfigOrDefault(appBackend, TernaryBoolean.UNDEFINED, config("rocksdb", true), cl, null);
    final CheckpointStorage storage = CheckpointStorageLoader.load(null, null, backend, config(), cl, null);
    assertDelegateStateBackend(backend, MockStateBackend.class, storage, MockStateBackend.class);
    assertTrue(((MockStateBackend) (((ChangelogStateBackend) backend).getDelegatedStateBackend())).isConfigUpdated());
}

Example 7 with CheckpointStorage

use of org.apache.flink.runtime.state.CheckpointStorage in project flink by apache.

the class StreamTask method createCheckpointStorage.

private CheckpointStorage createCheckpointStorage(StateBackend backend) throws Exception {
    final CheckpointStorage fromApplication = configuration.getCheckpointStorage(getUserCodeClassLoader());
    final Path savepointDir = configuration.getSavepointDir(getUserCodeClassLoader());
    return CheckpointStorageLoader.load(fromApplication, savepointDir, backend, getEnvironment().getTaskManagerInfo().getConfiguration(), getUserCodeClassLoader(), LOG);
}

Example 8 with CheckpointStorage

use of org.apache.flink.runtime.state.CheckpointStorage in project flink by apache.

the class Checkpoints method disposeSavepoint.

public static void disposeSavepoint(String pointer, Configuration configuration, ClassLoader classLoader, @Nullable Logger logger) throws IOException, FlinkException {
    checkNotNull(pointer, "location");
    checkNotNull(configuration, "configuration");
    checkNotNull(classLoader, "classLoader");
    CheckpointStorage storage = loadCheckpointStorage(configuration, classLoader, logger);
    disposeSavepoint(pointer, storage, classLoader);
}

Example 9 with CheckpointStorage

use of org.apache.flink.runtime.state.CheckpointStorage in project flink by apache.

the class SchedulerTestingUtils method enableCheckpointing.

public static void enableCheckpointing(final JobGraph jobGraph, @Nullable StateBackend stateBackend, @Nullable CheckpointStorage checkpointStorage) {
    final CheckpointCoordinatorConfiguration config = new CheckpointCoordinatorConfiguration(// disable periodical checkpointing
    Long.MAX_VALUE, DEFAULT_CHECKPOINT_TIMEOUT_MS, 0, 1, CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION, false, false, 0, 0);
    SerializedValue<StateBackend> serializedStateBackend = null;
    if (stateBackend != null) {
        try {
            serializedStateBackend = new SerializedValue<>(stateBackend);
        } catch (IOException e) {
            throw new RuntimeException("could not serialize state backend", e);
        }
    }
    SerializedValue<CheckpointStorage> serializedCheckpointStorage = null;
    if (checkpointStorage != null) {
        try {
            serializedCheckpointStorage = new SerializedValue<>(checkpointStorage);
        } catch (IOException e) {
            throw new RuntimeException("could not serialize checkpoint storage", e);
        }
    }
    jobGraph.setSnapshotSettings(new JobCheckpointingSettings(config, serializedStateBackend, TernaryBoolean.UNDEFINED, serializedCheckpointStorage, null));
}

Example 10 with CheckpointStorage

use of org.apache.flink.runtime.state.CheckpointStorage in project flink by apache.

the class StreamingJobGraphGenerator method configureCheckpointing.

private void configureCheckpointing() {
    CheckpointConfig cfg = streamGraph.getCheckpointConfig();
    long interval = cfg.getCheckpointInterval();
    if (interval < MINIMAL_CHECKPOINT_TIME) {
        // interval of max value means disable periodic checkpoint
        interval = Long.MAX_VALUE;
    }
    // --- configure options ---
    CheckpointRetentionPolicy retentionAfterTermination;
    if (cfg.isExternalizedCheckpointsEnabled()) {
        CheckpointConfig.ExternalizedCheckpointCleanup cleanup = cfg.getExternalizedCheckpointCleanup();
        // Sanity check
        if (cleanup == null) {
            throw new IllegalStateException("Externalized checkpoints enabled, but no cleanup mode configured.");
        }
        retentionAfterTermination = cleanup.deleteOnCancellation() ? CheckpointRetentionPolicy.RETAIN_ON_FAILURE : CheckpointRetentionPolicy.RETAIN_ON_CANCELLATION;
    } else {
        retentionAfterTermination = CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION;
    }
    // --- configure the master-side checkpoint hooks ---
    final ArrayList<MasterTriggerRestoreHook.Factory> hooks = new ArrayList<>();
    for (StreamNode node : streamGraph.getStreamNodes()) {
        if (node.getOperatorFactory() instanceof UdfStreamOperatorFactory) {
            Function f = ((UdfStreamOperatorFactory) node.getOperatorFactory()).getUserFunction();
            if (f instanceof WithMasterCheckpointHook) {
                hooks.add(new FunctionMasterCheckpointHookFactory((WithMasterCheckpointHook<?>) f));
            }
        }
    }
    // because the hooks can have user-defined code, they need to be stored as
    // eagerly serialized values
    final SerializedValue<MasterTriggerRestoreHook.Factory[]> serializedHooks;
    if (hooks.isEmpty()) {
        serializedHooks = null;
    } else {
        try {
            MasterTriggerRestoreHook.Factory[] asArray = hooks.toArray(new MasterTriggerRestoreHook.Factory[hooks.size()]);
            serializedHooks = new SerializedValue<>(asArray);
        } catch (IOException e) {
            throw new FlinkRuntimeException("Trigger/restore hook is not serializable", e);
        }
    }
    // because the state backend can have user-defined code, it needs to be stored as
    // eagerly serialized value
    final SerializedValue<StateBackend> serializedStateBackend;
    if (streamGraph.getStateBackend() == null) {
        serializedStateBackend = null;
    } else {
        try {
            serializedStateBackend = new SerializedValue<StateBackend>(streamGraph.getStateBackend());
        } catch (IOException e) {
            throw new FlinkRuntimeException("State backend is not serializable", e);
        }
    }
    // because the checkpoint storage can have user-defined code, it needs to be stored as
    // eagerly serialized value
    final SerializedValue<CheckpointStorage> serializedCheckpointStorage;
    if (streamGraph.getCheckpointStorage() == null) {
        serializedCheckpointStorage = null;
    } else {
        try {
            serializedCheckpointStorage = new SerializedValue<>(streamGraph.getCheckpointStorage());
        } catch (IOException e) {
            throw new FlinkRuntimeException("Checkpoint storage is not serializable", e);
        }
    }
    // --- done, put it all together ---
    JobCheckpointingSettings settings = new JobCheckpointingSettings(CheckpointCoordinatorConfiguration.builder().setCheckpointInterval(interval).setCheckpointTimeout(cfg.getCheckpointTimeout()).setMinPauseBetweenCheckpoints(cfg.getMinPauseBetweenCheckpoints()).setMaxConcurrentCheckpoints(cfg.getMaxConcurrentCheckpoints()).setCheckpointRetentionPolicy(retentionAfterTermination).setExactlyOnce(getCheckpointingMode(cfg) == CheckpointingMode.EXACTLY_ONCE).setTolerableCheckpointFailureNumber(cfg.getTolerableCheckpointFailureNumber()).setUnalignedCheckpointsEnabled(cfg.isUnalignedCheckpointsEnabled()).setCheckpointIdOfIgnoredInFlightData(cfg.getCheckpointIdOfIgnoredInFlightData()).setAlignedCheckpointTimeout(cfg.getAlignedCheckpointTimeout().toMillis()).setEnableCheckpointsAfterTasksFinish(streamGraph.isEnableCheckpointsAfterTasksFinish()).build(), serializedStateBackend, streamGraph.isChangelogStateBackendEnabled(), serializedCheckpointStorage, serializedHooks);
    jobGraph.setSnapshotSettings(settings);
}