Examples with FileSystemCheckpointStorage org.apache.flink.runtime.state.storage.FileSystemCheckpointStorage used on opensource projects

Example 1 with FileSystemCheckpointStorage

use of org.apache.flink.runtime.state.storage.FileSystemCheckpointStorage in project flink by apache.

the class CheckpointStorageLoaderTest method testLoadFileSystemCheckpointStorageMixed.

/**
 * Validates taking the application-defined file system state backend and adding with additional
 * parameters from the cluster configuration, but giving precedence to application-defined
 * parameters over configuration-defined parameters.
 */
@Test
public void testLoadFileSystemCheckpointStorageMixed() throws Exception {
    final Path appCheckpointDir = new Path(tmp.newFolder().toURI());
    final String checkpointDir = new Path(tmp.newFolder().toURI()).toString();
    final String savepointDir = new Path(tmp.newFolder().toURI()).toString();
    final Path expectedSavepointsPath = new Path(savepointDir);
    final int threshold = 1000000;
    final int writeBufferSize = 4000000;
    final FileSystemCheckpointStorage storage = new FileSystemCheckpointStorage(appCheckpointDir, threshold, writeBufferSize);
    final Configuration config = new Configuration();
    config.set(CheckpointingOptions.CHECKPOINT_STORAGE, // this should not be picked up
    "jobmanager");
    config.set(CheckpointingOptions.CHECKPOINTS_DIRECTORY, // this should not be picked up
    checkpointDir);
    config.set(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);
    config.set(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, // this should not be picked up
    MemorySize.parse("20"));
    config.setInteger(CheckpointingOptions.FS_WRITE_BUFFER_SIZE, // this should not be picked up
    3000000);
    final CheckpointStorage loadedStorage = CheckpointStorageLoader.load(storage, null, new ModernStateBackend(), config, cl, log);
    Assert.assertThat(loadedStorage, Matchers.instanceOf(FileSystemCheckpointStorage.class));
    final FileSystemCheckpointStorage fs = (FileSystemCheckpointStorage) loadedStorage;
    Assert.assertThat(fs.getCheckpointPath(), normalizedPath(appCheckpointDir));
    Assert.assertThat(fs.getSavepointPath(), normalizedPath(expectedSavepointsPath));
    Assert.assertEquals(threshold, fs.getMinFileSizeThreshold());
    Assert.assertEquals(writeBufferSize, fs.getWriteBufferSize());
}

Example 2 with FileSystemCheckpointStorage

use of org.apache.flink.runtime.state.storage.FileSystemCheckpointStorage in project flink by apache.

the class CheckpointConfig method setCheckpointStorage.

/**
 * Configures the application to write out checkpoint snapshots to the configured directory. See
 * {@link FileSystemCheckpointStorage} for more details on checkpointing to a file system.
 *
 * @param checkpointDirectory The path to write checkpoint metadata to.
 * @see #setCheckpointStorage(String)
 */
@PublicEvolving
public void setCheckpointStorage(Path checkpointDirectory) {
    Preconditions.checkNotNull(checkpointDirectory, "Checkpoint directory must not be null");
    this.storage = new FileSystemCheckpointStorage(checkpointDirectory);
}

Example 3 with FileSystemCheckpointStorage

use of org.apache.flink.runtime.state.storage.FileSystemCheckpointStorage in project flink by apache.

the class FileSinkCompactionSwitchITCase method createJobGraph.

private JobGraph createJobGraph(String path, String cpPath, boolean compactionEnabled, boolean isFinite, SharedReference<ConcurrentHashMap<Integer, Integer>> sendCountMap) {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    Configuration config = new Configuration();
    config.set(ExecutionOptions.RUNTIME_MODE, RuntimeExecutionMode.STREAMING);
    // disable changelog state in case it's randomly enabled, since it will fail the savepoint
    config.set(StateChangelogOptions.ENABLE_STATE_CHANGE_LOG, false);
    env.configure(config, getClass().getClassLoader());
    env.enableCheckpointing(100, CheckpointingMode.EXACTLY_ONCE);
    env.setRestartStrategy(RestartStrategies.noRestart());
    env.getCheckpointConfig().setCheckpointStorage(new FileSystemCheckpointStorage(cpPath));
    env.setStateBackend(new HashMapStateBackend());
    env.addSource(new CountingTestSource(latchId, NUM_RECORDS, isFinite, sendCountMap)).setParallelism(NUM_SOURCES).sinkTo(createFileSink(path, compactionEnabled)).uid("sink").setParallelism(NUM_SINKS);
    StreamGraph streamGraph = env.getStreamGraph();
    return streamGraph.getJobGraph();
}

Example 4 with FileSystemCheckpointStorage

use of org.apache.flink.runtime.state.storage.FileSystemCheckpointStorage in project flink by apache.

the class CheckpointConfig method setCheckpointStorage.

/**
 * Configures the application to write out checkpoint snapshots to the configured directory. See
 * {@link FileSystemCheckpointStorage} for more details on checkpointing to a file system.
 *
 * @param checkpointDirectory The path to write checkpoint metadata to.
 * @see #setCheckpointStorage(CheckpointStorage)
 */
@PublicEvolving
public void setCheckpointStorage(String checkpointDirectory) {
    Preconditions.checkNotNull(checkpointDirectory, "Checkpoint directory must not be null");
    this.storage = new FileSystemCheckpointStorage(checkpointDirectory);
}

Example 5 with FileSystemCheckpointStorage

use of org.apache.flink.runtime.state.storage.FileSystemCheckpointStorage in project flink by apache.

the class CheckpointConfig method setCheckpointStorage.

/**
 * Configures the application to write out checkpoint snapshots to the configured directory. See
 * {@link FileSystemCheckpointStorage} for more details on checkpointing to a file system.
 *
 * @param checkpointDirectory The path to write checkpoint metadata to.
 * @see #setCheckpointStorage(CheckpointStorage)
 */
@PublicEvolving
public void setCheckpointStorage(URI checkpointDirectory) {
    Preconditions.checkNotNull(checkpointDirectory, "Checkpoint directory must not be null");
    this.storage = new FileSystemCheckpointStorage(checkpointDirectory);
}