Examples with FsStateBackend org.apache.flink.runtime.state.filesystem.FsStateBackend used on opensource projects

Example 16 with FsStateBackend

use of org.apache.flink.runtime.state.filesystem.FsStateBackend in project flink by apache.

the class TaskManagerProcessFailureStreamingRecoveryITCase method testTaskManagerFailure.

@Override
public void testTaskManagerFailure(int jobManagerPort, final File coordinateDir) throws Exception {
    final File tempCheckpointDir = new File(new File(ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH), UUID.randomUUID().toString());
    assertTrue("Cannot create directory for checkpoints", tempCheckpointDir.mkdirs());
    StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", jobManagerPort);
    env.setParallelism(PARALLELISM);
    env.getConfig().disableSysoutLogging();
    env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 1000));
    env.enableCheckpointing(200);
    env.setStateBackend(new FsStateBackend(tempCheckpointDir.getAbsoluteFile().toURI()));
    DataStream<Long> result = env.addSource(new SleepyDurableGenerateSequence(coordinateDir, DATA_COUNT)).map(new MapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            return value;
        }
    }).startNewChain().map(new Mapper(coordinateDir));
    //write result to temporary file
    result.addSink(new CheckpointedSink(DATA_COUNT));
    try {
        // blocking call until execution is done
        env.execute();
    // TODO: Figure out why this fails when ran with other tests
    // Check whether checkpoints have been cleaned up properly
    // assertDirectoryEmpty(tempCheckpointDir);
    } finally {
        // clean up
        if (tempCheckpointDir.exists()) {
            FileUtils.deleteDirectory(tempCheckpointDir);
        }
    }
}

Example 17 with FsStateBackend

use of org.apache.flink.runtime.state.filesystem.FsStateBackend in project flink by apache.

the class KubernetesHighAvailabilityRecoverFromSavepointITCase method createJobGraph.

private JobGraph createJobGraph() throws Exception {
    final StreamExecutionEnvironment sEnv = StreamExecutionEnvironment.getExecutionEnvironment();
    final StateBackend stateBackend = new FsStateBackend(temporaryFolder.newFolder().toURI(), 1);
    sEnv.setStateBackend(stateBackend);
    sEnv.addSource(new InfiniteSourceFunction()).keyBy(e -> e).flatMap(new RichFlatMapFunction<Integer, Integer>() {

        private static final long serialVersionUID = 1L;

        ValueState<Integer> state;

        @Override
        public void open(Configuration parameters) throws Exception {
            super.open(parameters);
            ValueStateDescriptor<Integer> descriptor = new ValueStateDescriptor<>("total", Types.INT);
            state = getRuntimeContext().getState(descriptor);
        }

        @Override
        public void flatMap(Integer value, Collector<Integer> out) throws Exception {
            final Integer current = state.value();
            if (current != null) {
                value += current;
            }
            state.update(value);
            out.collect(value);
        }
    }).uid(FLAT_MAP_UID).addSink(new DiscardingSink<>());
    return sEnv.getStreamGraph().getJobGraph();
}

Example 18 with FsStateBackend

use of org.apache.flink.runtime.state.filesystem.FsStateBackend in project flink by apache.

the class WritableSavepointITCase method testRocksDBStateBackend.

@Test
public void testRocksDBStateBackend() throws Exception {
    StateBackend backend = new RocksDBStateBackend(new FsStateBackend(TEMPORARY_FOLDER.newFolder().toURI(), FILE_STATE_SIZE));
    testStateBootstrapAndModification(backend);
}

Example 19 with FsStateBackend

use of org.apache.flink.runtime.state.filesystem.FsStateBackend in project beam by apache.

the class FlinkExecutionEnvironments method configureStateBackend.

private static void configureStateBackend(FlinkPipelineOptions options, StreamExecutionEnvironment env) {
    final StateBackend stateBackend;
    if (options.getStateBackend() != null) {
        final String storagePath = options.getStateBackendStoragePath();
        Preconditions.checkArgument(storagePath != null, "State backend was set to '%s' but no storage path was provided.", options.getStateBackend());
        if (options.getStateBackend().equalsIgnoreCase("rocksdb")) {
            try {
                stateBackend = new RocksDBStateBackend(storagePath);
            } catch (Exception e) {
                throw new RuntimeException("Could not create RocksDB state backend. Make sure it is included in the path.", e);
            }
        } else if (options.getStateBackend().equalsIgnoreCase("filesystem")) {
            stateBackend = new FsStateBackend(storagePath);
        } else {
            throw new IllegalArgumentException(String.format("Unknown state backend '%s'. Use 'rocksdb' or 'filesystem' or configure via Flink config file.", options.getStateBackend()));
        }
    } else if (options.getStateBackendFactory() != null) {
        // Legacy way of setting the state backend
        stateBackend = InstanceBuilder.ofType(FlinkStateBackendFactory.class).fromClass(options.getStateBackendFactory()).build().createStateBackend(options);
    } else {
        stateBackend = null;
    }
    if (stateBackend != null) {
        env.setStateBackend(stateBackend);
    }
}

Example 20 with FsStateBackend

use of org.apache.flink.runtime.state.filesystem.FsStateBackend in project flink by apache.

the class StateBackendLoadingTest method testLoadFileSystemStateBackend.

// ------------------------------------------------------------------------
// File System State Backend
// ------------------------------------------------------------------------
/**
 * Validates loading a file system state backend with additional parameters from the cluster
 * configuration.
 */
@Test
public void testLoadFileSystemStateBackend() throws Exception {
    final String checkpointDir = new Path(tmp.newFolder().toURI()).toString();
    final String savepointDir = new Path(tmp.newFolder().toURI()).toString();
    final Path expectedCheckpointsPath = new Path(checkpointDir);
    final Path expectedSavepointsPath = new Path(savepointDir);
    final MemorySize threshold = MemorySize.parse("900kb");
    final int minWriteBufferSize = 1024;
    // we configure with the explicit string (rather than
    // AbstractStateBackend#X_STATE_BACKEND_NAME)
    // to guard against config-breaking changes of the name
    final Configuration config1 = new Configuration();
    config1.setString(backendKey, "filesystem");
    config1.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir);
    config1.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);
    config1.set(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, threshold);
    config1.setInteger(CheckpointingOptions.FS_WRITE_BUFFER_SIZE, minWriteBufferSize);
    final Configuration config2 = new Configuration();
    config2.setString(backendKey, FsStateBackendFactory.class.getName());
    config2.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir);
    config2.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);
    config2.set(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, threshold);
    config1.setInteger(CheckpointingOptions.FS_WRITE_BUFFER_SIZE, minWriteBufferSize);
    StateBackend backend1 = StateBackendLoader.loadStateBackendFromConfig(config1, cl, null);
    StateBackend backend2 = StateBackendLoader.loadStateBackendFromConfig(config2, cl, null);
    assertTrue(backend1 instanceof HashMapStateBackend);
    assertTrue(backend2 instanceof FsStateBackend);
    HashMapStateBackend fs1 = (HashMapStateBackend) backend1;
    FsStateBackend fs2 = (FsStateBackend) backend2;
    assertEquals(expectedCheckpointsPath, fs2.getCheckpointPath());
    assertEquals(expectedSavepointsPath, fs2.getSavepointPath());
    assertEquals(threshold.getBytes(), fs2.getMinFileSizeThreshold());
    assertEquals(Math.max(threshold.getBytes(), minWriteBufferSize), fs2.getWriteBufferSize());
}