Examples with FsStateBackend org.apache.flink.runtime.state.filesystem.FsStateBackend used on opensource projects

Example 6 with FsStateBackend

use of org.apache.flink.runtime.state.filesystem.FsStateBackend in project flink by apache.

the class StatefulStreamingJob method main.

public static void main(String[] args) throws Exception {
    final ParameterTool pt = ParameterTool.fromArgs(args);
    final String checkpointDir = pt.getRequired("checkpoint.dir");
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStateBackend(new FsStateBackend(checkpointDir));
    env.setRestartStrategy(RestartStrategies.noRestart());
    env.enableCheckpointing(1000L);
    env.getConfig().disableGenericTypes();
    env.addSource(new MySource()).uid("my-source").keyBy(anInt -> 0).map(new MyStatefulFunction()).uid("my-map").addSink(new DiscardingSink<>()).uid("my-sink");
    env.execute();
}

Example 7 with FsStateBackend

use of org.apache.flink.runtime.state.filesystem.FsStateBackend in project flink by apache.

the class RocksDBKeyedStateBackendTestFactory method getRocksDBStateBackend.

private RocksDBStateBackend getRocksDBStateBackend(TemporaryFolder tmp) throws IOException {
    String dbPath = tmp.newFolder().getAbsolutePath();
    String checkpointPath = tmp.newFolder().toURI().toString();
    RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath), true);
    backend.setDbStoragePath(dbPath);
    return backend;
}

Example 8 with FsStateBackend

use of org.apache.flink.runtime.state.filesystem.FsStateBackend in project flink by apache.

the class ManualWindowSpeedITCase method testTumblingIngestionTimeWindowsWithFsBackend.

@Test
public void testTumblingIngestionTimeWindowsWithFsBackend() throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(1);
    String checkpoints = tempFolder.newFolder().toURI().toString();
    env.setStateBackend(new FsStateBackend(checkpoints));
    env.addSource(new InfiniteTupleSource(1_000)).assignTimestampsAndWatermarks(IngestionTimeWatermarkStrategy.create()).keyBy(0).window(TumblingEventTimeWindows.of(Time.seconds(3))).reduce(new ReduceFunction<Tuple2<String, Integer>>() {

        private static final long serialVersionUID = 1L;

        @Override
        public Tuple2<String, Integer> reduce(Tuple2<String, Integer> value1, Tuple2<String, Integer> value2) throws Exception {
            return Tuple2.of(value1.f0, value1.f1 + value2.f1);
        }
    }).filter(new FilterFunction<Tuple2<String, Integer>>() {

        private static final long serialVersionUID = 1L;

        @Override
        public boolean filter(Tuple2<String, Integer> value) throws Exception {
            return value.f0.startsWith("Tuple 0");
        }
    }).print();
    env.execute();
}

Example 9 with FsStateBackend

use of org.apache.flink.runtime.state.filesystem.FsStateBackend in project flink by apache.

the class ManualWindowSpeedITCase method testTumblingIngestionTimeWindowsWithFsBackendWithLateness.

@Test
public void testTumblingIngestionTimeWindowsWithFsBackendWithLateness() throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(1);
    String checkpoints = tempFolder.newFolder().toURI().toString();
    env.setStateBackend(new FsStateBackend(checkpoints));
    env.addSource(new InfiniteTupleSource(10_000)).assignTimestampsAndWatermarks(IngestionTimeWatermarkStrategy.create()).keyBy(0).window(TumblingEventTimeWindows.of(Time.seconds(3))).allowedLateness(Time.seconds(1)).reduce(new ReduceFunction<Tuple2<String, Integer>>() {

        private static final long serialVersionUID = 1L;

        @Override
        public Tuple2<String, Integer> reduce(Tuple2<String, Integer> value1, Tuple2<String, Integer> value2) throws Exception {
            return Tuple2.of(value1.f0, value1.f1 + value2.f1);
        }
    }).filter(new FilterFunction<Tuple2<String, Integer>>() {

        private static final long serialVersionUID = 1L;

        @Override
        public boolean filter(Tuple2<String, Integer> value) throws Exception {
            return value.f0.startsWith("Tuple 0");
        }
    }).print();
    env.execute();
}

Example 10 with FsStateBackend

use of org.apache.flink.runtime.state.filesystem.FsStateBackend in project flink by apache.

the class CheckpointingCustomKvStateProgram method main.

public static void main(String[] args) throws Exception {
    final String checkpointPath = args[0];
    final String outputPath = args[1];
    final int parallelism = 1;
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(parallelism);
    env.enableCheckpointing(100);
    env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 1000));
    env.setStateBackend(new FsStateBackend(checkpointPath));
    DataStream<Integer> source = env.addSource(new InfiniteIntegerSource());
    source.map(new MapFunction<Integer, Tuple2<Integer, Integer>>() {

        private static final long serialVersionUID = 1L;

        @Override
        public Tuple2<Integer, Integer> map(Integer value) throws Exception {
            return new Tuple2<>(ThreadLocalRandom.current().nextInt(parallelism), value);
        }
    }).keyBy(new KeySelector<Tuple2<Integer, Integer>, Integer>() {

        private static final long serialVersionUID = 1L;

        @Override
        public Integer getKey(Tuple2<Integer, Integer> value) throws Exception {
            return value.f0;
        }
    }).flatMap(new ReducingStateFlatMap()).writeAsText(outputPath, FileSystem.WriteMode.OVERWRITE);
    env.execute();
}