Examples with RocksDBStateBackend org.apache.flink.contrib.streaming.state.RocksDBStateBackend used on opensource projects

Example 21 with RocksDBStateBackend

use of org.apache.flink.contrib.streaming.state.RocksDBStateBackend in project flink by apache.

the class ManualWindowSpeedITCase method testTumblingIngestionTimeWindowsWithRocksDBBackendWithLateness.

@Test
public void testTumblingIngestionTimeWindowsWithRocksDBBackendWithLateness() throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(1);
    env.setStateBackend(new RocksDBStateBackend(new MemoryStateBackend()));
    env.addSource(new InfiniteTupleSource(10_000)).assignTimestampsAndWatermarks(IngestionTimeWatermarkStrategy.create()).keyBy(0).window(TumblingEventTimeWindows.of(Time.seconds(3))).allowedLateness(Time.seconds(1)).reduce(new ReduceFunction<Tuple2<String, Integer>>() {

        private static final long serialVersionUID = 1L;

        @Override
        public Tuple2<String, Integer> reduce(Tuple2<String, Integer> value1, Tuple2<String, Integer> value2) throws Exception {
            return Tuple2.of(value1.f0, value1.f1 + value2.f1);
        }
    }).filter(new FilterFunction<Tuple2<String, Integer>>() {

        private static final long serialVersionUID = 1L;

        @Override
        public boolean filter(Tuple2<String, Integer> value) throws Exception {
            return value.f0.startsWith("Tuple 0");
        }
    }).print();
    env.execute();
}

Example 22 with RocksDBStateBackend

use of org.apache.flink.contrib.streaming.state.RocksDBStateBackend in project flink by apache.

the class KeyedStateCheckpointingITCase method testWithRocksDbBackendFull.

@Test
public void testWithRocksDbBackendFull() throws Exception {
    RocksDBStateBackend fullRocksDbBackend = new RocksDBStateBackend(new MemoryStateBackend(MAX_MEM_STATE_SIZE), false);
    fullRocksDbBackend.setDbStoragePath(tmpFolder.newFolder().getAbsolutePath());
    testProgramWithBackend(fullRocksDbBackend);
}

Example 23 with RocksDBStateBackend

use of org.apache.flink.contrib.streaming.state.RocksDBStateBackend in project flink by apache.

the class LegacyStatefulJobSavepointMigrationITCase method writeSavepoint.

/**
 * Manually run this to write binary snapshot data.
 */
@Test
@Ignore
public void writeSavepoint() throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    switch(flinkGenerateSavepointBackendType) {
        case StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME:
            env.setStateBackend(new RocksDBStateBackend(new MemoryStateBackend()));
            break;
        case StateBackendLoader.MEMORY_STATE_BACKEND_NAME:
            env.setStateBackend(new MemoryStateBackend());
            break;
        default:
            throw new UnsupportedOperationException();
    }
    env.enableCheckpointing(500);
    env.setParallelism(4);
    env.setMaxParallelism(4);
    env.addSource(new LegacyCheckpointedSource(NUM_SOURCE_ELEMENTS)).setMaxParallelism(1).uid("LegacyCheckpointedSource").flatMap(new LegacyCheckpointedFlatMap()).startNewChain().uid("LegacyCheckpointedFlatMap").keyBy(0).flatMap(new LegacyCheckpointedFlatMapWithKeyedState()).startNewChain().uid("LegacyCheckpointedFlatMapWithKeyedState").keyBy(0).flatMap(new KeyedStateSettingFlatMap()).startNewChain().uid("KeyedStateSettingFlatMap").keyBy(0).transform("custom_operator", new TypeHint<Tuple2<Long, Long>>() {
    }.getTypeInfo(), new CheckpointedUdfOperator(new LegacyCheckpointedFlatMapWithKeyedState())).uid("LegacyCheckpointedOperator").keyBy(0).transform("timely_stateful_operator", new TypeHint<Tuple2<Long, Long>>() {
    }.getTypeInfo(), new TimelyStatefulOperator()).uid("TimelyStatefulOperator").addSink(new AccumulatorCountingSink<Tuple2<Long, Long>>());
    executeAndSnapshot(env, "src/test/resources/" + getSavepointPath(flinkGenerateSavepointVersion, flinkGenerateSavepointBackendType), SnapshotType.SAVEPOINT_CANONICAL, new Tuple2<>(AccumulatorCountingSink.NUM_ELEMENTS_ACCUMULATOR, NUM_SOURCE_ELEMENTS));
}