Examples with StreamExecutionEnvironment org.apache.flink.streaming.api.environment.StreamExecutionEnvironment used on opensource projects

Example 56 with StreamExecutionEnvironment

use of org.apache.flink.streaming.api.environment.StreamExecutionEnvironment in project flink by apache.

the class WindowCheckpointingITCase method testAggregatingSlidingProcessingTimeWindow.

@Test
public void testAggregatingSlidingProcessingTimeWindow() {
    final int NUM_ELEMENTS = 3000;
    FailingSource.reset();
    try {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
        env.setParallelism(PARALLELISM);
        env.setStreamTimeCharacteristic(timeCharacteristic);
        env.getConfig().setAutoWatermarkInterval(10);
        env.enableCheckpointing(100);
        env.setRestartStrategy(RestartStrategies.fixedDelayRestart(3, 0));
        env.getConfig().disableSysoutLogging();
        env.addSource(new FailingSource(NUM_ELEMENTS, NUM_ELEMENTS / 3)).map(new MapFunction<Tuple2<Long, IntType>, Tuple2<Long, IntType>>() {

            @Override
            public Tuple2<Long, IntType> map(Tuple2<Long, IntType> value) {
                value.f1.value = 1;
                return value;
            }
        }).rebalance().keyBy(0).timeWindow(Time.of(150, MILLISECONDS), Time.of(50, MILLISECONDS)).reduce(new ReduceFunction<Tuple2<Long, IntType>>() {

            @Override
            public Tuple2<Long, IntType> reduce(Tuple2<Long, IntType> a, Tuple2<Long, IntType> b) {
                return new Tuple2<>(a.f0, new IntType(1));
            }
        }).addSink(new ValidatingSink(NUM_ELEMENTS, 3)).setParallelism(1);
        tryExecute(env, "Tumbling Window Test");
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 57 with StreamExecutionEnvironment

use of org.apache.flink.streaming.api.environment.StreamExecutionEnvironment in project flink by apache.

the class WindowCheckpointingITCase method testAggregatingTumblingProcessingTimeWindow.

@Test
public void testAggregatingTumblingProcessingTimeWindow() {
    final int NUM_ELEMENTS = 3000;
    FailingSource.reset();
    try {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
        env.setParallelism(PARALLELISM);
        env.setStreamTimeCharacteristic(timeCharacteristic);
        env.getConfig().setAutoWatermarkInterval(10);
        env.enableCheckpointing(100);
        env.setRestartStrategy(RestartStrategies.fixedDelayRestart(3, 0));
        env.getConfig().disableSysoutLogging();
        env.addSource(new FailingSource(NUM_ELEMENTS, NUM_ELEMENTS / 3)).map(new MapFunction<Tuple2<Long, IntType>, Tuple2<Long, IntType>>() {

            @Override
            public Tuple2<Long, IntType> map(Tuple2<Long, IntType> value) {
                value.f1.value = 1;
                return value;
            }
        }).rebalance().keyBy(0).timeWindow(Time.of(100, MILLISECONDS)).reduce(new ReduceFunction<Tuple2<Long, IntType>>() {

            @Override
            public Tuple2<Long, IntType> reduce(Tuple2<Long, IntType> a, Tuple2<Long, IntType> b) {
                return new Tuple2<>(a.f0, new IntType(1));
            }
        }).addSink(new ValidatingSink(NUM_ELEMENTS, 1)).setParallelism(1);
        tryExecute(env, "Tumbling Window Test");
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 58 with StreamExecutionEnvironment

use of org.apache.flink.streaming.api.environment.StreamExecutionEnvironment in project flink by apache.

the class WindowCheckpointingITCase method testTumblingProcessingTimeWindow.

// ------------------------------------------------------------------------
@Test
public void testTumblingProcessingTimeWindow() {
    final int NUM_ELEMENTS = 3000;
    FailingSource.reset();
    try {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
        env.setParallelism(PARALLELISM);
        env.setStreamTimeCharacteristic(timeCharacteristic);
        env.getConfig().setAutoWatermarkInterval(10);
        env.enableCheckpointing(100);
        env.setRestartStrategy(RestartStrategies.fixedDelayRestart(3, 0));
        env.getConfig().disableSysoutLogging();
        env.addSource(new FailingSource(NUM_ELEMENTS, NUM_ELEMENTS / 3)).rebalance().keyBy(0).timeWindow(Time.of(100, MILLISECONDS)).apply(new RichWindowFunction<Tuple2<Long, IntType>, Tuple2<Long, IntType>, Tuple, TimeWindow>() {

            private boolean open = false;

            @Override
            public void open(Configuration parameters) {
                assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
                open = true;
            }

            @Override
            public void apply(Tuple tuple, TimeWindow window, Iterable<Tuple2<Long, IntType>> values, Collector<Tuple2<Long, IntType>> out) {
                // validate that the function has been opened properly
                assertTrue(open);
                for (Tuple2<Long, IntType> value : values) {
                    assertEquals(value.f0.intValue(), value.f1.value);
                    out.collect(new Tuple2<Long, IntType>(value.f0, new IntType(1)));
                }
            }
        }).addSink(new ValidatingSink(NUM_ELEMENTS, 1)).setParallelism(1);
        tryExecute(env, "Tumbling Window Test");
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 59 with StreamExecutionEnvironment

use of org.apache.flink.streaming.api.environment.StreamExecutionEnvironment in project flink by apache.

the class StatefulUDFSavepointMigrationITCase method testSavepointRestoreFromFlink11.

@Test
public void testSavepointRestoreFromFlink11() throws Exception {
    final int EXPECTED_SUCCESSFUL_CHECKS = 21;
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
    // we only test memory state backend yet
    env.setStateBackend(new MemoryStateBackend());
    env.enableCheckpointing(500);
    env.setParallelism(4);
    env.setMaxParallelism(4);
    // create source
    env.addSource(new RestoringCheckingSource(NUM_SOURCE_ELEMENTS)).setMaxParallelism(1).uid("LegacyCheckpointedSource").flatMap(new RestoringCheckingFlatMap()).startNewChain().uid("LegacyCheckpointedFlatMap").keyBy(0).flatMap(new RestoringCheckingFlatMapWithKeyedState()).startNewChain().uid("LegacyCheckpointedFlatMapWithKeyedState").keyBy(0).flatMap(new KeyedStateCheckingFlatMap()).startNewChain().uid("KeyedStateSettingFlatMap").keyBy(0).transform("custom_operator", new TypeHint<Tuple2<Long, Long>>() {
    }.getTypeInfo(), new RestoringCheckingUdfOperator(new RestoringCheckingFlatMapWithKeyedState())).uid("LegacyCheckpointedOperator").addSink(new AccumulatorCountingSink<Tuple2<Long, Long>>(EXPECTED_ELEMENTS_ACCUMULATOR));
    restoreAndExecute(env, getResourceFilename("stateful-udf-migration-itcase-flink1.1-savepoint"), new Tuple2<>(SUCCESSFUL_CHECK_ACCUMULATOR, EXPECTED_SUCCESSFUL_CHECKS));
}

Example 60 with StreamExecutionEnvironment

use of org.apache.flink.streaming.api.environment.StreamExecutionEnvironment in project flink by apache.

the class StatefulUDFSavepointMigrationITCase method testSavepointRestoreFromFlink11FromRocksDB.

@Test
public void testSavepointRestoreFromFlink11FromRocksDB() throws Exception {
    final int EXPECTED_SUCCESSFUL_CHECKS = 21;
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
    // we only test memory state backend yet
    env.setStateBackend(new RocksDBStateBackend(new MemoryStateBackend()));
    env.enableCheckpointing(500);
    env.setParallelism(4);
    env.setMaxParallelism(4);
    // create source
    env.addSource(new RestoringCheckingSource(NUM_SOURCE_ELEMENTS)).setMaxParallelism(1).uid("LegacyCheckpointedSource").flatMap(new RestoringCheckingFlatMap()).startNewChain().uid("LegacyCheckpointedFlatMap").keyBy(0).flatMap(new RestoringCheckingFlatMapWithKeyedState()).startNewChain().uid("LegacyCheckpointedFlatMapWithKeyedState").keyBy(0).flatMap(new KeyedStateCheckingFlatMap()).startNewChain().uid("KeyedStateSettingFlatMap").keyBy(0).transform("custom_operator", new TypeHint<Tuple2<Long, Long>>() {
    }.getTypeInfo(), new RestoringCheckingUdfOperator(new RestoringCheckingFlatMapWithKeyedState())).uid("LegacyCheckpointedOperator").addSink(new AccumulatorCountingSink<Tuple2<Long, Long>>(EXPECTED_ELEMENTS_ACCUMULATOR));
    restoreAndExecute(env, getResourceFilename("stateful-udf-migration-itcase-flink1.1-savepoint-rocksdb"), new Tuple2<>(SUCCESSFUL_CHECK_ACCUMULATOR, EXPECTED_SUCCESSFUL_CHECKS));
}