Examples with FailingSource org.apache.flink.test.checkpointing.utils.FailingSource used on opensource projects

Example 1 with FailingSource

use of org.apache.flink.test.checkpointing.utils.FailingSource in project flink by apache.

the class EventTimeWindowCheckpointingITCase method doTestTumblingTimeWindowWithKVState.

public void doTestTumblingTimeWindowWithKVState(int maxParallelism) {
    final int numElementsPerKey = numElementsPerKey();
    final int windowSize = windowSize();
    final int numKeys = numKeys();
    try {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(PARALLELISM);
        env.setMaxParallelism(maxParallelism);
        env.enableCheckpointing(100);
        env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
        env.setStateBackend(this.stateBackend);
        env.getConfig().setUseSnapshotCompression(true);
        env.addSource(new FailingSource(new KeyedEventTimeGenerator(numKeys, windowSize), numElementsPerKey)).rebalance().keyBy(0).window(TumblingEventTimeWindows.of(Time.milliseconds(windowSize))).apply(new RichWindowFunction<Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, Tuple, TimeWindow>() {

            private boolean open = false;

            private ValueState<Integer> count;

            @Override
            public void open(Configuration parameters) {
                assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
                open = true;
                count = getRuntimeContext().getState(new ValueStateDescriptor<>("count", Integer.class, 0));
            }

            @Override
            public void apply(Tuple tuple, TimeWindow window, Iterable<Tuple2<Long, IntType>> values, Collector<Tuple4<Long, Long, Long, IntType>> out) throws Exception {
                // different count results for each key
                if (count.value() == 0) {
                    count.update(tuple.<Long>getField(0).intValue());
                }
                // validate that the function has been opened properly
                assertTrue(open);
                count.update(count.value() + 1);
                out.collect(new Tuple4<>(tuple.<Long>getField(0), window.getStart(), window.getEnd(), new IntType(count.value())));
            }
        }).addSink(new ValidatingSink<>(new CountingSinkValidatorUpdateFun(), new SinkValidatorCheckFun(numKeys, numElementsPerKey, windowSize))).setParallelism(1);
        env.execute("Tumbling Window Test");
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 2 with FailingSource

use of org.apache.flink.test.checkpointing.utils.FailingSource in project flink by apache.

the class EventTimeWindowCheckpointingITCase method testPreAggregatedSlidingTimeWindow.

@Test
public void testPreAggregatedSlidingTimeWindow() {
    final int numElementsPerKey = numElementsPerKey();
    final int windowSize = windowSize();
    final int windowSlide = windowSlide();
    final int numKeys = numKeys();
    try {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(PARALLELISM);
        env.enableCheckpointing(100);
        env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
        env.setStateBackend(this.stateBackend);
        env.getConfig().setUseSnapshotCompression(true);
        env.addSource(new FailingSource(new KeyedEventTimeGenerator(numKeys, windowSlide), numElementsPerKey)).rebalance().keyBy(0).window(SlidingEventTimeWindows.of(Time.milliseconds(windowSize), Time.milliseconds(windowSlide))).reduce(new ReduceFunction<Tuple2<Long, IntType>>() {

            @Override
            public Tuple2<Long, IntType> reduce(Tuple2<Long, IntType> a, Tuple2<Long, IntType> b) {
                // validate that the function has been opened properly
                return new Tuple2<>(a.f0, new IntType(a.f1.value + b.f1.value));
            }
        }, new RichWindowFunction<Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, Tuple, TimeWindow>() {

            private boolean open = false;

            @Override
            public void open(Configuration parameters) {
                assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
                open = true;
            }

            @Override
            public void apply(Tuple tuple, TimeWindow window, Iterable<Tuple2<Long, IntType>> input, Collector<Tuple4<Long, Long, Long, IntType>> out) {
                // validate that the function has been opened properly
                assertTrue(open);
                for (Tuple2<Long, IntType> in : input) {
                    out.collect(new Tuple4<>(in.f0, window.getStart(), window.getEnd(), in.f1));
                }
            }
        }).addSink(new ValidatingSink<>(new SinkValidatorUpdateFun(numElementsPerKey), new SinkValidatorCheckFun(numKeys, numElementsPerKey, windowSlide))).setParallelism(1);
        env.execute("Tumbling Window Test");
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 3 with FailingSource

use of org.apache.flink.test.checkpointing.utils.FailingSource in project flink by apache.

the class EventTimeWindowCheckpointingITCase method testTumblingTimeWindow.

// ------------------------------------------------------------------------
@Test
public void testTumblingTimeWindow() {
    final int numElementsPerKey = numElementsPerKey();
    final int windowSize = windowSize();
    final int numKeys = numKeys();
    try {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(PARALLELISM);
        env.enableCheckpointing(100);
        env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
        env.setStateBackend(this.stateBackend);
        env.getConfig().setUseSnapshotCompression(true);
        env.addSource(new FailingSource(new KeyedEventTimeGenerator(numKeys, windowSize), numElementsPerKey)).rebalance().keyBy(0).window(TumblingEventTimeWindows.of(Time.milliseconds(windowSize))).apply(new RichWindowFunction<Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, Tuple, TimeWindow>() {

            private boolean open = false;

            @Override
            public void open(Configuration parameters) {
                assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
                open = true;
            }

            @Override
            public void apply(Tuple tuple, TimeWindow window, Iterable<Tuple2<Long, IntType>> values, Collector<Tuple4<Long, Long, Long, IntType>> out) {
                // validate that the function has been opened properly
                assertTrue(open);
                int sum = 0;
                long key = -1;
                for (Tuple2<Long, IntType> value : values) {
                    sum += value.f1.value;
                    key = value.f0;
                }
                final Tuple4<Long, Long, Long, IntType> result = new Tuple4<>(key, window.getStart(), window.getEnd(), new IntType(sum));
                out.collect(result);
            }
        }).addSink(new ValidatingSink<>(new SinkValidatorUpdateFun(numElementsPerKey), new SinkValidatorCheckFun(numKeys, numElementsPerKey, windowSize))).setParallelism(1);
        env.execute("Tumbling Window Test");
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 4 with FailingSource

use of org.apache.flink.test.checkpointing.utils.FailingSource in project flink by apache.

the class EventTimeAllWindowCheckpointingITCase method testSlidingTimeWindow.

@Test
public void testSlidingTimeWindow() {
    final int numElementsPerKey = 3000;
    final int windowSize = 1000;
    final int windowSlide = 100;
    final int numKeys = 1;
    try {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(PARALLELISM);
        env.enableCheckpointing(100);
        env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
        env.addSource(new FailingSource(new EventTimeWindowCheckpointingITCase.KeyedEventTimeGenerator(numKeys, windowSlide), numElementsPerKey)).rebalance().windowAll(SlidingEventTimeWindows.of(Time.milliseconds(windowSize), Time.milliseconds(windowSlide))).apply(new RichAllWindowFunction<Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, TimeWindow>() {

            private boolean open = false;

            @Override
            public void open(Configuration parameters) {
                assertEquals(1, getRuntimeContext().getNumberOfParallelSubtasks());
                open = true;
            }

            @Override
            public void apply(TimeWindow window, Iterable<Tuple2<Long, IntType>> values, Collector<Tuple4<Long, Long, Long, IntType>> out) {
                // validate that the function has been opened properly
                assertTrue(open);
                int sum = 0;
                long key = -1;
                for (Tuple2<Long, IntType> value : values) {
                    sum += value.f1.value;
                    key = value.f0;
                }
                out.collect(new Tuple4<>(key, window.getStart(), window.getEnd(), new IntType(sum)));
            }
        }).addSink(new ValidatingSink<>(new EventTimeWindowCheckpointingITCase.SinkValidatorUpdateFun(numElementsPerKey), new EventTimeWindowCheckpointingITCase.SinkValidatorCheckFun(numKeys, numElementsPerKey, windowSlide))).setParallelism(1);
        env.execute("Sliding Window Test");
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 5 with FailingSource

use of org.apache.flink.test.checkpointing.utils.FailingSource in project flink by apache.

the class ProcessingTimeWindowCheckpointingITCase method testSlidingProcessingTimeWindow.

@Test
public void testSlidingProcessingTimeWindow() {
    final int numElements = 3000;
    try {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(PARALLELISM);
        env.getConfig().setAutoWatermarkInterval(10);
        env.enableCheckpointing(100);
        env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
        SinkValidatorUpdaterAndChecker updaterAndChecker = new SinkValidatorUpdaterAndChecker(numElements, 3);
        env.addSource(new FailingSource(new Generator(), numElements, true)).rebalance().keyBy(0).window(SlidingProcessingTimeWindows.of(Time.milliseconds(150), Time.milliseconds(50))).apply(new RichWindowFunction<Tuple2<Long, IntType>, Tuple2<Long, IntType>, Tuple, TimeWindow>() {

            private boolean open = false;

            @Override
            public void open(Configuration parameters) {
                assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
                open = true;
            }

            @Override
            public void apply(Tuple tuple, TimeWindow window, Iterable<Tuple2<Long, IntType>> values, Collector<Tuple2<Long, IntType>> out) {
                // validate that the function has been opened properly
                assertTrue(open);
                for (Tuple2<Long, IntType> value : values) {
                    assertEquals(value.f0.intValue(), value.f1.value);
                    out.collect(new Tuple2<>(value.f0, new IntType(1)));
                }
            }
        }).addSink(new ValidatingSink<>(updaterAndChecker, updaterAndChecker, true)).setParallelism(1);
        tryExecute(env, "Sliding Window Test");
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}