use of org.apache.flink.test.checkpointing.utils.ValidatingSink in project flink by apache.
the class ProcessingTimeWindowCheckpointingITCase method testAggregatingSlidingProcessingTimeWindow.
@Test
public void testAggregatingSlidingProcessingTimeWindow() {
final int numElements = 3000;
try {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(PARALLELISM);
env.getConfig().setAutoWatermarkInterval(10);
env.enableCheckpointing(100);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
SinkValidatorUpdaterAndChecker updaterAndChecker = new SinkValidatorUpdaterAndChecker(numElements, 3);
env.addSource(new FailingSource(new Generator(), numElements, true)).map(new MapFunction<Tuple2<Long, IntType>, Tuple2<Long, IntType>>() {
@Override
public Tuple2<Long, IntType> map(Tuple2<Long, IntType> value) {
value.f1.value = 1;
return value;
}
}).rebalance().keyBy(0).window(SlidingProcessingTimeWindows.of(Time.milliseconds(150), Time.milliseconds(50))).reduce(new ReduceFunction<Tuple2<Long, IntType>>() {
@Override
public Tuple2<Long, IntType> reduce(Tuple2<Long, IntType> a, Tuple2<Long, IntType> b) {
return new Tuple2<>(a.f0, new IntType(1));
}
}).addSink(new ValidatingSink<>(updaterAndChecker, updaterAndChecker, true)).setParallelism(1);
tryExecute(env, "Aggregating Sliding Window Test");
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
use of org.apache.flink.test.checkpointing.utils.ValidatingSink in project flink by apache.
the class ProcessingTimeWindowCheckpointingITCase method testAggregatingTumblingProcessingTimeWindow.
@Test
public void testAggregatingTumblingProcessingTimeWindow() {
final int numElements = 3000;
try {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(PARALLELISM);
env.getConfig().setAutoWatermarkInterval(10);
env.enableCheckpointing(100);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
SinkValidatorUpdaterAndChecker updaterAndChecker = new SinkValidatorUpdaterAndChecker(numElements, 1);
env.addSource(new FailingSource(new Generator(), numElements, true)).map(new MapFunction<Tuple2<Long, IntType>, Tuple2<Long, IntType>>() {
@Override
public Tuple2<Long, IntType> map(Tuple2<Long, IntType> value) {
value.f1.value = 1;
return value;
}
}).rebalance().keyBy(0).window(TumblingProcessingTimeWindows.of(Time.milliseconds(100))).reduce(new ReduceFunction<Tuple2<Long, IntType>>() {
@Override
public Tuple2<Long, IntType> reduce(Tuple2<Long, IntType> a, Tuple2<Long, IntType> b) {
return new Tuple2<>(a.f0, new IntType(1));
}
}).addSink(new ValidatingSink<>(updaterAndChecker, updaterAndChecker, true)).setParallelism(1);
tryExecute(env, "Aggregating Tumbling Window Test");
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
use of org.apache.flink.test.checkpointing.utils.ValidatingSink in project flink by apache.
the class ProcessingTimeWindowCheckpointingITCase method testTumblingProcessingTimeWindow.
// ------------------------------------------------------------------------
@Test
public void testTumblingProcessingTimeWindow() {
final int numElements = 3000;
try {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(PARALLELISM);
env.getConfig().setAutoWatermarkInterval(10);
env.enableCheckpointing(100);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
SinkValidatorUpdaterAndChecker updaterAndChecker = new SinkValidatorUpdaterAndChecker(numElements, 1);
env.addSource(new FailingSource(new Generator(), numElements, true)).rebalance().keyBy(0).window(TumblingProcessingTimeWindows.of(Time.milliseconds(100))).apply(new RichWindowFunction<Tuple2<Long, IntType>, Tuple2<Long, IntType>, Tuple, TimeWindow>() {
private boolean open = false;
@Override
public void open(Configuration parameters) {
assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
open = true;
}
@Override
public void apply(Tuple tuple, TimeWindow window, Iterable<Tuple2<Long, IntType>> values, Collector<Tuple2<Long, IntType>> out) {
// validate that the function has been opened properly
assertTrue(open);
for (Tuple2<Long, IntType> value : values) {
assertEquals(value.f0.intValue(), value.f1.value);
out.collect(new Tuple2<>(value.f0, new IntType(1)));
}
}
}).addSink(new ValidatingSink<>(updaterAndChecker, updaterAndChecker, true)).setParallelism(1);
tryExecute(env, "Tumbling Window Test");
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
use of org.apache.flink.test.checkpointing.utils.ValidatingSink in project flink by apache.
the class EventTimeWindowCheckpointingITCase method testPreAggregatedTumblingTimeWindow.
@Test
public void testPreAggregatedTumblingTimeWindow() {
final int numElementsPerKey = numElementsPerKey();
final int windowSize = windowSize();
final int numKeys = numKeys();
try {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(PARALLELISM);
env.enableCheckpointing(100);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
env.setStateBackend(this.stateBackend);
env.getConfig().setUseSnapshotCompression(true);
env.addSource(new FailingSource(new KeyedEventTimeGenerator(numKeys, windowSize), numElementsPerKey)).rebalance().keyBy(0).window(TumblingEventTimeWindows.of(Time.milliseconds(windowSize))).reduce(new ReduceFunction<Tuple2<Long, IntType>>() {
@Override
public Tuple2<Long, IntType> reduce(Tuple2<Long, IntType> a, Tuple2<Long, IntType> b) {
return new Tuple2<>(a.f0, new IntType(a.f1.value + b.f1.value));
}
}, new RichWindowFunction<Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, Tuple, TimeWindow>() {
private boolean open = false;
@Override
public void open(Configuration parameters) {
assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
open = true;
}
@Override
public void apply(Tuple tuple, TimeWindow window, Iterable<Tuple2<Long, IntType>> input, Collector<Tuple4<Long, Long, Long, IntType>> out) {
// validate that the function has been opened properly
assertTrue(open);
for (Tuple2<Long, IntType> in : input) {
final Tuple4<Long, Long, Long, IntType> output = new Tuple4<>(in.f0, window.getStart(), window.getEnd(), in.f1);
out.collect(output);
}
}
}).addSink(new ValidatingSink<>(new SinkValidatorUpdateFun(numElementsPerKey), new SinkValidatorCheckFun(numKeys, numElementsPerKey, windowSize))).setParallelism(1);
env.execute("Tumbling Window Test");
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
use of org.apache.flink.test.checkpointing.utils.ValidatingSink in project flink by apache.
the class EventTimeWindowCheckpointingITCase method testSlidingTimeWindow.
@Test
public void testSlidingTimeWindow() {
final int numElementsPerKey = numElementsPerKey();
final int windowSize = windowSize();
final int windowSlide = windowSlide();
final int numKeys = numKeys();
try {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setMaxParallelism(2 * PARALLELISM);
env.setParallelism(PARALLELISM);
env.enableCheckpointing(100);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
env.setStateBackend(this.stateBackend);
env.getConfig().setUseSnapshotCompression(true);
env.addSource(new FailingSource(new KeyedEventTimeGenerator(numKeys, windowSlide), numElementsPerKey)).rebalance().keyBy(0).window(SlidingEventTimeWindows.of(Time.milliseconds(windowSize), Time.milliseconds(windowSlide))).apply(new RichWindowFunction<Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, Tuple, TimeWindow>() {
private boolean open = false;
@Override
public void open(Configuration parameters) {
assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
open = true;
}
@Override
public void apply(Tuple tuple, TimeWindow window, Iterable<Tuple2<Long, IntType>> values, Collector<Tuple4<Long, Long, Long, IntType>> out) {
// validate that the function has been opened properly
assertTrue(open);
int sum = 0;
long key = -1;
for (Tuple2<Long, IntType> value : values) {
sum += value.f1.value;
key = value.f0;
}
final Tuple4<Long, Long, Long, IntType> output = new Tuple4<>(key, window.getStart(), window.getEnd(), new IntType(sum));
out.collect(output);
}
}).addSink(new ValidatingSink<>(new SinkValidatorUpdateFun(numElementsPerKey), new SinkValidatorCheckFun(numKeys, numElementsPerKey, windowSlide))).setParallelism(1);
env.execute("Tumbling Window Test");
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
Aggregations