Examples with StreamExecutionEnvironment org.apache.flink.streaming.api.environment.StreamExecutionEnvironment used on opensource projects

Example 81 with StreamExecutionEnvironment

use of org.apache.flink.streaming.api.environment.StreamExecutionEnvironment in project flink by apache.

the class ManualWindowSpeedITCase method testTumblingIngestionTimeWindowsWithFsBackendWithLateness.

@Test
public void testTumblingIngestionTimeWindowsWithFsBackendWithLateness() throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
    env.setParallelism(1);
    String checkpoints = tempFolder.newFolder().toURI().toString();
    env.setStateBackend(new FsStateBackend(checkpoints));
    env.addSource(new InfiniteTupleSource(10_000)).keyBy(0).timeWindow(Time.seconds(3)).allowedLateness(Time.seconds(1)).reduce(new ReduceFunction<Tuple2<String, Integer>>() {

        private static final long serialVersionUID = 1L;

        @Override
        public Tuple2<String, Integer> reduce(Tuple2<String, Integer> value1, Tuple2<String, Integer> value2) throws Exception {
            return Tuple2.of(value1.f0, value1.f1 + value2.f1);
        }
    }).filter(new FilterFunction<Tuple2<String, Integer>>() {

        private static final long serialVersionUID = 1L;

        @Override
        public boolean filter(Tuple2<String, Integer> value) throws Exception {
            return value.f0.startsWith("Tuple 0");
        }
    }).print();
    env.execute();
}

Example 82 with StreamExecutionEnvironment

use of org.apache.flink.streaming.api.environment.StreamExecutionEnvironment in project flink by apache.

the class ManualWindowSpeedITCase method testTumblingIngestionTimeWindowsWithRocksDBBackend.

@Test
public void testTumblingIngestionTimeWindowsWithRocksDBBackend() throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
    env.setParallelism(1);
    env.setStateBackend(new RocksDBStateBackend(new MemoryStateBackend()));
    env.addSource(new InfiniteTupleSource(10_000)).keyBy(0).timeWindow(Time.seconds(3)).reduce(new ReduceFunction<Tuple2<String, Integer>>() {

        private static final long serialVersionUID = 1L;

        @Override
        public Tuple2<String, Integer> reduce(Tuple2<String, Integer> value1, Tuple2<String, Integer> value2) throws Exception {
            return Tuple2.of(value1.f0, value1.f1 + value2.f1);
        }
    }).filter(new FilterFunction<Tuple2<String, Integer>>() {

        private static final long serialVersionUID = 1L;

        @Override
        public boolean filter(Tuple2<String, Integer> value) throws Exception {
            return value.f0.startsWith("Tuple 0");
        }
    }).print();
    env.execute();
}

Example 83 with StreamExecutionEnvironment

use of org.apache.flink.streaming.api.environment.StreamExecutionEnvironment in project flink by apache.

the class ManualWindowSpeedITCase method testAlignedProcessingTimeWindows.

@Test
public void testAlignedProcessingTimeWindows() throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);
    env.setParallelism(1);
    env.setStateBackend(new RocksDBStateBackend(new MemoryStateBackend()));
    env.addSource(new InfiniteTupleSource(10_000)).keyBy(0).timeWindow(Time.seconds(3)).reduce(new ReduceFunction<Tuple2<String, Integer>>() {

        private static final long serialVersionUID = 1L;

        @Override
        public Tuple2<String, Integer> reduce(Tuple2<String, Integer> value1, Tuple2<String, Integer> value2) throws Exception {
            return Tuple2.of(value1.f0, value1.f1 + value2.f1);
        }
    }).filter(new FilterFunction<Tuple2<String, Integer>>() {

        private static final long serialVersionUID = 1L;

        @Override
        public boolean filter(Tuple2<String, Integer> value) throws Exception {
            return value.f0.startsWith("Tuple 0");
        }
    }).print();
    env.execute();
}

Example 84 with StreamExecutionEnvironment

use of org.apache.flink.streaming.api.environment.StreamExecutionEnvironment in project flink by apache.

the class StreamingOperatorsITCase method testGroupedFoldOperation.

/**
	 * Tests the proper functioning of the streaming fold operator. For this purpose, a stream
	 * of Tuple2<Integer, Integer> is created. The stream is grouped according to the first tuple
	 * value. Each group is folded where the second tuple value is summed up.
	 *
	 * This test relies on the hash function used by the {@link DataStream#keyBy}, which is
	 * assumed to be {@link MathUtils#murmurHash}.
	 */
@Test
public void testGroupedFoldOperation() throws Exception {
    int numElements = 10;
    final int numKeys = 2;
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<Tuple2<Integer, Integer>> sourceStream = env.addSource(new TupleSource(numElements, numKeys));
    SplitStream<Tuple2<Integer, Integer>> splittedResult = sourceStream.keyBy(0).fold(0, new FoldFunction<Tuple2<Integer, Integer>, Integer>() {

        private static final long serialVersionUID = 4875723041825726082L;

        @Override
        public Integer fold(Integer accumulator, Tuple2<Integer, Integer> value) throws Exception {
            return accumulator + value.f1;
        }
    }).map(new RichMapFunction<Integer, Tuple2<Integer, Integer>>() {

        private static final long serialVersionUID = 8538355101606319744L;

        int key = -1;

        @Override
        public Tuple2<Integer, Integer> map(Integer value) throws Exception {
            if (key == -1) {
                key = MathUtils.murmurHash(value) % numKeys;
            }
            return new Tuple2<>(key, value);
        }
    }).split(new OutputSelector<Tuple2<Integer, Integer>>() {

        private static final long serialVersionUID = -8439325199163362470L;

        @Override
        public Iterable<String> select(Tuple2<Integer, Integer> value) {
            List<String> output = new ArrayList<>();
            output.add(value.f0 + "");
            return output;
        }
    });
    final MemorySinkFunction sinkFunction1 = new MemorySinkFunction(0);
    final List<Integer> actualResult1 = new ArrayList<>();
    MemorySinkFunction.registerCollection(0, actualResult1);
    splittedResult.select("0").map(new MapFunction<Tuple2<Integer, Integer>, Integer>() {

        private static final long serialVersionUID = 2114608668010092995L;

        @Override
        public Integer map(Tuple2<Integer, Integer> value) throws Exception {
            return value.f1;
        }
    }).addSink(sinkFunction1);
    final MemorySinkFunction sinkFunction2 = new MemorySinkFunction(1);
    final List<Integer> actualResult2 = new ArrayList<>();
    MemorySinkFunction.registerCollection(1, actualResult2);
    splittedResult.select("1").map(new MapFunction<Tuple2<Integer, Integer>, Integer>() {

        private static final long serialVersionUID = 5631104389744681308L;

        @Override
        public Integer map(Tuple2<Integer, Integer> value) throws Exception {
            return value.f1;
        }
    }).addSink(sinkFunction2);
    Collection<Integer> expected1 = new ArrayList<>(10);
    Collection<Integer> expected2 = new ArrayList<>(10);
    int counter1 = 0;
    int counter2 = 0;
    for (int i = 0; i < numElements; i++) {
        if (MathUtils.murmurHash(i) % numKeys == 0) {
            counter1 += i;
            expected1.add(counter1);
        } else {
            counter2 += i;
            expected2.add(counter2);
        }
    }
    env.execute();
    Collections.sort(actualResult1);
    Collections.sort(actualResult2);
    Assert.assertEquals(expected1, actualResult1);
    Assert.assertEquals(expected2, actualResult2);
    MemorySinkFunction.clear();
}

Example 85 with StreamExecutionEnvironment

use of org.apache.flink.streaming.api.environment.StreamExecutionEnvironment in project flink by apache.

the class PartitionerITCase method testForwardFailsHightToLowParallelism.

@Test(expected = UnsupportedOperationException.class)
public void testForwardFailsHightToLowParallelism() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    // this does a rebalance that works
    DataStream<Integer> src = env.fromElements(1, 2, 3).map(new NoOpIntMap());
    // this doesn't work because it goes from 3 to 1
    src.forward().map(new NoOpIntMap()).setParallelism(1);
    env.execute();
}