Examples with StreamExecutionEnvironment org.apache.flink.streaming.api.environment.StreamExecutionEnvironment used on opensource projects

Example 21 with StreamExecutionEnvironment

use of org.apache.flink.streaming.api.environment.StreamExecutionEnvironment in project flink by apache.

the class WindowTranslationTest method testApplyWithPreReducerEventTime.

/**
	 * Test for the deprecated .apply(Reducer, WindowFunction).
	 */
@Test
@SuppressWarnings("rawtypes")
public void testApplyWithPreReducerEventTime() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
    DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));
    DummyReducer reducer = new DummyReducer();
    DataStream<Tuple3<String, String, Integer>> window = source.keyBy(new TupleKeySelector()).window(TumblingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS))).apply(reducer, new WindowFunction<Tuple2<String, Integer>, Tuple3<String, String, Integer>, String, TimeWindow>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void apply(String key, TimeWindow window, Iterable<Tuple2<String, Integer>> values, Collector<Tuple3<String, String, Integer>> out) throws Exception {
            for (Tuple2<String, Integer> in : values) {
                out.collect(new Tuple3<>(in.f0, in.f0, in.f1));
            }
        }
    });
    OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>>) window.getTransformation();
    OneInputStreamOperator<Tuple2<String, Integer>, Tuple3<String, String, Integer>> operator = transform.getOperator();
    Assert.assertTrue(operator instanceof WindowOperator);
    WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
    Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
    Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingEventTimeWindows);
    Assert.assertTrue(winOperator.getStateDescriptor() instanceof ReducingStateDescriptor);
    processElementAndEnsureOutput(operator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}

Example 22 with StreamExecutionEnvironment

use of org.apache.flink.streaming.api.environment.StreamExecutionEnvironment in project flink by apache.

the class WindowTranslationTest method testFoldProcessingTime.

@Test
@SuppressWarnings("rawtypes")
public void testFoldProcessingTime() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);
    DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));
    DataStream<Tuple3<String, String, Integer>> window = source.keyBy(new TupleKeySelector()).window(SlidingProcessingTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS))).fold(new Tuple3<>("", "", 0), new DummyFolder());
    OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>>) window.getTransformation();
    OneInputStreamOperator<Tuple2<String, Integer>, Tuple3<String, String, Integer>> operator = transform.getOperator();
    Assert.assertTrue(operator instanceof WindowOperator);
    WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
    Assert.assertTrue(winOperator.getTrigger() instanceof ProcessingTimeTrigger);
    Assert.assertTrue(winOperator.getWindowAssigner() instanceof SlidingProcessingTimeWindows);
    Assert.assertTrue(winOperator.getStateDescriptor() instanceof FoldingStateDescriptor);
    processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}

Example 23 with StreamExecutionEnvironment

use of org.apache.flink.streaming.api.environment.StreamExecutionEnvironment in project flink by apache.

the class CEPITCase method testSimpleOrFilterPatternCEP.

/**
	 * Checks that a certain event sequence is recognized with an OR filter
	 * @throws Exception
	 */
@Test
public void testSimpleOrFilterPatternCEP() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<Event> input = env.fromElements(new Event(1, "start", 1.0), new Event(2, "middle", 2.0), new Event(3, "end", 3.0), new Event(4, "start", 4.0), new Event(5, "middle", 5.0), new Event(6, "end", 6.0));
    Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new FilterFunction<Event>() {

        @Override
        public boolean filter(Event value) throws Exception {
            return value.getName().equals("start");
        }
    }).followedBy("middle").where(new FilterFunction<Event>() {

        @Override
        public boolean filter(Event value) throws Exception {
            return value.getPrice() == 2.0;
        }
    }).or(new FilterFunction<Event>() {

        @Override
        public boolean filter(Event value) throws Exception {
            return value.getPrice() == 5.0;
        }
    }).followedBy("end").where(new FilterFunction<Event>() {

        @Override
        public boolean filter(Event value) throws Exception {
            return value.getName().equals("end");
        }
    });
    DataStream<String> result = CEP.pattern(input, pattern).select(new PatternSelectFunction<Event, String>() {

        @Override
        public String select(Map<String, Event> pattern) {
            StringBuilder builder = new StringBuilder();
            builder.append(pattern.get("start").getId()).append(",").append(pattern.get("middle").getId()).append(",").append(pattern.get("end").getId());
            return builder.toString();
        }
    });
    result.writeAsText(resultPath, FileSystem.WriteMode.OVERWRITE);
    // expected sequence of matching event ids
    expected = "1,5,6\n1,2,3\n4,5,6\n1,2,6";
    env.execute();
}

Example 24 with StreamExecutionEnvironment

use of org.apache.flink.streaming.api.environment.StreamExecutionEnvironment in project flink by apache.

the class CEPITCase method testSimplePatternWithSingleState.

@Test
public void testSimplePatternWithSingleState() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<Tuple2<Integer, Integer>> input = env.fromElements(new Tuple2<>(0, 1), new Tuple2<>(0, 2));
    Pattern<Tuple2<Integer, Integer>, ?> pattern = Pattern.<Tuple2<Integer, Integer>>begin("start").where(new FilterFunction<Tuple2<Integer, Integer>>() {

        @Override
        public boolean filter(Tuple2<Integer, Integer> rec) throws Exception {
            return rec.f1 == 1;
        }
    });
    PatternStream<Tuple2<Integer, Integer>> pStream = CEP.pattern(input, pattern);
    DataStream<Tuple2<Integer, Integer>> result = pStream.select(new PatternSelectFunction<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>>() {

        @Override
        public Tuple2<Integer, Integer> select(Map<String, Tuple2<Integer, Integer>> pattern) throws Exception {
            return pattern.get("start");
        }
    });
    result.writeAsText(resultPath, FileSystem.WriteMode.OVERWRITE);
    expected = "(0,1)";
    env.execute();
}

Example 25 with StreamExecutionEnvironment

use of org.apache.flink.streaming.api.environment.StreamExecutionEnvironment in project flink by apache.

the class CEPITCase method testSimplePatternEventTime.

@Test
public void testSimplePatternEventTime() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
    // (Event, timestamp)
    DataStream<Event> input = env.fromElements(Tuple2.of(new Event(1, "start", 1.0), 5L), Tuple2.of(new Event(2, "middle", 2.0), 1L), Tuple2.of(new Event(3, "end", 3.0), 3L), Tuple2.of(new Event(4, "end", 4.0), 10L), Tuple2.of(new Event(5, "middle", 5.0), 7L), // last element for high final watermark
    Tuple2.of(new Event(5, "middle", 5.0), 100L)).assignTimestampsAndWatermarks(new AssignerWithPunctuatedWatermarks<Tuple2<Event, Long>>() {

        @Override
        public long extractTimestamp(Tuple2<Event, Long> element, long previousTimestamp) {
            return element.f1;
        }

        @Override
        public Watermark checkAndGetNextWatermark(Tuple2<Event, Long> lastElement, long extractedTimestamp) {
            return new Watermark(lastElement.f1 - 5);
        }
    }).map(new MapFunction<Tuple2<Event, Long>, Event>() {

        @Override
        public Event map(Tuple2<Event, Long> value) throws Exception {
            return value.f0;
        }
    });
    Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new FilterFunction<Event>() {

        @Override
        public boolean filter(Event value) throws Exception {
            return value.getName().equals("start");
        }
    }).followedBy("middle").where(new FilterFunction<Event>() {

        @Override
        public boolean filter(Event value) throws Exception {
            return value.getName().equals("middle");
        }
    }).followedBy("end").where(new FilterFunction<Event>() {

        @Override
        public boolean filter(Event value) throws Exception {
            return value.getName().equals("end");
        }
    });
    DataStream<String> result = CEP.pattern(input, pattern).select(new PatternSelectFunction<Event, String>() {

        @Override
        public String select(Map<String, Event> pattern) {
            StringBuilder builder = new StringBuilder();
            builder.append(pattern.get("start").getId()).append(",").append(pattern.get("middle").getId()).append(",").append(pattern.get("end").getId());
            return builder.toString();
        }
    });
    result.writeAsText(resultPath, FileSystem.WriteMode.OVERWRITE);
    // the expected sequence of matching event ids
    expected = "1,5,4";
    env.execute();
}