Examples with StreamExecutionEnvironment org.apache.flink.streaming.api.environment.StreamExecutionEnvironment used on opensource projects

Example 86 with StreamExecutionEnvironment

use of org.apache.flink.streaming.api.environment.StreamExecutionEnvironment in project flink by apache.

the class SelfConnectionITCase method differentDataStreamSameChain.

/**
	 * We connect two different data streams in a chain to a CoMap.
	 */
@Test
public void differentDataStreamSameChain() throws Exception {
    TestListResultSink<String> resultSink = new TestListResultSink<String>();
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(1);
    DataStream<Integer> src = env.fromElements(1, 3, 5);
    DataStream<String> stringMap = src.map(new MapFunction<Integer, String>() {

        private static final long serialVersionUID = 1L;

        @Override
        public String map(Integer value) throws Exception {
            return "x " + value;
        }
    });
    stringMap.connect(src).map(new CoMapFunction<String, Integer, String>() {

        private static final long serialVersionUID = 1L;

        @Override
        public String map1(String value) {
            return value;
        }

        @Override
        public String map2(Integer value) {
            return String.valueOf(value + 1);
        }
    }).addSink(resultSink);
    env.execute();
    List<String> expected = Arrays.asList("x 1", "x 3", "x 5", "2", "4", "6");
    List<String> result = resultSink.getResult();
    Collections.sort(expected);
    Collections.sort(result);
    assertEquals(expected, result);
}

Example 87 with StreamExecutionEnvironment

use of org.apache.flink.streaming.api.environment.StreamExecutionEnvironment in project flink by apache.

the class SelfConnectionITCase method differentDataStreamDifferentChain.

/**
	 * We connect two different data streams in different chains to a CoMap.
	 * (This is not actually self-connect.)
	 */
@Test
public void differentDataStreamDifferentChain() {
    TestListResultSink<String> resultSink = new TestListResultSink<String>();
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(3);
    DataStream<Integer> src = env.fromElements(1, 3, 5).disableChaining();
    DataStream<String> stringMap = src.flatMap(new FlatMapFunction<Integer, String>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void flatMap(Integer value, Collector<String> out) throws Exception {
            out.collect("x " + value);
        }
    }).keyBy(new KeySelector<String, Integer>() {

        private static final long serialVersionUID = 1L;

        @Override
        public Integer getKey(String value) throws Exception {
            return value.length();
        }
    });
    DataStream<Long> longMap = src.map(new MapFunction<Integer, Long>() {

        private static final long serialVersionUID = 1L;

        @Override
        public Long map(Integer value) throws Exception {
            return (long) (value + 1);
        }
    }).keyBy(new KeySelector<Long, Integer>() {

        private static final long serialVersionUID = 1L;

        @Override
        public Integer getKey(Long value) throws Exception {
            return value.intValue();
        }
    });
    stringMap.connect(longMap).map(new CoMapFunction<String, Long, String>() {

        private static final long serialVersionUID = 1L;

        @Override
        public String map1(String value) {
            return value;
        }

        @Override
        public String map2(Long value) {
            return value.toString();
        }
    }).addSink(resultSink);
    try {
        env.execute();
    } catch (Exception e) {
        e.printStackTrace();
    }
    List<String> expected = Arrays.asList("x 1", "x 3", "x 5", "2", "4", "6");
    List<String> result = resultSink.getResult();
    Collections.sort(expected);
    Collections.sort(result);
    assertEquals(expected, result);
}

Example 88 with StreamExecutionEnvironment

use of org.apache.flink.streaming.api.environment.StreamExecutionEnvironment in project flink by apache.

the class SideOutputITCase method testSideOutputNameClash.

@Test
public void testSideOutputNameClash() throws Exception {
    final OutputTag<String> sideOutputTag1 = new OutputTag<String>("side") {
    };
    final OutputTag<Integer> sideOutputTag2 = new OutputTag<Integer>("side") {
    };
    TestListResultSink<String> sideOutputResultSink1 = new TestListResultSink<>();
    TestListResultSink<Integer> sideOutputResultSink2 = new TestListResultSink<>();
    StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
    see.setParallelism(3);
    DataStream<Integer> dataStream = see.fromCollection(elements);
    SingleOutputStreamOperator<Integer> passThroughtStream = dataStream.process(new ProcessFunction<Integer, Integer>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void processElement(Integer value, Context ctx, Collector<Integer> out) throws Exception {
            out.collect(value);
            ctx.output(sideOutputTag1, "sideout-" + String.valueOf(value));
            ctx.output(sideOutputTag2, 13);
        }
    });
    passThroughtStream.getSideOutput(sideOutputTag1).addSink(sideOutputResultSink1);
    expectedException.expect(UnsupportedOperationException.class);
    passThroughtStream.getSideOutput(sideOutputTag2).addSink(sideOutputResultSink2);
}

Example 89 with StreamExecutionEnvironment

use of org.apache.flink.streaming.api.environment.StreamExecutionEnvironment in project flink by apache.

the class SideOutputITCase method testSideOutputWithMultipleConsumersWithObjectReuse.

@Test
public void testSideOutputWithMultipleConsumersWithObjectReuse() throws Exception {
    final OutputTag<String> sideOutputTag = new OutputTag<String>("side") {
    };
    TestListResultSink<String> sideOutputResultSink1 = new TestListResultSink<>();
    TestListResultSink<String> sideOutputResultSink2 = new TestListResultSink<>();
    TestListResultSink<Integer> resultSink = new TestListResultSink<>();
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().enableObjectReuse();
    env.setParallelism(3);
    DataStream<Integer> dataStream = env.fromCollection(elements);
    SingleOutputStreamOperator<Integer> passThroughtStream = dataStream.process(new ProcessFunction<Integer, Integer>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void processElement(Integer value, Context ctx, Collector<Integer> out) throws Exception {
            out.collect(value);
            ctx.output(sideOutputTag, "sideout-" + String.valueOf(value));
        }
    });
    passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink1);
    passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink2);
    passThroughtStream.addSink(resultSink);
    env.execute();
    assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink1.getSortedResult());
    assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink2.getSortedResult());
    assertEquals(Arrays.asList(1, 2, 3, 4, 5), resultSink.getSortedResult());
}

Example 90 with StreamExecutionEnvironment

use of org.apache.flink.streaming.api.environment.StreamExecutionEnvironment in project flink by apache.

the class SideOutputITCase method testSideOutputWithMultipleConsumers.

@Test
public void testSideOutputWithMultipleConsumers() throws Exception {
    final OutputTag<String> sideOutputTag = new OutputTag<String>("side") {
    };
    TestListResultSink<String> sideOutputResultSink1 = new TestListResultSink<>();
    TestListResultSink<String> sideOutputResultSink2 = new TestListResultSink<>();
    TestListResultSink<Integer> resultSink = new TestListResultSink<>();
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(3);
    DataStream<Integer> dataStream = env.fromCollection(elements);
    SingleOutputStreamOperator<Integer> passThroughtStream = dataStream.process(new ProcessFunction<Integer, Integer>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void processElement(Integer value, Context ctx, Collector<Integer> out) throws Exception {
            out.collect(value);
            ctx.output(sideOutputTag, "sideout-" + String.valueOf(value));
        }
    });
    passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink1);
    passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink2);
    passThroughtStream.addSink(resultSink);
    env.execute();
    assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink1.getSortedResult());
    assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink2.getSortedResult());
    assertEquals(Arrays.asList(1, 2, 3, 4, 5), resultSink.getSortedResult());
}