Examples with OperatorChain org.apache.flink.streaming.runtime.tasks.OperatorChain used on opensource projects

Example 1 with OperatorChain

use of org.apache.flink.streaming.runtime.tasks.OperatorChain in project flink by apache.

the class StreamOperatorChainingTest method testMultiChainingWithSplit.

/**
	 * Verify that multi-chaining works with object reuse enabled.
	 */
private void testMultiChainingWithSplit(StreamExecutionEnvironment env) throws Exception {
    // the actual elements will not be used
    DataStream<Integer> input = env.fromElements(1, 2, 3);
    sink1Results = new ArrayList<>();
    sink2Results = new ArrayList<>();
    sink3Results = new ArrayList<>();
    input = input.map(new MapFunction<Integer, Integer>() {

        private static final long serialVersionUID = 1L;

        @Override
        public Integer map(Integer value) throws Exception {
            return value;
        }
    });
    SplitStream<Integer> split = input.split(new OutputSelector<Integer>() {

        private static final long serialVersionUID = 1L;

        @Override
        public Iterable<String> select(Integer value) {
            if (value.equals(1)) {
                return Collections.singletonList("one");
            } else {
                return Collections.singletonList("other");
            }
        }
    });
    split.select("one").map(new MapFunction<Integer, String>() {

        private static final long serialVersionUID = 1L;

        @Override
        public String map(Integer value) throws Exception {
            return "First 1: " + value;
        }
    }).addSink(new SinkFunction<String>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void invoke(String value) throws Exception {
            sink1Results.add(value);
        }
    });
    split.select("one").map(new MapFunction<Integer, String>() {

        private static final long serialVersionUID = 1L;

        @Override
        public String map(Integer value) throws Exception {
            return "First 2: " + value;
        }
    }).addSink(new SinkFunction<String>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void invoke(String value) throws Exception {
            sink2Results.add(value);
        }
    });
    split.select("other").map(new MapFunction<Integer, String>() {

        private static final long serialVersionUID = 1L;

        @Override
        public String map(Integer value) throws Exception {
            return "Second: " + value;
        }
    }).addSink(new SinkFunction<String>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void invoke(String value) throws Exception {
            sink3Results.add(value);
        }
    });
    // be build our own StreamTask and OperatorChain
    JobGraph jobGraph = env.getStreamGraph().getJobGraph();
    Assert.assertTrue(jobGraph.getVerticesSortedTopologicallyFromSources().size() == 2);
    JobVertex chainedVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(1);
    Configuration configuration = chainedVertex.getConfiguration();
    StreamConfig streamConfig = new StreamConfig(configuration);
    StreamMap<Integer, Integer> headOperator = streamConfig.getStreamOperator(Thread.currentThread().getContextClassLoader());
    StreamTask<Integer, StreamMap<Integer, Integer>> mockTask = createMockTask(streamConfig, chainedVertex.getName());
    OperatorChain<Integer, StreamMap<Integer, Integer>> operatorChain = new OperatorChain<>(mockTask);
    headOperator.setup(mockTask, streamConfig, operatorChain.getChainEntryPoint());
    for (StreamOperator<?> operator : operatorChain.getAllOperators()) {
        if (operator != null) {
            operator.open();
        }
    }
    headOperator.processElement(new StreamRecord<>(1));
    headOperator.processElement(new StreamRecord<>(2));
    headOperator.processElement(new StreamRecord<>(3));
    assertThat(sink1Results, contains("First 1: 1"));
    assertThat(sink2Results, contains("First 2: 1"));
    assertThat(sink3Results, contains("Second: 2", "Second: 3"));
}

Example 2 with OperatorChain

use of org.apache.flink.streaming.runtime.tasks.OperatorChain in project flink by apache.

the class StreamOperatorChainingTest method testMultiChaining.

/**
	 * Verify that multi-chaining works.
	 */
private void testMultiChaining(StreamExecutionEnvironment env) throws Exception {
    // the actual elements will not be used
    DataStream<Integer> input = env.fromElements(1, 2, 3);
    sink1Results = new ArrayList<>();
    sink2Results = new ArrayList<>();
    input = input.map(new MapFunction<Integer, Integer>() {

        private static final long serialVersionUID = 1L;

        @Override
        public Integer map(Integer value) throws Exception {
            return value;
        }
    });
    input.map(new MapFunction<Integer, String>() {

        private static final long serialVersionUID = 1L;

        @Override
        public String map(Integer value) throws Exception {
            return "First: " + value;
        }
    }).addSink(new SinkFunction<String>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void invoke(String value) throws Exception {
            sink1Results.add(value);
        }
    });
    input.map(new MapFunction<Integer, String>() {

        private static final long serialVersionUID = 1L;

        @Override
        public String map(Integer value) throws Exception {
            return "Second: " + value;
        }
    }).addSink(new SinkFunction<String>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void invoke(String value) throws Exception {
            sink2Results.add(value);
        }
    });
    // be build our own StreamTask and OperatorChain
    JobGraph jobGraph = env.getStreamGraph().getJobGraph();
    Assert.assertTrue(jobGraph.getVerticesSortedTopologicallyFromSources().size() == 2);
    JobVertex chainedVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(1);
    Configuration configuration = chainedVertex.getConfiguration();
    StreamConfig streamConfig = new StreamConfig(configuration);
    StreamMap<Integer, Integer> headOperator = streamConfig.getStreamOperator(Thread.currentThread().getContextClassLoader());
    StreamTask<Integer, StreamMap<Integer, Integer>> mockTask = createMockTask(streamConfig, chainedVertex.getName());
    OperatorChain<Integer, StreamMap<Integer, Integer>> operatorChain = new OperatorChain<>(mockTask);
    headOperator.setup(mockTask, streamConfig, operatorChain.getChainEntryPoint());
    for (StreamOperator<?> operator : operatorChain.getAllOperators()) {
        if (operator != null) {
            operator.open();
        }
    }
    headOperator.processElement(new StreamRecord<>(1));
    headOperator.processElement(new StreamRecord<>(2));
    headOperator.processElement(new StreamRecord<>(3));
    assertThat(sink1Results, contains("First: 1", "First: 2", "First: 3"));
    assertThat(sink2Results, contains("Second: 1", "Second: 2", "Second: 3"));
}