Examples with StreamGraph org.apache.flink.streaming.api.graph.StreamGraph used on opensource projects

Example 16 with StreamGraph

use of org.apache.flink.streaming.api.graph.StreamGraph in project flink by apache.

the class StreamContextEnvironment method execute.

@Override
public JobExecutionResult execute(String jobName) throws Exception {
    Preconditions.checkNotNull("Streaming Job name should not be null.");
    StreamGraph streamGraph = this.getStreamGraph();
    streamGraph.setJobName(jobName);
    transformations.clear();
    // execute the programs
    if (ctx instanceof DetachedEnvironment) {
        LOG.warn("Job was executed in detached mode, the results will be available on completion.");
        ((DetachedEnvironment) ctx).setDetachedPlan(streamGraph);
        return DetachedEnvironment.DetachedJobExecutionResult.INSTANCE;
    } else {
        return ctx.getClient().run(streamGraph, ctx.getJars(), ctx.getClasspaths(), ctx.getUserCodeClassLoader(), ctx.getSavepointRestoreSettings()).getJobExecutionResult();
    }
}

Example 17 with StreamGraph

use of org.apache.flink.streaming.api.graph.StreamGraph in project flink by apache.

the class StreamExecutionEnvironmentTest method getOperatorFromDataStream.

/////////////////////////////////////////////////////////////
// Utilities
/////////////////////////////////////////////////////////////
private static StreamOperator<?> getOperatorFromDataStream(DataStream<?> dataStream) {
    StreamExecutionEnvironment env = dataStream.getExecutionEnvironment();
    StreamGraph streamGraph = env.getStreamGraph();
    return streamGraph.getStreamNode(dataStream.getId()).getOperator();
}

Example 18 with StreamGraph

use of org.apache.flink.streaming.api.graph.StreamGraph in project flink by apache.

the class DataStreamTest method testUnion.

/**
	 * Tests union functionality. This ensures that self-unions and unions of streams
	 * with differing parallelism work.
	 *
	 * @throws Exception
	 */
@Test
public void testUnion() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(4);
    DataStream<Long> input1 = env.generateSequence(0, 0).map(new MapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            return null;
        }
    });
    DataStream<Long> selfUnion = input1.union(input1).map(new MapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            return null;
        }
    });
    DataStream<Long> input6 = env.generateSequence(0, 0).map(new MapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            return null;
        }
    });
    DataStream<Long> selfUnionDifferentPartition = input6.broadcast().union(input6).map(new MapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            return null;
        }
    });
    DataStream<Long> input2 = env.generateSequence(0, 0).map(new MapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            return null;
        }
    }).setParallelism(4);
    DataStream<Long> input3 = env.generateSequence(0, 0).map(new MapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            return null;
        }
    }).setParallelism(2);
    DataStream<Long> unionDifferingParallelism = input2.union(input3).map(new MapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            return null;
        }
    }).setParallelism(4);
    DataStream<Long> input4 = env.generateSequence(0, 0).map(new MapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            return null;
        }
    }).setParallelism(2);
    DataStream<Long> input5 = env.generateSequence(0, 0).map(new MapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            return null;
        }
    }).setParallelism(4);
    DataStream<Long> unionDifferingPartitioning = input4.broadcast().union(input5).map(new MapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            return null;
        }
    }).setParallelism(4);
    StreamGraph streamGraph = env.getStreamGraph();
    // verify self union
    assertTrue(streamGraph.getStreamNode(selfUnion.getId()).getInEdges().size() == 2);
    for (StreamEdge edge : streamGraph.getStreamNode(selfUnion.getId()).getInEdges()) {
        assertTrue(edge.getPartitioner() instanceof ForwardPartitioner);
    }
    // verify self union with differnt partitioners
    assertTrue(streamGraph.getStreamNode(selfUnionDifferentPartition.getId()).getInEdges().size() == 2);
    boolean hasForward = false;
    boolean hasBroadcast = false;
    for (StreamEdge edge : streamGraph.getStreamNode(selfUnionDifferentPartition.getId()).getInEdges()) {
        if (edge.getPartitioner() instanceof ForwardPartitioner) {
            hasForward = true;
        }
        if (edge.getPartitioner() instanceof BroadcastPartitioner) {
            hasBroadcast = true;
        }
    }
    assertTrue(hasForward && hasBroadcast);
    // verify union of streams with differing parallelism
    assertTrue(streamGraph.getStreamNode(unionDifferingParallelism.getId()).getInEdges().size() == 2);
    for (StreamEdge edge : streamGraph.getStreamNode(unionDifferingParallelism.getId()).getInEdges()) {
        if (edge.getSourceId() == input2.getId()) {
            assertTrue(edge.getPartitioner() instanceof ForwardPartitioner);
        } else if (edge.getSourceId() == input3.getId()) {
            assertTrue(edge.getPartitioner() instanceof RebalancePartitioner);
        } else {
            fail("Wrong input edge.");
        }
    }
    // verify union of streams with differing partitionings
    assertTrue(streamGraph.getStreamNode(unionDifferingPartitioning.getId()).getInEdges().size() == 2);
    for (StreamEdge edge : streamGraph.getStreamNode(unionDifferingPartitioning.getId()).getInEdges()) {
        if (edge.getSourceId() == input4.getId()) {
            assertTrue(edge.getPartitioner() instanceof BroadcastPartitioner);
        } else if (edge.getSourceId() == input5.getId()) {
            assertTrue(edge.getPartitioner() instanceof ForwardPartitioner);
        } else {
            fail("Wrong input edge.");
        }
    }
}

Example 19 with StreamGraph

use of org.apache.flink.streaming.api.graph.StreamGraph in project flink by apache.

the class DataStreamTest method getOperatorForDataStream.

/////////////////////////////////////////////////////////////
// Utilities
/////////////////////////////////////////////////////////////
private static StreamOperator<?> getOperatorForDataStream(DataStream<?> dataStream) {
    StreamExecutionEnvironment env = dataStream.getExecutionEnvironment();
    StreamGraph streamGraph = env.getStreamGraph();
    return streamGraph.getStreamNode(dataStream.getId()).getOperator();
}

Example 20 with StreamGraph

use of org.apache.flink.streaming.api.graph.StreamGraph in project flink by apache.

the class FoldApplyProcessWindowFunctionTest method testFoldWindowFunctionOutputTypeConfigurable.

/**
	 * Tests that the FoldWindowFunction gets the output type serializer set by the
	 * StreamGraphGenerator and checks that the FoldWindowFunction computes the correct result.
	 */
@Test
public void testFoldWindowFunctionOutputTypeConfigurable() throws Exception {
    StreamExecutionEnvironment env = new DummyStreamExecutionEnvironment();
    List<StreamTransformation<?>> transformations = new ArrayList<>();
    int initValue = 1;
    FoldApplyProcessWindowFunction<Integer, TimeWindow, Integer, Integer, Integer> foldWindowFunction = new FoldApplyProcessWindowFunction<>(initValue, new FoldFunction<Integer, Integer>() {

        @Override
        public Integer fold(Integer accumulator, Integer value) throws Exception {
            return accumulator + value;
        }
    }, new ProcessWindowFunction<Integer, Integer, Integer, TimeWindow>() {

        @Override
        public void process(Integer integer, Context context, Iterable<Integer> input, Collector<Integer> out) throws Exception {
            for (Integer in : input) {
                out.collect(in);
            }
        }
    }, BasicTypeInfo.INT_TYPE_INFO);
    AccumulatingProcessingTimeWindowOperator<Integer, Integer, Integer> windowOperator = new AccumulatingProcessingTimeWindowOperator<>(new InternalIterableProcessWindowFunction<>(foldWindowFunction), new KeySelector<Integer, Integer>() {

        private static final long serialVersionUID = -7951310554369722809L;

        @Override
        public Integer getKey(Integer value) throws Exception {
            return value;
        }
    }, IntSerializer.INSTANCE, IntSerializer.INSTANCE, 3000, 3000);
    SourceFunction<Integer> sourceFunction = new SourceFunction<Integer>() {

        private static final long serialVersionUID = 8297735565464653028L;

        @Override
        public void run(SourceContext<Integer> ctx) throws Exception {
        }

        @Override
        public void cancel() {
        }
    };
    SourceTransformation<Integer> source = new SourceTransformation<>("", new StreamSource<>(sourceFunction), BasicTypeInfo.INT_TYPE_INFO, 1);
    transformations.add(new OneInputTransformation<>(source, "test", windowOperator, BasicTypeInfo.INT_TYPE_INFO, 1));
    StreamGraph streamGraph = StreamGraphGenerator.generate(env, transformations, 1);
    List<Integer> result = new ArrayList<>();
    List<Integer> input = new ArrayList<>();
    List<Integer> expected = new ArrayList<>();
    input.add(1);
    input.add(2);
    input.add(3);
    for (int value : input) {
        initValue += value;
    }
    expected.add(initValue);
    foldWindowFunction.process(0, foldWindowFunction.new Context() {

        @Override
        public TimeWindow window() {
            return new TimeWindow(0, 1);
        }
    }, input, new ListCollector<>(result));
    Assert.assertEquals(expected, result);
}