Examples with StreamGraph org.apache.flink.streaming.api.graph.StreamGraph used on opensource projects

Example 1 with StreamGraph

use of org.apache.flink.streaming.api.graph.StreamGraph in project flink by apache.

the class IterateITCase method testmultipleHeadsTailsWithTailPartitioning.

@Test
public void testmultipleHeadsTailsWithTailPartitioning() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<Integer> source1 = env.fromElements(1, 2, 3, 4, 5).shuffle().map(NoOpIntMap);
    DataStream<Integer> source2 = env.fromElements(1, 2, 3, 4, 5).map(NoOpIntMap);
    IterativeStream<Integer> iter1 = source1.union(source2).iterate();
    DataStream<Integer> head1 = iter1.map(NoOpIntMap).name("map1");
    DataStream<Integer> head2 = iter1.map(NoOpIntMap).setParallelism(DEFAULT_PARALLELISM / 2).name("shuffle").rebalance();
    DataStreamSink<Integer> head3 = iter1.map(NoOpIntMap).setParallelism(DEFAULT_PARALLELISM / 2).addSink(new ReceiveCheckNoOpSink<Integer>());
    DataStreamSink<Integer> head4 = iter1.map(NoOpIntMap).addSink(new ReceiveCheckNoOpSink<Integer>());
    SplitStream<Integer> source3 = env.fromElements(1, 2, 3, 4, 5).map(NoOpIntMap).name("split").split(new EvenOddOutputSelector());
    iter1.closeWith(source3.select("even").union(head1.map(NoOpIntMap).name("bc").broadcast(), head2.map(NoOpIntMap).shuffle()));
    StreamGraph graph = env.getStreamGraph();
    JobGraph jg = graph.getJobGraph();
    assertEquals(1, graph.getIterationSourceSinkPairs().size());
    Tuple2<StreamNode, StreamNode> sourceSinkPair = graph.getIterationSourceSinkPairs().iterator().next();
    StreamNode itSource = sourceSinkPair.f0;
    StreamNode itSink = sourceSinkPair.f1;
    assertEquals(4, itSource.getOutEdges().size());
    assertEquals(3, itSink.getInEdges().size());
    assertEquals(itSource.getParallelism(), itSink.getParallelism());
    for (StreamEdge edge : itSource.getOutEdges()) {
        if (edge.getTargetVertex().getOperatorName().equals("map1")) {
            assertTrue(edge.getPartitioner() instanceof ForwardPartitioner);
            assertEquals(4, edge.getTargetVertex().getParallelism());
        } else if (edge.getTargetVertex().getOperatorName().equals("shuffle")) {
            assertTrue(edge.getPartitioner() instanceof RebalancePartitioner);
            assertEquals(2, edge.getTargetVertex().getParallelism());
        }
    }
    for (StreamEdge edge : itSink.getInEdges()) {
        String tailName = edge.getSourceVertex().getOperatorName();
        if (tailName.equals("split")) {
            assertTrue(edge.getPartitioner() instanceof ForwardPartitioner);
            assertTrue(edge.getSelectedNames().contains("even"));
        } else if (tailName.equals("bc")) {
            assertTrue(edge.getPartitioner() instanceof BroadcastPartitioner);
        } else if (tailName.equals("shuffle")) {
            assertTrue(edge.getPartitioner() instanceof ShufflePartitioner);
        }
    }
    // Test co-location
    JobVertex itSource1 = null;
    JobVertex itSink1 = null;
    for (JobVertex vertex : jg.getVertices()) {
        if (vertex.getName().contains("IterationSource")) {
            itSource1 = vertex;
        } else if (vertex.getName().contains("IterationSink")) {
            itSink1 = vertex;
        }
    }
    assertTrue(itSource1.getCoLocationGroup() != null);
    assertTrue(itSink1.getCoLocationGroup() != null);
    assertEquals(itSource1.getCoLocationGroup(), itSink1.getCoLocationGroup());
}

Example 2 with StreamGraph

use of org.apache.flink.streaming.api.graph.StreamGraph in project flink by apache.

the class IterateITCase method testImmutabilityWithCoiteration.

@Test
public void testImmutabilityWithCoiteration() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    // for rebalance
    DataStream<Integer> source = env.fromElements(1, 10).map(NoOpIntMap);
    IterativeStream<Integer> iter1 = source.iterate();
    // Calling withFeedbackType should create a new iteration
    ConnectedIterativeStreams<Integer, String> iter2 = iter1.withFeedbackType(String.class);
    iter1.closeWith(iter1.map(NoOpIntMap)).print();
    iter2.closeWith(iter2.map(NoOpCoMap)).print();
    StreamGraph graph = env.getStreamGraph();
    assertEquals(2, graph.getIterationSourceSinkPairs().size());
    for (Tuple2<StreamNode, StreamNode> sourceSinkPair : graph.getIterationSourceSinkPairs()) {
        assertEquals(sourceSinkPair.f0.getOutEdges().get(0).getTargetVertex(), sourceSinkPair.f1.getInEdges().get(0).getSourceVertex());
    }
}

Example 3 with StreamGraph

use of org.apache.flink.streaming.api.graph.StreamGraph in project flink by apache.

the class RemoteStreamEnvironment method execute.

@Override
public JobExecutionResult execute(String jobName) throws ProgramInvocationException {
    StreamGraph streamGraph = getStreamGraph();
    streamGraph.setJobName(jobName);
    transformations.clear();
    return executeRemotely(streamGraph, jarFiles);
}

Example 4 with StreamGraph

use of org.apache.flink.streaming.api.graph.StreamGraph in project flink by apache.

the class StreamPlanEnvironment method execute.

@Override
public JobExecutionResult execute(String jobName) throws Exception {
    StreamGraph streamGraph = getStreamGraph();
    streamGraph.setJobName(jobName);
    transformations.clear();
    if (env instanceof OptimizerPlanEnvironment) {
        ((OptimizerPlanEnvironment) env).setPlan(streamGraph);
    } else if (env instanceof PreviewPlanEnvironment) {
        ((PreviewPlanEnvironment) env).setPreview(streamGraph.getStreamingPlanAsJSON());
    }
    throw new OptimizerPlanEnvironment.ProgramAbortException();
}

Example 5 with StreamGraph

use of org.apache.flink.streaming.api.graph.StreamGraph in project flink by apache.

the class FoldApplyProcessWindowFunctionTest method testFoldAllWindowFunctionOutputTypeConfigurable.

/**
	 * Tests that the FoldWindowFunction gets the output type serializer set by the
	 * StreamGraphGenerator and checks that the FoldWindowFunction computes the correct result.
	 */
@Test
public void testFoldAllWindowFunctionOutputTypeConfigurable() throws Exception {
    StreamExecutionEnvironment env = new DummyStreamExecutionEnvironment();
    List<StreamTransformation<?>> transformations = new ArrayList<>();
    int initValue = 1;
    FoldApplyProcessAllWindowFunction<TimeWindow, Integer, Integer, Integer> foldWindowFunction = new FoldApplyProcessAllWindowFunction<>(initValue, new FoldFunction<Integer, Integer>() {

        @Override
        public Integer fold(Integer accumulator, Integer value) throws Exception {
            return accumulator + value;
        }
    }, new ProcessAllWindowFunction<Integer, Integer, TimeWindow>() {

        @Override
        public void process(Context context, Iterable<Integer> input, Collector<Integer> out) throws Exception {
            for (Integer in : input) {
                out.collect(in);
            }
        }
    }, BasicTypeInfo.INT_TYPE_INFO);
    AccumulatingProcessingTimeWindowOperator<Byte, Integer, Integer> windowOperator = new AccumulatingProcessingTimeWindowOperator<>(new InternalIterableProcessAllWindowFunction<>(foldWindowFunction), new KeySelector<Integer, Byte>() {

        private static final long serialVersionUID = -7951310554369722809L;

        @Override
        public Byte getKey(Integer value) throws Exception {
            return 0;
        }
    }, ByteSerializer.INSTANCE, IntSerializer.INSTANCE, 3000, 3000);
    SourceFunction<Integer> sourceFunction = new SourceFunction<Integer>() {

        private static final long serialVersionUID = 8297735565464653028L;

        @Override
        public void run(SourceContext<Integer> ctx) throws Exception {
        }

        @Override
        public void cancel() {
        }
    };
    SourceTransformation<Integer> source = new SourceTransformation<>("", new StreamSource<>(sourceFunction), BasicTypeInfo.INT_TYPE_INFO, 1);
    transformations.add(new OneInputTransformation<>(source, "test", windowOperator, BasicTypeInfo.INT_TYPE_INFO, 1));
    StreamGraph streamGraph = StreamGraphGenerator.generate(env, transformations, 1);
    List<Integer> result = new ArrayList<>();
    List<Integer> input = new ArrayList<>();
    List<Integer> expected = new ArrayList<>();
    input.add(1);
    input.add(2);
    input.add(3);
    for (int value : input) {
        initValue += value;
    }
    expected.add(initValue);
    foldWindowFunction.process(foldWindowFunction.new Context() {

        @Override
        public TimeWindow window() {
            return new TimeWindow(0, 1);
        }
    }, input, new ListCollector<>(result));
    Assert.assertEquals(expected, result);
}