Examples with ShufflePartitioner org.apache.flink.streaming.runtime.partitioner.ShufflePartitioner used on opensource projects

Example 1 with ShufflePartitioner

use of org.apache.flink.streaming.runtime.partitioner.ShufflePartitioner in project flink by apache.

the class StreamGraphGeneratorTest method testVirtualTransformations.

/**
 * This tests whether virtual Transformations behave correctly.
 *
 * <p>Verifies that partitioning, output selector, selected names are correctly set in the
 * StreamGraph when they are intermixed.
 */
@Test
public void testVirtualTransformations() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<Integer> source = env.fromElements(1, 10);
    DataStream<Integer> rebalanceMap = source.rebalance().map(new NoOpIntMap());
    // verify that only the partitioning that was set last is used
    DataStream<Integer> broadcastMap = rebalanceMap.forward().global().broadcast().map(new NoOpIntMap());
    broadcastMap.addSink(new DiscardingSink<>());
    DataStream<Integer> broadcastOperator = rebalanceMap.map(new NoOpIntMap()).name("broadcast");
    DataStream<Integer> map1 = broadcastOperator.broadcast();
    DataStream<Integer> globalOperator = rebalanceMap.map(new NoOpIntMap()).name("global");
    DataStream<Integer> map2 = globalOperator.global();
    DataStream<Integer> shuffleOperator = rebalanceMap.map(new NoOpIntMap()).name("shuffle");
    DataStream<Integer> map3 = shuffleOperator.shuffle();
    SingleOutputStreamOperator<Integer> unionedMap = map1.union(map2).union(map3).map(new NoOpIntMap()).name("union");
    unionedMap.addSink(new DiscardingSink<>());
    StreamGraph graph = env.getStreamGraph();
    // rebalanceMap
    assertTrue(graph.getStreamNode(rebalanceMap.getId()).getInEdges().get(0).getPartitioner() instanceof RebalancePartitioner);
    // verify that only last partitioning takes precedence
    assertTrue(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
    assertEquals(rebalanceMap.getId(), graph.getSourceVertex(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0)).getId());
    // verify that partitioning in unions is preserved
    assertTrue(graph.getStreamNode(broadcastOperator.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
    assertTrue(graph.getStreamNode(globalOperator.getId()).getOutEdges().get(0).getPartitioner() instanceof GlobalPartitioner);
    assertTrue(graph.getStreamNode(shuffleOperator.getId()).getOutEdges().get(0).getPartitioner() instanceof ShufflePartitioner);
}

Example 2 with ShufflePartitioner

use of org.apache.flink.streaming.runtime.partitioner.ShufflePartitioner in project flink by apache.

the class IterateITCase method testmultipleHeadsTailsWithTailPartitioning.

@Test
public void testmultipleHeadsTailsWithTailPartitioning() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<Integer> source1 = env.fromElements(1, 2, 3, 4, 5).shuffle().map(noOpIntMap);
    DataStream<Integer> source2 = env.fromElements(1, 2, 3, 4, 5).map(noOpIntMap);
    IterativeStream<Integer> iter1 = source1.union(source2).iterate();
    DataStream<Integer> head1 = iter1.map(noOpIntMap).name("map1");
    DataStream<Integer> head2 = iter1.map(noOpIntMap).setParallelism(parallelism / 2).name("shuffle").rebalance();
    DataStreamSink<Integer> head3 = iter1.map(noOpIntMap).setParallelism(parallelism / 2).addSink(new ReceiveCheckNoOpSink<Integer>());
    DataStreamSink<Integer> head4 = iter1.map(noOpIntMap).addSink(new ReceiveCheckNoOpSink<Integer>());
    OutputTag<Integer> even = new OutputTag<Integer>("even") {
    };
    OutputTag<Integer> odd = new OutputTag<Integer>("odd") {
    };
    SingleOutputStreamOperator<Object> source3 = env.fromElements(1, 2, 3, 4, 5).map(noOpIntMap).name("split").process(new ProcessFunction<Integer, Object>() {

        @Override
        public void processElement(Integer value, Context ctx, Collector<Object> out) throws Exception {
            if (value % 2 == 0) {
                ctx.output(even, value);
            } else {
                ctx.output(odd, value);
            }
        }
    });
    iter1.closeWith(source3.getSideOutput(even).union(head1.map(noOpIntMap).name("bc").broadcast(), head2.map(noOpIntMap).shuffle()));
    StreamGraph graph = env.getStreamGraph();
    JobGraph jg = graph.getJobGraph();
    assertEquals(1, graph.getIterationSourceSinkPairs().size());
    Tuple2<StreamNode, StreamNode> sourceSinkPair = graph.getIterationSourceSinkPairs().iterator().next();
    StreamNode itSource = sourceSinkPair.f0;
    StreamNode itSink = sourceSinkPair.f1;
    assertEquals(4, itSource.getOutEdges().size());
    assertEquals(3, itSink.getInEdges().size());
    assertEquals(itSource.getParallelism(), itSink.getParallelism());
    for (StreamEdge edge : itSource.getOutEdges()) {
        if (graph.getTargetVertex(edge).getOperatorName().equals("map1")) {
            assertTrue(edge.getPartitioner() instanceof ForwardPartitioner);
            assertEquals(4, graph.getTargetVertex(edge).getParallelism());
        } else if (graph.getTargetVertex(edge).getOperatorName().equals("shuffle")) {
            assertTrue(edge.getPartitioner() instanceof RebalancePartitioner);
            assertEquals(2, graph.getTargetVertex(edge).getParallelism());
        }
    }
    for (StreamEdge edge : itSink.getInEdges()) {
        String tailName = graph.getSourceVertex(edge).getOperatorName();
        if (tailName.equals("split")) {
            assertTrue(edge.getPartitioner() instanceof ForwardPartitioner);
        } else if (tailName.equals("bc")) {
            assertTrue(edge.getPartitioner() instanceof BroadcastPartitioner);
        } else if (tailName.equals("shuffle")) {
            assertTrue(edge.getPartitioner() instanceof ShufflePartitioner);
        }
    }
    // Test co-location
    JobVertex itSource1 = null;
    JobVertex itSink1 = null;
    for (JobVertex vertex : jg.getVertices()) {
        if (vertex.getName().contains("IterationSource")) {
            itSource1 = vertex;
        } else if (vertex.getName().contains("IterationSink")) {
            itSink1 = vertex;
        }
    }
    assertTrue(itSource1.getCoLocationGroup() != null);
    assertTrue(itSink1.getCoLocationGroup() != null);
    assertEquals(itSource1.getCoLocationGroup(), itSink1.getCoLocationGroup());
}

Example 3 with ShufflePartitioner

use of org.apache.flink.streaming.runtime.partitioner.ShufflePartitioner in project flink by apache.

the class DataStreamTest method testChannelSelectors.

@Test
public void testChannelSelectors() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStreamSource<Long> src = env.generateSequence(0, 0);
    DataStream<Long> broadcast = src.broadcast();
    DataStreamSink<Long> broadcastSink = broadcast.print();
    StreamPartitioner<?> broadcastPartitioner = getStreamGraph(env).getStreamEdges(src.getId(), broadcastSink.getTransformation().getId()).get(0).getPartitioner();
    assertTrue(broadcastPartitioner instanceof BroadcastPartitioner);
    DataStream<Long> shuffle = src.shuffle();
    DataStreamSink<Long> shuffleSink = shuffle.print();
    StreamPartitioner<?> shufflePartitioner = getStreamGraph(env).getStreamEdges(src.getId(), shuffleSink.getTransformation().getId()).get(0).getPartitioner();
    assertTrue(shufflePartitioner instanceof ShufflePartitioner);
    DataStream<Long> forward = src.forward();
    DataStreamSink<Long> forwardSink = forward.print();
    StreamPartitioner<?> forwardPartitioner = getStreamGraph(env).getStreamEdges(src.getId(), forwardSink.getTransformation().getId()).get(0).getPartitioner();
    assertTrue(forwardPartitioner instanceof ForwardPartitioner);
    DataStream<Long> rebalance = src.rebalance();
    DataStreamSink<Long> rebalanceSink = rebalance.print();
    StreamPartitioner<?> rebalancePartitioner = getStreamGraph(env).getStreamEdges(src.getId(), rebalanceSink.getTransformation().getId()).get(0).getPartitioner();
    assertTrue(rebalancePartitioner instanceof RebalancePartitioner);
    DataStream<Long> global = src.global();
    DataStreamSink<Long> globalSink = global.print();
    StreamPartitioner<?> globalPartitioner = getStreamGraph(env).getStreamEdges(src.getId(), globalSink.getTransformation().getId()).get(0).getPartitioner();
    assertTrue(globalPartitioner instanceof GlobalPartitioner);
}

Example 4 with ShufflePartitioner

use of org.apache.flink.streaming.runtime.partitioner.ShufflePartitioner in project flink by apache.

the class IterateITCase method testmultipleHeadsTailsSimple.

@Test
public void testmultipleHeadsTailsSimple() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<Integer> source1 = env.fromElements(1, 2, 3, 4, 5).shuffle().map(noOpIntMap).name("ParallelizeMapShuffle");
    DataStream<Integer> source2 = env.fromElements(1, 2, 3, 4, 5).map(noOpIntMap).name("ParallelizeMapRebalance");
    IterativeStream<Integer> iter1 = source1.union(source2).iterate();
    DataStream<Integer> head1 = iter1.map(noOpIntMap).name("IterRebalanceMap").setParallelism(parallelism / 2);
    DataStream<Integer> head2 = iter1.map(noOpIntMap).name("IterForwardMap");
    DataStreamSink<Integer> head3 = iter1.map(noOpIntMap).setParallelism(parallelism / 2).addSink(new ReceiveCheckNoOpSink<Integer>());
    DataStreamSink<Integer> head4 = iter1.map(noOpIntMap).addSink(new ReceiveCheckNoOpSink<Integer>());
    OutputTag<Integer> even = new OutputTag<Integer>("even") {
    };
    OutputTag<Integer> odd = new OutputTag<Integer>("odd") {
    };
    SingleOutputStreamOperator<Object> source3 = env.fromElements(1, 2, 3, 4, 5).map(noOpIntMap).name("EvenOddSourceMap").process(new ProcessFunction<Integer, Object>() {

        @Override
        public void processElement(Integer value, Context ctx, Collector<Object> out) throws Exception {
            if (value % 2 == 0) {
                ctx.output(even, value);
            } else {
                ctx.output(odd, value);
            }
        }
    });
    iter1.closeWith(source3.getSideOutput(even).union(head1.rebalance().map(noOpIntMap).broadcast(), head2.shuffle()));
    StreamGraph graph = env.getStreamGraph();
    JobGraph jg = graph.getJobGraph();
    assertEquals(1, graph.getIterationSourceSinkPairs().size());
    Tuple2<StreamNode, StreamNode> sourceSinkPair = graph.getIterationSourceSinkPairs().iterator().next();
    StreamNode itSource = sourceSinkPair.f0;
    StreamNode itSink = sourceSinkPair.f1;
    assertEquals(4, itSource.getOutEdges().size());
    assertEquals(3, itSink.getInEdges().size());
    assertEquals(itSource.getParallelism(), itSink.getParallelism());
    for (StreamEdge edge : itSource.getOutEdges()) {
        if (graph.getTargetVertex(edge).getOperatorName().equals("IterRebalanceMap")) {
            assertTrue(edge.getPartitioner() instanceof RebalancePartitioner);
        } else if (graph.getTargetVertex(edge).getOperatorName().equals("IterForwardMap")) {
            assertTrue(edge.getPartitioner() instanceof ForwardPartitioner);
        }
    }
    for (StreamEdge edge : itSink.getInEdges()) {
        if (graph.getStreamNode(edge.getSourceId()).getOperatorName().equals("ParallelizeMapShuffle")) {
            assertTrue(edge.getPartitioner() instanceof ShufflePartitioner);
        }
        if (graph.getStreamNode(edge.getSourceId()).getOperatorName().equals("ParallelizeMapForward")) {
            assertTrue(edge.getPartitioner() instanceof ForwardPartitioner);
        }
        if (graph.getStreamNode(edge.getSourceId()).getOperatorName().equals("EvenOddSourceMap")) {
            assertTrue(edge.getPartitioner() instanceof ForwardPartitioner);
        }
    }
    // Test co-location
    JobVertex itSource1 = null;
    JobVertex itSink1 = null;
    for (JobVertex vertex : jg.getVertices()) {
        if (vertex.getName().contains("IterationSource")) {
            itSource1 = vertex;
        } else if (vertex.getName().contains("IterationSink")) {
            itSink1 = vertex;
        }
    }
    assertTrue(itSource1.getCoLocationGroup() != null);
    assertEquals(itSource1.getCoLocationGroup(), itSink1.getCoLocationGroup());
}