Examples with GlobalPartitioner org.apache.flink.streaming.runtime.partitioner.GlobalPartitioner used on opensource projects

Example 1 with GlobalPartitioner

use of org.apache.flink.streaming.runtime.partitioner.GlobalPartitioner in project flink by apache.

the class StreamGraphGeneratorTest method testVirtualTransformations.

/**
	 * This tests whether virtual Transformations behave correctly.
	 *
	 * <p>
	 * Verifies that partitioning, output selector, selected names are correctly set in the
	 * StreamGraph when they are intermixed.
	 */
@Test
public void testVirtualTransformations() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<Integer> source = env.fromElements(1, 10);
    DataStream<Integer> rebalanceMap = source.rebalance().map(new NoOpIntMap());
    // verify that only the partitioning that was set last is used
    DataStream<Integer> broadcastMap = rebalanceMap.forward().global().broadcast().map(new NoOpIntMap());
    broadcastMap.addSink(new DiscardingSink<Integer>());
    // verify that partitioning is preserved across union and split/select
    EvenOddOutputSelector selector1 = new EvenOddOutputSelector();
    EvenOddOutputSelector selector2 = new EvenOddOutputSelector();
    EvenOddOutputSelector selector3 = new EvenOddOutputSelector();
    DataStream<Integer> map1Operator = rebalanceMap.map(new NoOpIntMap());
    DataStream<Integer> map1 = map1Operator.broadcast().split(selector1).select("even");
    DataStream<Integer> map2Operator = rebalanceMap.map(new NoOpIntMap());
    DataStream<Integer> map2 = map2Operator.split(selector2).select("odd").global();
    DataStream<Integer> map3Operator = rebalanceMap.map(new NoOpIntMap());
    DataStream<Integer> map3 = map3Operator.global().split(selector3).select("even").shuffle();
    SingleOutputStreamOperator<Integer> unionedMap = map1.union(map2).union(map3).map(new NoOpIntMap());
    unionedMap.addSink(new DiscardingSink<Integer>());
    StreamGraph graph = env.getStreamGraph();
    // rebalanceMap
    assertTrue(graph.getStreamNode(rebalanceMap.getId()).getInEdges().get(0).getPartitioner() instanceof RebalancePartitioner);
    // verify that only last partitioning takes precedence
    assertTrue(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
    assertEquals(rebalanceMap.getId(), graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0).getSourceVertex().getId());
    // verify that partitioning in unions is preserved and that it works across split/select
    assertTrue(graph.getStreamNode(map1Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
    assertTrue(graph.getStreamNode(map1Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("even"));
    assertTrue(graph.getStreamNode(map1Operator.getId()).getOutputSelectors().contains(selector1));
    assertTrue(graph.getStreamNode(map2Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof GlobalPartitioner);
    assertTrue(graph.getStreamNode(map2Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("odd"));
    assertTrue(graph.getStreamNode(map2Operator.getId()).getOutputSelectors().contains(selector2));
    assertTrue(graph.getStreamNode(map3Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof ShufflePartitioner);
    assertTrue(graph.getStreamNode(map3Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("even"));
    assertTrue(graph.getStreamNode(map3Operator.getId()).getOutputSelectors().contains(selector3));
}

Example 2 with GlobalPartitioner

use of org.apache.flink.streaming.runtime.partitioner.GlobalPartitioner in project flink by apache.

the class DataStreamTest method testChannelSelectors.

@Test
public void testChannelSelectors() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStreamSource<Long> src = env.generateSequence(0, 0);
    DataStream<Long> broadcast = src.broadcast();
    DataStreamSink<Long> broadcastSink = broadcast.print();
    StreamPartitioner<?> broadcastPartitioner = env.getStreamGraph().getStreamEdges(src.getId(), broadcastSink.getTransformation().getId()).get(0).getPartitioner();
    assertTrue(broadcastPartitioner instanceof BroadcastPartitioner);
    DataStream<Long> shuffle = src.shuffle();
    DataStreamSink<Long> shuffleSink = shuffle.print();
    StreamPartitioner<?> shufflePartitioner = env.getStreamGraph().getStreamEdges(src.getId(), shuffleSink.getTransformation().getId()).get(0).getPartitioner();
    assertTrue(shufflePartitioner instanceof ShufflePartitioner);
    DataStream<Long> forward = src.forward();
    DataStreamSink<Long> forwardSink = forward.print();
    StreamPartitioner<?> forwardPartitioner = env.getStreamGraph().getStreamEdges(src.getId(), forwardSink.getTransformation().getId()).get(0).getPartitioner();
    assertTrue(forwardPartitioner instanceof ForwardPartitioner);
    DataStream<Long> rebalance = src.rebalance();
    DataStreamSink<Long> rebalanceSink = rebalance.print();
    StreamPartitioner<?> rebalancePartitioner = env.getStreamGraph().getStreamEdges(src.getId(), rebalanceSink.getTransformation().getId()).get(0).getPartitioner();
    assertTrue(rebalancePartitioner instanceof RebalancePartitioner);
    DataStream<Long> global = src.global();
    DataStreamSink<Long> globalSink = global.print();
    StreamPartitioner<?> globalPartitioner = env.getStreamGraph().getStreamEdges(src.getId(), globalSink.getTransformation().getId()).get(0).getPartitioner();
    assertTrue(globalPartitioner instanceof GlobalPartitioner);
}