Examples with BroadcastPartitioner org.apache.flink.streaming.runtime.partitioner.BroadcastPartitioner used on opensource projects

Example 6 with BroadcastPartitioner

use of org.apache.flink.streaming.runtime.partitioner.BroadcastPartitioner in project flink by apache.

the class StreamTaskTestHarness method setupOutputForSingletonOperatorChain.

/**
 * Users of the test harness can call this utility method to setup the stream config if there
 * will only be a single operator to be tested. The method will setup the outgoing network
 * connection for the operator.
 *
 * <p>For more advanced test cases such as testing chains of multiple operators with the
 * harness, please manually configure the stream config.
 */
public void setupOutputForSingletonOperatorChain() {
    Preconditions.checkState(!setupCalled, "This harness was already setup.");
    setupCalled = true;
    streamConfig.setChainStart();
    streamConfig.setTimeCharacteristic(TimeCharacteristic.EventTime);
    streamConfig.setNumberOfOutputs(1);
    streamConfig.setTypeSerializerOut(outputSerializer);
    streamConfig.setVertexID(0);
    streamConfig.setOperatorID(new OperatorID(4711L, 123L));
    StreamOperator<OUT> dummyOperator = new AbstractStreamOperator<OUT>() {

        private static final long serialVersionUID = 1L;
    };
    List<StreamEdge> outEdgesInOrder = new LinkedList<>();
    StreamNode sourceVertexDummy = new StreamNode(0, "group", null, dummyOperator, "source dummy", SourceStreamTask.class);
    StreamNode targetVertexDummy = new StreamNode(1, "group", null, dummyOperator, "target dummy", SourceStreamTask.class);
    outEdgesInOrder.add(new StreamEdge(sourceVertexDummy, targetVertexDummy, 0, new BroadcastPartitioner<>(), null));
    streamConfig.setOutEdgesInOrder(outEdgesInOrder);
    streamConfig.setNonChainedOutputs(outEdgesInOrder);
}

Example 7 with BroadcastPartitioner

use of org.apache.flink.streaming.runtime.partitioner.BroadcastPartitioner in project flink by apache.

the class DataStreamTest method testChannelSelectors.

@Test
public void testChannelSelectors() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStreamSource<Long> src = env.generateSequence(0, 0);
    DataStream<Long> broadcast = src.broadcast();
    DataStreamSink<Long> broadcastSink = broadcast.print();
    StreamPartitioner<?> broadcastPartitioner = getStreamGraph(env).getStreamEdges(src.getId(), broadcastSink.getTransformation().getId()).get(0).getPartitioner();
    assertTrue(broadcastPartitioner instanceof BroadcastPartitioner);
    DataStream<Long> shuffle = src.shuffle();
    DataStreamSink<Long> shuffleSink = shuffle.print();
    StreamPartitioner<?> shufflePartitioner = getStreamGraph(env).getStreamEdges(src.getId(), shuffleSink.getTransformation().getId()).get(0).getPartitioner();
    assertTrue(shufflePartitioner instanceof ShufflePartitioner);
    DataStream<Long> forward = src.forward();
    DataStreamSink<Long> forwardSink = forward.print();
    StreamPartitioner<?> forwardPartitioner = getStreamGraph(env).getStreamEdges(src.getId(), forwardSink.getTransformation().getId()).get(0).getPartitioner();
    assertTrue(forwardPartitioner instanceof ForwardPartitioner);
    DataStream<Long> rebalance = src.rebalance();
    DataStreamSink<Long> rebalanceSink = rebalance.print();
    StreamPartitioner<?> rebalancePartitioner = getStreamGraph(env).getStreamEdges(src.getId(), rebalanceSink.getTransformation().getId()).get(0).getPartitioner();
    assertTrue(rebalancePartitioner instanceof RebalancePartitioner);
    DataStream<Long> global = src.global();
    DataStreamSink<Long> globalSink = global.print();
    StreamPartitioner<?> globalPartitioner = getStreamGraph(env).getStreamEdges(src.getId(), globalSink.getTransformation().getId()).get(0).getPartitioner();
    assertTrue(globalPartitioner instanceof GlobalPartitioner);
}

Example 8 with BroadcastPartitioner

use of org.apache.flink.streaming.runtime.partitioner.BroadcastPartitioner in project flink by apache.

the class DataStreamTest method testUnion.

/**
 * Tests union functionality. This ensures that self-unions and unions of streams with differing
 * parallelism work.
 *
 * @throws Exception
 */
@Test
public void testUnion() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(4);
    DataStream<Long> input1 = env.generateSequence(0, 0).map(new MapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            return null;
        }
    });
    DataStream<Long> selfUnion = input1.union(input1).map(new MapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            return null;
        }
    });
    DataStream<Long> input6 = env.generateSequence(0, 0).map(new MapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            return null;
        }
    });
    DataStream<Long> selfUnionDifferentPartition = input6.broadcast().union(input6).map(new MapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            return null;
        }
    });
    DataStream<Long> input2 = env.generateSequence(0, 0).map(new MapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            return null;
        }
    }).setParallelism(4);
    DataStream<Long> input3 = env.generateSequence(0, 0).map(new MapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            return null;
        }
    }).setParallelism(2);
    DataStream<Long> unionDifferingParallelism = input2.union(input3).map(new MapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            return null;
        }
    }).setParallelism(4);
    DataStream<Long> input4 = env.generateSequence(0, 0).map(new MapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            return null;
        }
    }).setParallelism(2);
    DataStream<Long> input5 = env.generateSequence(0, 0).map(new MapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            return null;
        }
    }).setParallelism(4);
    DataStream<Long> unionDifferingPartitioning = input4.broadcast().union(input5).map(new MapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            return null;
        }
    }).setParallelism(4);
    StreamGraph streamGraph = getStreamGraph(env);
    // verify self union
    assertTrue(streamGraph.getStreamNode(selfUnion.getId()).getInEdges().size() == 2);
    for (StreamEdge edge : streamGraph.getStreamNode(selfUnion.getId()).getInEdges()) {
        assertTrue(edge.getPartitioner() instanceof ForwardPartitioner);
    }
    // verify self union with different partitioners
    assertTrue(streamGraph.getStreamNode(selfUnionDifferentPartition.getId()).getInEdges().size() == 2);
    boolean hasForward = false;
    boolean hasBroadcast = false;
    for (StreamEdge edge : streamGraph.getStreamNode(selfUnionDifferentPartition.getId()).getInEdges()) {
        if (edge.getPartitioner() instanceof ForwardPartitioner) {
            hasForward = true;
        }
        if (edge.getPartitioner() instanceof BroadcastPartitioner) {
            hasBroadcast = true;
        }
    }
    assertTrue(hasForward && hasBroadcast);
    // verify union of streams with differing parallelism
    assertTrue(streamGraph.getStreamNode(unionDifferingParallelism.getId()).getInEdges().size() == 2);
    for (StreamEdge edge : streamGraph.getStreamNode(unionDifferingParallelism.getId()).getInEdges()) {
        if (edge.getSourceId() == input2.getId()) {
            assertTrue(edge.getPartitioner() instanceof ForwardPartitioner);
        } else if (edge.getSourceId() == input3.getId()) {
            assertTrue(edge.getPartitioner() instanceof RebalancePartitioner);
        } else {
            fail("Wrong input edge.");
        }
    }
    // verify union of streams with differing partitionings
    assertTrue(streamGraph.getStreamNode(unionDifferingPartitioning.getId()).getInEdges().size() == 2);
    for (StreamEdge edge : streamGraph.getStreamNode(unionDifferingPartitioning.getId()).getInEdges()) {
        if (edge.getSourceId() == input4.getId()) {
            assertTrue(edge.getPartitioner() instanceof BroadcastPartitioner);
        } else if (edge.getSourceId() == input5.getId()) {
            assertTrue(edge.getPartitioner() instanceof ForwardPartitioner);
        } else {
            fail("Wrong input edge.");
        }
    }
}