Examples with RescalePartitioner org.apache.flink.streaming.runtime.partitioner.RescalePartitioner used on opensource projects

Example 1 with RescalePartitioner

use of org.apache.flink.streaming.runtime.partitioner.RescalePartitioner in project flink by apache.

the class StreamingJobGraphGeneratorTest method testExchangeModeUndefined.

/**
 * Test setting exchange mode to {@link StreamExchangeMode#UNDEFINED}.
 */
@Test
public void testExchangeModeUndefined() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    // fromElements -> Map -> Print
    DataStream<Integer> sourceDataStream = env.fromElements(1, 2, 3);
    DataStream<Integer> partitionAfterSourceDataStream = new DataStream<>(env, new PartitionTransformation<>(sourceDataStream.getTransformation(), new ForwardPartitioner<>(), StreamExchangeMode.UNDEFINED));
    DataStream<Integer> mapDataStream = partitionAfterSourceDataStream.map(value -> value).setParallelism(1);
    DataStream<Integer> partitionAfterMapDataStream = new DataStream<>(env, new PartitionTransformation<>(mapDataStream.getTransformation(), new RescalePartitioner<>(), StreamExchangeMode.UNDEFINED));
    partitionAfterMapDataStream.print().setParallelism(2);
    JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
    List<JobVertex> verticesSorted = jobGraph.getVerticesSortedTopologicallyFromSources();
    assertEquals(2, verticesSorted.size());
    // it can be chained with UNDEFINED exchange mode
    JobVertex sourceAndMapVertex = verticesSorted.get(0);
    // UNDEFINED exchange mode is translated into PIPELINED_BOUNDED result partition by default
    assertEquals(ResultPartitionType.PIPELINED_BOUNDED, sourceAndMapVertex.getProducedDataSets().get(0).getResultType());
}

Example 2 with RescalePartitioner

use of org.apache.flink.streaming.runtime.partitioner.RescalePartitioner in project flink by apache.

the class StreamingJobGraphGeneratorTest method testExchangeModePipelined.

/**
 * Test setting exchange mode to {@link StreamExchangeMode#PIPELINED}.
 */
@Test
public void testExchangeModePipelined() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    // fromElements -> Map -> Print
    DataStream<Integer> sourceDataStream = env.fromElements(1, 2, 3);
    DataStream<Integer> partitionAfterSourceDataStream = new DataStream<>(env, new PartitionTransformation<>(sourceDataStream.getTransformation(), new ForwardPartitioner<>(), StreamExchangeMode.PIPELINED));
    DataStream<Integer> mapDataStream = partitionAfterSourceDataStream.map(value -> value).setParallelism(1);
    DataStream<Integer> partitionAfterMapDataStream = new DataStream<>(env, new PartitionTransformation<>(mapDataStream.getTransformation(), new RescalePartitioner<>(), StreamExchangeMode.PIPELINED));
    partitionAfterMapDataStream.print().setParallelism(2);
    JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
    List<JobVertex> verticesSorted = jobGraph.getVerticesSortedTopologicallyFromSources();
    assertEquals(2, verticesSorted.size());
    // it can be chained with PIPELINED exchange mode
    JobVertex sourceAndMapVertex = verticesSorted.get(0);
    // PIPELINED exchange mode is translated into PIPELINED_BOUNDED result partition
    assertEquals(ResultPartitionType.PIPELINED_BOUNDED, sourceAndMapVertex.getProducedDataSets().get(0).getResultType());
}

Example 3 with RescalePartitioner

use of org.apache.flink.streaming.runtime.partitioner.RescalePartitioner in project flink by apache.

the class StreamingJobGraphGeneratorTest method testExchangeModeBatch.

/**
 * Test setting exchange mode to {@link StreamExchangeMode#BATCH}.
 */
@Test
public void testExchangeModeBatch() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setRuntimeMode(RuntimeExecutionMode.BATCH);
    env.setBufferTimeout(-1);
    // fromElements -> Map -> Print
    DataStream<Integer> sourceDataStream = env.fromElements(1, 2, 3);
    DataStream<Integer> partitionAfterSourceDataStream = new DataStream<>(env, new PartitionTransformation<>(sourceDataStream.getTransformation(), new ForwardPartitioner<>(), StreamExchangeMode.BATCH));
    DataStream<Integer> mapDataStream = partitionAfterSourceDataStream.map(value -> value).setParallelism(1);
    DataStream<Integer> partitionAfterMapDataStream = new DataStream<>(env, new PartitionTransformation<>(mapDataStream.getTransformation(), new RescalePartitioner<>(), StreamExchangeMode.BATCH));
    partitionAfterMapDataStream.print().setParallelism(2);
    JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
    List<JobVertex> verticesSorted = jobGraph.getVerticesSortedTopologicallyFromSources();
    assertEquals(3, verticesSorted.size());
    // it can not be chained with BATCH exchange mode
    JobVertex sourceVertex = verticesSorted.get(0);
    JobVertex mapVertex = verticesSorted.get(1);
    // BATCH exchange mode is translated into BLOCKING result partition
    assertEquals(ResultPartitionType.BLOCKING, sourceVertex.getProducedDataSets().get(0).getResultType());
    assertEquals(ResultPartitionType.BLOCKING, mapVertex.getProducedDataSets().get(0).getResultType());
}

Example 4 with RescalePartitioner

use of org.apache.flink.streaming.runtime.partitioner.RescalePartitioner in project flink by apache.

the class StreamingJobGraphGeneratorWithGlobalStreamExchangeModeTest method createStreamGraph.

/**
 * Topology: source(parallelism=1) --(forward)--> map1(parallelism=1) --(rescale)-->
 * map2(parallelism=2) --(rebalance)--> sink(parallelism=2).
 */
private static StreamGraph createStreamGraph(GlobalStreamExchangeMode globalStreamExchangeMode) {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    if (globalStreamExchangeMode != GlobalStreamExchangeMode.ALL_EDGES_PIPELINED) {
        env.setBufferTimeout(-1);
    }
    final DataStream<Integer> source = env.fromElements(1, 2, 3).setParallelism(1);
    final DataStream<Integer> forward = new DataStream<>(env, new PartitionTransformation<>(source.getTransformation(), new ForwardPartitioner<>(), StreamExchangeMode.UNDEFINED));
    final DataStream<Integer> map1 = forward.map(i -> i).startNewChain().setParallelism(1);
    final DataStream<Integer> rescale = new DataStream<>(env, new PartitionTransformation<>(map1.getTransformation(), new RescalePartitioner<>(), StreamExchangeMode.UNDEFINED));
    final DataStream<Integer> map2 = rescale.map(i -> i).setParallelism(2);
    map2.rebalance().print().setParallelism(2);
    final StreamGraph streamGraph = env.getStreamGraph();
    streamGraph.setGlobalStreamExchangeMode(globalStreamExchangeMode);
    return streamGraph;
}