Examples with PartitionTransformation org.apache.flink.streaming.api.transformations.PartitionTransformation used on opensource projects

Example 11 with PartitionTransformation

use of org.apache.flink.streaming.api.transformations.PartitionTransformation in project flink by apache.

the class StreamingJobGraphGeneratorWithGlobalStreamExchangeModeTest method createStreamGraph.

/**
 * Topology: source(parallelism=1) --(forward)--> map1(parallelism=1) --(rescale)-->
 * map2(parallelism=2) --(rebalance)--> sink(parallelism=2).
 */
private static StreamGraph createStreamGraph(GlobalStreamExchangeMode globalStreamExchangeMode) {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    if (globalStreamExchangeMode != GlobalStreamExchangeMode.ALL_EDGES_PIPELINED) {
        env.setBufferTimeout(-1);
    }
    final DataStream<Integer> source = env.fromElements(1, 2, 3).setParallelism(1);
    final DataStream<Integer> forward = new DataStream<>(env, new PartitionTransformation<>(source.getTransformation(), new ForwardPartitioner<>(), StreamExchangeMode.UNDEFINED));
    final DataStream<Integer> map1 = forward.map(i -> i).startNewChain().setParallelism(1);
    final DataStream<Integer> rescale = new DataStream<>(env, new PartitionTransformation<>(map1.getTransformation(), new RescalePartitioner<>(), StreamExchangeMode.UNDEFINED));
    final DataStream<Integer> map2 = rescale.map(i -> i).setParallelism(2);
    map2.rebalance().print().setParallelism(2);
    final StreamGraph streamGraph = env.getStreamGraph();
    streamGraph.setGlobalStreamExchangeMode(globalStreamExchangeMode);
    return streamGraph;
}

Example 12 with PartitionTransformation

use of org.apache.flink.streaming.api.transformations.PartitionTransformation in project flink by apache.

the class StreamGraphGeneratorTest method testResetBatchExchangeModeInStreamingExecution.

@Test
public void testResetBatchExchangeModeInStreamingExecution() {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<Integer> sourceDataStream = env.fromElements(1, 2, 3);
    PartitionTransformation<Integer> transformation = new PartitionTransformation<>(sourceDataStream.getTransformation(), new RebalancePartitioner<>(), StreamExchangeMode.BATCH);
    DataStream<Integer> partitionStream = new DataStream<>(env, transformation);
    partitionStream.map(value -> value).print();
    final StreamGraph streamGraph = env.getStreamGraph();
    Assertions.assertThat(streamGraph.getStreamEdges(1, 3)).hasSize(1).satisfies(e -> Assertions.assertThat(e.get(0).getExchangeMode()).isEqualTo(StreamExchangeMode.UNDEFINED));
}

Example 13 with PartitionTransformation

use of org.apache.flink.streaming.api.transformations.PartitionTransformation in project flink by apache.

the class BatchExecExchange method translateToPlanInternal.

@SuppressWarnings("unchecked")
@Override
protected Transformation<RowData> translateToPlanInternal(PlannerBase planner, ExecNodeConfig config) {
    final ExecEdge inputEdge = getInputEdges().get(0);
    final Transformation<RowData> inputTransform = (Transformation<RowData>) inputEdge.translateToPlan(planner);
    final RowType inputType = (RowType) inputEdge.getOutputType();
    boolean requireUndefinedExchangeMode = false;
    final StreamPartitioner<RowData> partitioner;
    final int parallelism;
    final InputProperty inputProperty = getInputProperties().get(0);
    final RequiredDistribution requiredDistribution = inputProperty.getRequiredDistribution();
    final InputProperty.DistributionType distributionType = requiredDistribution.getType();
    switch(distributionType) {
        case ANY:
            partitioner = null;
            parallelism = ExecutionConfig.PARALLELISM_DEFAULT;
            break;
        case BROADCAST:
            partitioner = new BroadcastPartitioner<>();
            parallelism = ExecutionConfig.PARALLELISM_DEFAULT;
            break;
        case SINGLETON:
            partitioner = new GlobalPartitioner<>();
            parallelism = 1;
            break;
        case HASH:
            partitioner = createHashPartitioner(((HashDistribution) requiredDistribution), inputType, config);
            parallelism = ExecutionConfig.PARALLELISM_DEFAULT;
            break;
        case KEEP_INPUT_AS_IS:
            KeepInputAsIsDistribution keepInputAsIsDistribution = (KeepInputAsIsDistribution) requiredDistribution;
            if (keepInputAsIsDistribution.isStrict()) {
                // explicitly use ForwardPartitioner to guarantee the data distribution is
                // exactly the same as input
                partitioner = new ForwardPartitioner<>();
                requireUndefinedExchangeMode = true;
            } else {
                RequiredDistribution inputDistribution = ((KeepInputAsIsDistribution) requiredDistribution).getInputDistribution();
                checkArgument(inputDistribution instanceof HashDistribution, "Only HashDistribution is supported now");
                partitioner = new ForwardForConsecutiveHashPartitioner<>(createHashPartitioner(((HashDistribution) inputDistribution), inputType, config));
            }
            parallelism = inputTransform.getParallelism();
            break;
        default:
            throw new TableException(distributionType + "is not supported now!");
    }
    final StreamExchangeMode exchangeMode = requireUndefinedExchangeMode ? StreamExchangeMode.UNDEFINED : getBatchStreamExchangeMode(config, requiredExchangeMode);
    final Transformation<RowData> transformation = new PartitionTransformation<>(inputTransform, partitioner, exchangeMode);
    transformation.setParallelism(parallelism);
    transformation.setOutputType(InternalTypeInfo.of(getOutputType()));
    return transformation;
}