Examples with PartitionTransformation org.apache.flink.streaming.api.transformations.PartitionTransformation used on opensource projects

Example 1 with PartitionTransformation

use of org.apache.flink.streaming.api.transformations.PartitionTransformation in project flink by apache.

the class PythonConfigUtil method configForwardPartitioner.

private static void configForwardPartitioner(Transformation<?> upTransformation, Transformation<?> transformation) throws IllegalAccessException, NoSuchFieldException {
    // set ForwardPartitioner
    PartitionTransformation<?> partitionTransform = new PartitionTransformation<>(upTransformation, new ForwardPartitioner<>());
    Field inputTransformationField = transformation.getClass().getDeclaredField("input");
    inputTransformationField.setAccessible(true);
    inputTransformationField.set(transformation, partitionTransform);
}

Example 2 with PartitionTransformation

use of org.apache.flink.streaming.api.transformations.PartitionTransformation in project flink by apache.

the class StreamExecExchange method translateToPlanInternal.

@SuppressWarnings("unchecked")
@Override
protected Transformation<RowData> translateToPlanInternal(PlannerBase planner, ExecNodeConfig config) {
    final Transformation<RowData> inputTransform = (Transformation<RowData>) getInputEdges().get(0).translateToPlan(planner);
    final StreamPartitioner<RowData> partitioner;
    final int parallelism;
    final InputProperty inputProperty = getInputProperties().get(0);
    final InputProperty.DistributionType distributionType = inputProperty.getRequiredDistribution().getType();
    switch(distributionType) {
        case SINGLETON:
            partitioner = new GlobalPartitioner<>();
            parallelism = 1;
            break;
        case HASH:
            // TODO Eliminate duplicate keys
            int[] keys = ((HashDistribution) inputProperty.getRequiredDistribution()).getKeys();
            InternalTypeInfo<RowData> inputType = (InternalTypeInfo<RowData>) inputTransform.getOutputType();
            RowDataKeySelector keySelector = KeySelectorUtil.getRowDataSelector(keys, inputType);
            partitioner = new KeyGroupStreamPartitioner<>(keySelector, DEFAULT_LOWER_BOUND_MAX_PARALLELISM);
            parallelism = ExecutionConfig.PARALLELISM_DEFAULT;
            break;
        default:
            throw new TableException(String.format("%s is not supported now!", distributionType));
    }
    final Transformation<RowData> transformation = new PartitionTransformation<>(inputTransform, partitioner);
    createTransformationMeta(EXCHANGE_TRANSFORMATION, config).fill(transformation);
    transformation.setParallelism(parallelism);
    transformation.setOutputType(InternalTypeInfo.of(getOutputType()));
    return transformation;
}

Example 3 with PartitionTransformation

use of org.apache.flink.streaming.api.transformations.PartitionTransformation in project flink by apache.

the class StreamingJobGraphGeneratorTest method testExchangeModeUndefined.

/**
 * Test setting exchange mode to {@link StreamExchangeMode#UNDEFINED}.
 */
@Test
public void testExchangeModeUndefined() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    // fromElements -> Map -> Print
    DataStream<Integer> sourceDataStream = env.fromElements(1, 2, 3);
    DataStream<Integer> partitionAfterSourceDataStream = new DataStream<>(env, new PartitionTransformation<>(sourceDataStream.getTransformation(), new ForwardPartitioner<>(), StreamExchangeMode.UNDEFINED));
    DataStream<Integer> mapDataStream = partitionAfterSourceDataStream.map(value -> value).setParallelism(1);
    DataStream<Integer> partitionAfterMapDataStream = new DataStream<>(env, new PartitionTransformation<>(mapDataStream.getTransformation(), new RescalePartitioner<>(), StreamExchangeMode.UNDEFINED));
    partitionAfterMapDataStream.print().setParallelism(2);
    JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
    List<JobVertex> verticesSorted = jobGraph.getVerticesSortedTopologicallyFromSources();
    assertEquals(2, verticesSorted.size());
    // it can be chained with UNDEFINED exchange mode
    JobVertex sourceAndMapVertex = verticesSorted.get(0);
    // UNDEFINED exchange mode is translated into PIPELINED_BOUNDED result partition by default
    assertEquals(ResultPartitionType.PIPELINED_BOUNDED, sourceAndMapVertex.getProducedDataSets().get(0).getResultType());
}

Example 4 with PartitionTransformation

use of org.apache.flink.streaming.api.transformations.PartitionTransformation in project flink by apache.

the class StreamingJobGraphGeneratorTest method testExchangeModePipelined.

/**
 * Test setting exchange mode to {@link StreamExchangeMode#PIPELINED}.
 */
@Test
public void testExchangeModePipelined() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    // fromElements -> Map -> Print
    DataStream<Integer> sourceDataStream = env.fromElements(1, 2, 3);
    DataStream<Integer> partitionAfterSourceDataStream = new DataStream<>(env, new PartitionTransformation<>(sourceDataStream.getTransformation(), new ForwardPartitioner<>(), StreamExchangeMode.PIPELINED));
    DataStream<Integer> mapDataStream = partitionAfterSourceDataStream.map(value -> value).setParallelism(1);
    DataStream<Integer> partitionAfterMapDataStream = new DataStream<>(env, new PartitionTransformation<>(mapDataStream.getTransformation(), new RescalePartitioner<>(), StreamExchangeMode.PIPELINED));
    partitionAfterMapDataStream.print().setParallelism(2);
    JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
    List<JobVertex> verticesSorted = jobGraph.getVerticesSortedTopologicallyFromSources();
    assertEquals(2, verticesSorted.size());
    // it can be chained with PIPELINED exchange mode
    JobVertex sourceAndMapVertex = verticesSorted.get(0);
    // PIPELINED exchange mode is translated into PIPELINED_BOUNDED result partition
    assertEquals(ResultPartitionType.PIPELINED_BOUNDED, sourceAndMapVertex.getProducedDataSets().get(0).getResultType());
}

Example 5 with PartitionTransformation

use of org.apache.flink.streaming.api.transformations.PartitionTransformation in project flink by apache.

the class CommonExecSink method applyKeyBy.

/**
 * Apply a primary key partition transformation to guarantee the strict ordering of changelog
 * messages.
 */
private Transformation<RowData> applyKeyBy(ReadableConfig config, Transformation<RowData> inputTransform, int[] primaryKeys, int sinkParallelism, int inputParallelism, boolean inputInsertOnly, boolean needMaterialize) {
    final ExecutionConfigOptions.SinkKeyedShuffle sinkShuffleByPk = config.get(ExecutionConfigOptions.TABLE_EXEC_SINK_KEYED_SHUFFLE);
    boolean sinkKeyBy = false;
    switch(sinkShuffleByPk) {
        case NONE:
            break;
        case AUTO:
            sinkKeyBy = inputInsertOnly && sinkParallelism != inputParallelism;
            break;
        case FORCE:
            // single parallelism has no problem
            sinkKeyBy = sinkParallelism != 1 || inputParallelism != 1;
            break;
    }
    if (!sinkKeyBy && !needMaterialize) {
        return inputTransform;
    }
    final RowDataKeySelector selector = KeySelectorUtil.getRowDataSelector(primaryKeys, getInputTypeInfo());
    final KeyGroupStreamPartitioner<RowData, RowData> partitioner = new KeyGroupStreamPartitioner<>(selector, KeyGroupRangeAssignment.DEFAULT_LOWER_BOUND_MAX_PARALLELISM);
    Transformation<RowData> partitionedTransform = new PartitionTransformation<>(inputTransform, partitioner);
    createTransformationMeta(PARTITIONER_TRANSFORMATION, "Partitioner", "Partitioner", config).fill(partitionedTransform);
    partitionedTransform.setParallelism(sinkParallelism);
    return partitionedTransform;
}