Search in sources :

Example 71 with Transformation

use of org.apache.flink.api.dag.Transformation in project flink by apache.

the class BatchExecExchange method translateToPlanInternal.

@SuppressWarnings("unchecked")
@Override
protected Transformation<RowData> translateToPlanInternal(PlannerBase planner, ExecNodeConfig config) {
    final ExecEdge inputEdge = getInputEdges().get(0);
    final Transformation<RowData> inputTransform = (Transformation<RowData>) inputEdge.translateToPlan(planner);
    final RowType inputType = (RowType) inputEdge.getOutputType();
    boolean requireUndefinedExchangeMode = false;
    final StreamPartitioner<RowData> partitioner;
    final int parallelism;
    final InputProperty inputProperty = getInputProperties().get(0);
    final RequiredDistribution requiredDistribution = inputProperty.getRequiredDistribution();
    final InputProperty.DistributionType distributionType = requiredDistribution.getType();
    switch(distributionType) {
        case ANY:
            partitioner = null;
            parallelism = ExecutionConfig.PARALLELISM_DEFAULT;
            break;
        case BROADCAST:
            partitioner = new BroadcastPartitioner<>();
            parallelism = ExecutionConfig.PARALLELISM_DEFAULT;
            break;
        case SINGLETON:
            partitioner = new GlobalPartitioner<>();
            parallelism = 1;
            break;
        case HASH:
            partitioner = createHashPartitioner(((HashDistribution) requiredDistribution), inputType, config);
            parallelism = ExecutionConfig.PARALLELISM_DEFAULT;
            break;
        case KEEP_INPUT_AS_IS:
            KeepInputAsIsDistribution keepInputAsIsDistribution = (KeepInputAsIsDistribution) requiredDistribution;
            if (keepInputAsIsDistribution.isStrict()) {
                // explicitly use ForwardPartitioner to guarantee the data distribution is
                // exactly the same as input
                partitioner = new ForwardPartitioner<>();
                requireUndefinedExchangeMode = true;
            } else {
                RequiredDistribution inputDistribution = ((KeepInputAsIsDistribution) requiredDistribution).getInputDistribution();
                checkArgument(inputDistribution instanceof HashDistribution, "Only HashDistribution is supported now");
                partitioner = new ForwardForConsecutiveHashPartitioner<>(createHashPartitioner(((HashDistribution) inputDistribution), inputType, config));
            }
            parallelism = inputTransform.getParallelism();
            break;
        default:
            throw new TableException(distributionType + "is not supported now!");
    }
    final StreamExchangeMode exchangeMode = requireUndefinedExchangeMode ? StreamExchangeMode.UNDEFINED : getBatchStreamExchangeMode(config, requiredExchangeMode);
    final Transformation<RowData> transformation = new PartitionTransformation<>(inputTransform, partitioner, exchangeMode);
    transformation.setParallelism(parallelism);
    transformation.setOutputType(InternalTypeInfo.of(getOutputType()));
    return transformation;
}
Also used : RequiredDistribution(org.apache.flink.table.planner.plan.nodes.exec.InputProperty.RequiredDistribution) PartitionTransformation(org.apache.flink.streaming.api.transformations.PartitionTransformation) Transformation(org.apache.flink.api.dag.Transformation) TableException(org.apache.flink.table.api.TableException) ExecEdge(org.apache.flink.table.planner.plan.nodes.exec.ExecEdge) InputProperty(org.apache.flink.table.planner.plan.nodes.exec.InputProperty) RowType(org.apache.flink.table.types.logical.RowType) PartitionTransformation(org.apache.flink.streaming.api.transformations.PartitionTransformation) HashDistribution(org.apache.flink.table.planner.plan.nodes.exec.InputProperty.HashDistribution) RowData(org.apache.flink.table.data.RowData) KeepInputAsIsDistribution(org.apache.flink.table.planner.plan.nodes.exec.InputProperty.KeepInputAsIsDistribution) StreamExchangeModeUtils.getBatchStreamExchangeMode(org.apache.flink.table.planner.utils.StreamExchangeModeUtils.getBatchStreamExchangeMode) StreamExchangeMode(org.apache.flink.streaming.api.transformations.StreamExchangeMode)

Example 72 with Transformation

use of org.apache.flink.api.dag.Transformation in project flink by apache.

the class BatchExecHashAggregate method translateToPlanInternal.

@SuppressWarnings("unchecked")
@Override
protected Transformation<RowData> translateToPlanInternal(PlannerBase planner, ExecNodeConfig config) {
    final ExecEdge inputEdge = getInputEdges().get(0);
    final Transformation<RowData> inputTransform = (Transformation<RowData>) inputEdge.translateToPlan(planner);
    final RowType inputRowType = (RowType) inputEdge.getOutputType();
    final RowType outputRowType = (RowType) getOutputType();
    final CodeGeneratorContext ctx = new CodeGeneratorContext(config.getTableConfig());
    final AggregateInfoList aggInfos = AggregateUtil.transformToBatchAggregateInfoList(aggInputRowType, JavaScalaConversionUtil.toScala(Arrays.asList(aggCalls)), // aggCallNeedRetractions
    null, // orderKeyIndexes
    null);
    final long managedMemory;
    final GeneratedOperator<OneInputStreamOperator<RowData, RowData>> generatedOperator;
    if (grouping.length == 0) {
        managedMemory = 0L;
        generatedOperator = AggWithoutKeysCodeGenerator.genWithoutKeys(ctx, planner.getRelBuilder(), aggInfos, inputRowType, outputRowType, isMerge, isFinal, "NoGrouping");
    } else {
        managedMemory = config.get(ExecutionConfigOptions.TABLE_EXEC_RESOURCE_HASH_AGG_MEMORY).getBytes();
        generatedOperator = new HashAggCodeGenerator(ctx, planner.getRelBuilder(), aggInfos, inputRowType, outputRowType, grouping, auxGrouping, isMerge, isFinal).genWithKeys();
    }
    return ExecNodeUtil.createOneInputTransformation(inputTransform, createTransformationName(config), createTransformationDescription(config), new CodeGenOperatorFactory<>(generatedOperator), InternalTypeInfo.of(outputRowType), inputTransform.getParallelism(), managedMemory);
}
Also used : RowData(org.apache.flink.table.data.RowData) Transformation(org.apache.flink.api.dag.Transformation) AggregateInfoList(org.apache.flink.table.planner.plan.utils.AggregateInfoList) ExecEdge(org.apache.flink.table.planner.plan.nodes.exec.ExecEdge) CodeGeneratorContext(org.apache.flink.table.planner.codegen.CodeGeneratorContext) OneInputStreamOperator(org.apache.flink.streaming.api.operators.OneInputStreamOperator) RowType(org.apache.flink.table.types.logical.RowType) HashAggCodeGenerator(org.apache.flink.table.planner.codegen.agg.batch.HashAggCodeGenerator)

Example 73 with Transformation

use of org.apache.flink.api.dag.Transformation in project flink by apache.

the class CommonExecWindowTableFunction method translateToPlanInternal.

@SuppressWarnings("unchecked")
@Override
protected Transformation<RowData> translateToPlanInternal(PlannerBase planner, ExecNodeConfig config) {
    final ExecEdge inputEdge = getInputEdges().get(0);
    final Transformation<RowData> inputTransform = (Transformation<RowData>) inputEdge.translateToPlan(planner);
    WindowAssigner<TimeWindow> windowAssigner = createWindowAssigner(windowingStrategy);
    final ZoneId shiftTimeZone = TimeWindowUtil.getShiftTimeZone(windowingStrategy.getTimeAttributeType(), config.getLocalTimeZone());
    WindowTableFunctionOperator windowTableFunctionOperator = new WindowTableFunctionOperator(windowAssigner, windowingStrategy.getTimeAttributeIndex(), shiftTimeZone);
    return ExecNodeUtil.createOneInputTransformation(inputTransform, createTransformationMeta(WINDOW_TRANSFORMATION, config), windowTableFunctionOperator, InternalTypeInfo.of(getOutputType()), inputTransform.getParallelism());
}
Also used : RowData(org.apache.flink.table.data.RowData) Transformation(org.apache.flink.api.dag.Transformation) ExecEdge(org.apache.flink.table.planner.plan.nodes.exec.ExecEdge) ZoneId(java.time.ZoneId) WindowTableFunctionOperator(org.apache.flink.table.runtime.operators.window.WindowTableFunctionOperator) TimeWindow(org.apache.flink.table.runtime.operators.window.TimeWindow)

Example 74 with Transformation

use of org.apache.flink.api.dag.Transformation in project flink by apache.

the class BatchExecSink method translateToPlanInternal.

@SuppressWarnings("unchecked")
@Override
protected Transformation<Object> translateToPlanInternal(PlannerBase planner, ExecNodeConfig config) {
    final Transformation<RowData> inputTransform = (Transformation<RowData>) getInputEdges().get(0).translateToPlan(planner);
    final DynamicTableSink tableSink = tableSinkSpec.getTableSink(planner.getFlinkContext());
    return createSinkTransformation(planner.getExecEnv(), config, inputTransform, tableSink, -1, false);
}
Also used : RowData(org.apache.flink.table.data.RowData) Transformation(org.apache.flink.api.dag.Transformation) DynamicTableSink(org.apache.flink.table.connector.sink.DynamicTableSink)

Example 75 with Transformation

use of org.apache.flink.api.dag.Transformation in project flink by apache.

the class BatchExecSortAggregate method translateToPlanInternal.

@SuppressWarnings("unchecked")
@Override
protected Transformation<RowData> translateToPlanInternal(PlannerBase planner, ExecNodeConfig config) {
    final ExecEdge inputEdge = getInputEdges().get(0);
    final Transformation<RowData> inputTransform = (Transformation<RowData>) inputEdge.translateToPlan(planner);
    final RowType inputRowType = (RowType) inputEdge.getOutputType();
    final RowType outputRowType = (RowType) getOutputType();
    final CodeGeneratorContext ctx = new CodeGeneratorContext(config.getTableConfig());
    final AggregateInfoList aggInfos = AggregateUtil.transformToBatchAggregateInfoList(aggInputRowType, JavaScalaConversionUtil.toScala(Arrays.asList(aggCalls)), null, null);
    final GeneratedOperator<OneInputStreamOperator<RowData, RowData>> generatedOperator;
    if (grouping.length == 0) {
        generatedOperator = AggWithoutKeysCodeGenerator.genWithoutKeys(ctx, planner.getRelBuilder(), aggInfos, inputRowType, outputRowType, isMerge, isFinal, "NoGrouping");
    } else {
        generatedOperator = SortAggCodeGenerator.genWithKeys(ctx, planner.getRelBuilder(), aggInfos, inputRowType, outputRowType, grouping, auxGrouping, isMerge, isFinal);
    }
    return ExecNodeUtil.createOneInputTransformation(inputTransform, createTransformationName(config), createTransformationDescription(config), new CodeGenOperatorFactory<>(generatedOperator), InternalTypeInfo.of(outputRowType), inputTransform.getParallelism());
}
Also used : RowData(org.apache.flink.table.data.RowData) Transformation(org.apache.flink.api.dag.Transformation) AggregateInfoList(org.apache.flink.table.planner.plan.utils.AggregateInfoList) ExecEdge(org.apache.flink.table.planner.plan.nodes.exec.ExecEdge) CodeGeneratorContext(org.apache.flink.table.planner.codegen.CodeGeneratorContext) OneInputStreamOperator(org.apache.flink.streaming.api.operators.OneInputStreamOperator) RowType(org.apache.flink.table.types.logical.RowType)

Aggregations

Transformation (org.apache.flink.api.dag.Transformation)98 RowData (org.apache.flink.table.data.RowData)69 ExecEdge (org.apache.flink.table.planner.plan.nodes.exec.ExecEdge)53 RowType (org.apache.flink.table.types.logical.RowType)50 OneInputTransformation (org.apache.flink.streaming.api.transformations.OneInputTransformation)45 TableException (org.apache.flink.table.api.TableException)28 RowDataKeySelector (org.apache.flink.table.runtime.keyselector.RowDataKeySelector)28 ArrayList (java.util.ArrayList)25 CodeGeneratorContext (org.apache.flink.table.planner.codegen.CodeGeneratorContext)21 Configuration (org.apache.flink.configuration.Configuration)19 TwoInputTransformation (org.apache.flink.streaming.api.transformations.TwoInputTransformation)18 List (java.util.List)17 PartitionTransformation (org.apache.flink.streaming.api.transformations.PartitionTransformation)17 AggregateInfoList (org.apache.flink.table.planner.plan.utils.AggregateInfoList)17 LogicalType (org.apache.flink.table.types.logical.LogicalType)16 Test (org.junit.Test)16 StreamExecutionEnvironment (org.apache.flink.streaming.api.environment.StreamExecutionEnvironment)13 SourceTransformation (org.apache.flink.streaming.api.transformations.SourceTransformation)13 Arrays (java.util.Arrays)11 Collections (java.util.Collections)10