Search in sources :

Example 46 with TableException

use of org.apache.flink.table.api.TableException in project flink by apache.

the class StreamPhysicalPythonOverAggregateRule method matches.

@Override
public boolean matches(RelOptRuleCall call) {
    FlinkLogicalOverAggregate logicWindow = call.rel(0);
    List<AggregateCall> aggCalls = logicWindow.groups.get(0).getAggregateCalls(logicWindow);
    boolean existGeneralPythonFunction = aggCalls.stream().anyMatch(x -> PythonUtil.isPythonAggregate(x, PythonFunctionKind.GENERAL));
    boolean existPandasFunction = aggCalls.stream().anyMatch(x -> PythonUtil.isPythonAggregate(x, PythonFunctionKind.PANDAS));
    boolean existJavaFunction = aggCalls.stream().anyMatch(x -> !PythonUtil.isPythonAggregate(x, null));
    if (existPandasFunction || existGeneralPythonFunction) {
        if (existGeneralPythonFunction) {
            throw new TableException("Non-Pandas Python UDAFs are not supported in stream mode currently.");
        }
        if (existJavaFunction) {
            throw new TableException("Python UDAF and Java/Scala UDAF cannot be used together.");
        }
        return true;
    } else {
        return false;
    }
}
Also used : AggregateCall(org.apache.calcite.rel.core.AggregateCall) TableException(org.apache.flink.table.api.TableException) FlinkLogicalOverAggregate(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalOverAggregate)

Example 47 with TableException

use of org.apache.flink.table.api.TableException in project flink by apache.

the class StreamPhysicalPythonGroupWindowAggregateRule method matches.

@Override
public boolean matches(RelOptRuleCall call) {
    FlinkLogicalWindowAggregate agg = call.rel(0);
    List<AggregateCall> aggCalls = agg.getAggCallList();
    // check if we have grouping sets
    if (agg.getGroupType() != Aggregate.Group.SIMPLE || agg.indicator) {
        throw new TableException("GROUPING SETS are currently not supported.");
    }
    boolean existGeneralPythonFunction = aggCalls.stream().anyMatch(x -> PythonUtil.isPythonAggregate(x, PythonFunctionKind.GENERAL));
    boolean existPandasFunction = aggCalls.stream().anyMatch(x -> PythonUtil.isPythonAggregate(x, PythonFunctionKind.PANDAS));
    boolean existJavaFunction = aggCalls.stream().anyMatch(x -> !PythonUtil.isPythonAggregate(x, null) && !PythonUtil.isBuiltInAggregate(x));
    if (existPandasFunction && existGeneralPythonFunction) {
        throw new TableException("Pandas UDAFs and General Python UDAFs are not supported in used together currently.");
    }
    if (existPandasFunction || existGeneralPythonFunction) {
        if (existJavaFunction) {
            throw new TableException("Python UDAF and Java/Scala UDAF cannot be used together.");
        }
        return true;
    } else {
        return false;
    }
}
Also used : AggregateCall(org.apache.calcite.rel.core.AggregateCall) TableException(org.apache.flink.table.api.TableException) FlinkLogicalWindowAggregate(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalWindowAggregate)

Example 48 with TableException

use of org.apache.flink.table.api.TableException in project flink by apache.

the class StreamPhysicalPythonGroupWindowAggregateRule method convert.

@Override
public RelNode convert(RelNode rel) {
    FlinkLogicalWindowAggregate agg = (FlinkLogicalWindowAggregate) rel;
    LogicalWindow window = agg.getWindow();
    List<AggregateCall> aggCalls = agg.getAggCallList();
    boolean isPandasPythonUDAF = aggCalls.stream().anyMatch(x -> PythonUtil.isPythonAggregate(x, PythonFunctionKind.PANDAS));
    if (isPandasPythonUDAF && window instanceof SessionGroupWindow) {
        throw new TableException("Session Group Window is currently not supported for Pandas UDAF.");
    }
    RelNode input = agg.getInput();
    RelOptCluster cluster = rel.getCluster();
    FlinkRelDistribution requiredDistribution;
    if (agg.getGroupCount() != 0) {
        requiredDistribution = FlinkRelDistribution.hash(agg.getGroupSet().asList(), true);
    } else {
        requiredDistribution = FlinkRelDistribution.SINGLETON();
    }
    RelTraitSet requiredTraitSet = input.getTraitSet().replace(FlinkConventions.STREAM_PHYSICAL()).replace(requiredDistribution);
    RelTraitSet providedTraitSet = rel.getTraitSet().replace(FlinkConventions.STREAM_PHYSICAL());
    RelNode newInput = RelOptRule.convert(input, requiredTraitSet);
    ReadableConfig config = ShortcutUtils.unwrapTableConfig(rel);
    WindowEmitStrategy emitStrategy = WindowEmitStrategy.apply(config, agg.getWindow());
    if (emitStrategy.produceUpdates()) {
        throw new TableException("Python Group Window Aggregate Function is currently not supported for early fired or lately fired.");
    }
    return new StreamPhysicalPythonGroupWindowAggregate(cluster, providedTraitSet, newInput, rel.getRowType(), agg.getGroupSet().toArray(), JavaScalaConversionUtil.toScala(aggCalls), agg.getWindow(), agg.getNamedProperties(), emitStrategy);
}
Also used : RelOptCluster(org.apache.calcite.plan.RelOptCluster) StreamPhysicalPythonGroupWindowAggregate(org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalPythonGroupWindowAggregate) TableException(org.apache.flink.table.api.TableException) RelTraitSet(org.apache.calcite.plan.RelTraitSet) AggregateCall(org.apache.calcite.rel.core.AggregateCall) FlinkRelDistribution(org.apache.flink.table.planner.plan.trait.FlinkRelDistribution) ReadableConfig(org.apache.flink.configuration.ReadableConfig) LogicalWindow(org.apache.flink.table.planner.plan.logical.LogicalWindow) RelNode(org.apache.calcite.rel.RelNode) WindowEmitStrategy(org.apache.flink.table.planner.plan.utils.WindowEmitStrategy) FlinkLogicalWindowAggregate(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalWindowAggregate) SessionGroupWindow(org.apache.flink.table.planner.plan.logical.SessionGroupWindow)

Example 49 with TableException

use of org.apache.flink.table.api.TableException in project flink by apache.

the class BatchPhysicalPythonAggregateRule method matches.

@Override
public boolean matches(RelOptRuleCall call) {
    FlinkLogicalAggregate agg = call.rel(0);
    List<AggregateCall> aggCalls = agg.getAggCallList();
    boolean existGeneralPythonFunction = aggCalls.stream().anyMatch(x -> PythonUtil.isPythonAggregate(x, PythonFunctionKind.GENERAL));
    boolean existPandasFunction = aggCalls.stream().anyMatch(x -> PythonUtil.isPythonAggregate(x, PythonFunctionKind.PANDAS));
    boolean existJavaFunction = aggCalls.stream().anyMatch(x -> !PythonUtil.isPythonAggregate(x, null));
    if (existPandasFunction || existGeneralPythonFunction) {
        if (existGeneralPythonFunction) {
            throw new TableException("non-Pandas UDAFs are not supported in batch mode currently.");
        }
        if (existJavaFunction) {
            throw new TableException("Python UDAF and Java/Scala UDAF cannot be used together.");
        }
        return true;
    } else {
        return false;
    }
}
Also used : AggregateCall(org.apache.calcite.rel.core.AggregateCall) TableException(org.apache.flink.table.api.TableException) FlinkLogicalAggregate(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalAggregate)

Example 50 with TableException

use of org.apache.flink.table.api.TableException in project flink by apache.

the class BatchPhysicalPythonWindowAggregateRule method matches.

@Override
public boolean matches(RelOptRuleCall call) {
    FlinkLogicalWindowAggregate agg = call.rel(0);
    List<AggregateCall> aggCalls = agg.getAggCallList();
    boolean existGeneralPythonFunction = aggCalls.stream().anyMatch(x -> PythonUtil.isPythonAggregate(x, PythonFunctionKind.GENERAL));
    boolean existPandasFunction = aggCalls.stream().anyMatch(x -> PythonUtil.isPythonAggregate(x, PythonFunctionKind.PANDAS));
    boolean existJavaFunction = aggCalls.stream().anyMatch(x -> !PythonUtil.isPythonAggregate(x, null));
    if (existPandasFunction || existGeneralPythonFunction) {
        if (existGeneralPythonFunction) {
            throw new TableException("non-Pandas UDAFs are not supported in batch mode currently.");
        }
        if (existJavaFunction) {
            throw new TableException("Python UDAF and Java/Scala UDAF cannot be used together.");
        }
        return true;
    } else {
        return false;
    }
}
Also used : AggregateCall(org.apache.calcite.rel.core.AggregateCall) TableException(org.apache.flink.table.api.TableException) FlinkLogicalWindowAggregate(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalWindowAggregate)

Aggregations

TableException (org.apache.flink.table.api.TableException)163 RowData (org.apache.flink.table.data.RowData)35 RowType (org.apache.flink.table.types.logical.RowType)35 Transformation (org.apache.flink.api.dag.Transformation)28 ArrayList (java.util.ArrayList)27 ExecEdge (org.apache.flink.table.planner.plan.nodes.exec.ExecEdge)24 LogicalType (org.apache.flink.table.types.logical.LogicalType)24 List (java.util.List)22 DataType (org.apache.flink.table.types.DataType)19 OneInputTransformation (org.apache.flink.streaming.api.transformations.OneInputTransformation)18 ValidationException (org.apache.flink.table.api.ValidationException)17 IOException (java.io.IOException)13 AggregateCall (org.apache.calcite.rel.core.AggregateCall)13 ValueLiteralExpression (org.apache.flink.table.expressions.ValueLiteralExpression)13 RowDataKeySelector (org.apache.flink.table.runtime.keyselector.RowDataKeySelector)13 Optional (java.util.Optional)11 Configuration (org.apache.flink.configuration.Configuration)11 StreamExecutionEnvironment (org.apache.flink.streaming.api.environment.StreamExecutionEnvironment)11 Constructor (java.lang.reflect.Constructor)10 Arrays (java.util.Arrays)9