Examples with AggregateCall org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.AggregateCall used on opensource projects

Example 51 with AggregateCall

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.AggregateCall in project flink by apache.

the class BatchExecPythonOverAggregate method translateToPlanInternal.

@SuppressWarnings("unchecked")
@Override
protected Transformation<RowData> translateToPlanInternal(PlannerBase planner, ExecNodeConfig config) {
    final ExecEdge inputEdge = getInputEdges().get(0);
    final Transformation<RowData> inputTransform = (Transformation<RowData>) inputEdge.translateToPlan(planner);
    final RowType inputType = (RowType) inputEdge.getOutputType();
    List<OverSpec.GroupSpec> groups = overSpec.getGroups();
    boolean[] isRangeWindows = new boolean[groups.size()];
    for (int i = 0; i < groups.size(); i++) {
        OverSpec.GroupSpec group = groups.get(i);
        List<AggregateCall> groupAggCalls = group.getAggCalls();
        aggCalls.addAll(groupAggCalls);
        for (int j = 0; j < groupAggCalls.size(); j++) {
            aggWindowIndex.add(i);
        }
        OverWindowMode mode = inferGroupMode(group);
        if (mode == OverWindowMode.ROW) {
            isRangeWindows[i] = false;
            if (isUnboundedWindow(group)) {
                lowerBoundary.add(Long.MIN_VALUE);
                upperBoundary.add(Long.MAX_VALUE);
            } else if (isUnboundedPrecedingWindow(group)) {
                lowerBoundary.add(Long.MIN_VALUE);
                upperBoundary.add(OverAggregateUtil.getLongBoundary(overSpec, group.getUpperBound()));
            } else if (isUnboundedFollowingWindow(group)) {
                lowerBoundary.add(OverAggregateUtil.getLongBoundary(overSpec, group.getLowerBound()));
                upperBoundary.add(Long.MAX_VALUE);
            } else if (isSlidingWindow(group)) {
                lowerBoundary.add(OverAggregateUtil.getLongBoundary(overSpec, group.getLowerBound()));
                upperBoundary.add(OverAggregateUtil.getLongBoundary(overSpec, group.getUpperBound()));
            } else {
                throw new TableException("Unsupported row window group spec " + group);
            }
        } else {
            isRangeWindows[i] = true;
            if (isUnboundedWindow(group)) {
                lowerBoundary.add(Long.MIN_VALUE);
                upperBoundary.add(Long.MAX_VALUE);
            } else if (isUnboundedPrecedingWindow(group)) {
                lowerBoundary.add(Long.MIN_VALUE);
                upperBoundary.add(OverAggregateUtil.getLongBoundary(overSpec, group.getUpperBound()));
            } else if (isUnboundedFollowingWindow(group)) {
                lowerBoundary.add(OverAggregateUtil.getLongBoundary(overSpec, group.getLowerBound()));
                upperBoundary.add(Long.MAX_VALUE);
            } else if (isSlidingWindow(group)) {
                lowerBoundary.add(OverAggregateUtil.getLongBoundary(overSpec, group.getLowerBound()));
                upperBoundary.add(OverAggregateUtil.getLongBoundary(overSpec, group.getUpperBound()));
            } else {
                throw new TableException("Unsupported range window group spec " + group);
            }
        }
    }
    Configuration pythonConfig = CommonPythonUtil.getMergedConfig(planner.getExecEnv(), config.getTableConfig());
    OneInputTransformation<RowData, RowData> transform = createPythonOneInputTransformation(inputTransform, inputType, InternalTypeInfo.of(getOutputType()).toRowType(), isRangeWindows, pythonConfig, config);
    if (CommonPythonUtil.isPythonWorkerUsingManagedMemory(pythonConfig)) {
        transform.declareManagedMemoryUseCaseAtSlotScope(ManagedMemoryUseCase.PYTHON);
    }
    return transform;
}

Example 52 with AggregateCall

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.AggregateCall in project flink by apache.

the class StreamPhysicalPythonGroupTableAggregateRule method matches.

@Override
public boolean matches(RelOptRuleCall call) {
    FlinkLogicalTableAggregate agg = call.rel(0);
    List<AggregateCall> aggCalls = agg.getAggCallList();
    boolean existGeneralPythonFunction = aggCalls.stream().anyMatch(x -> PythonUtil.isPythonAggregate(x, PythonFunctionKind.GENERAL));
    boolean existPandasFunction = aggCalls.stream().anyMatch(x -> PythonUtil.isPythonAggregate(x, PythonFunctionKind.PANDAS));
    boolean existJavaUserDefinedFunction = aggCalls.stream().anyMatch(x -> !PythonUtil.isPythonAggregate(x, null) && !PythonUtil.isBuiltInAggregate(x));
    if (existPandasFunction || existGeneralPythonFunction) {
        if (existPandasFunction) {
            throw new TableException("Pandas UDAFs are not supported in streaming mode currently.");
        }
        if (existJavaUserDefinedFunction) {
            throw new TableException("Python UDAF and Java/Scala UDAF cannot be used together.");
        }
        return true;
    } else {
        return false;
    }
}

Example 53 with AggregateCall

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.AggregateCall in project flink by apache.

the class StreamPhysicalPythonOverAggregateRule method matches.

@Override
public boolean matches(RelOptRuleCall call) {
    FlinkLogicalOverAggregate logicWindow = call.rel(0);
    List<AggregateCall> aggCalls = logicWindow.groups.get(0).getAggregateCalls(logicWindow);
    boolean existGeneralPythonFunction = aggCalls.stream().anyMatch(x -> PythonUtil.isPythonAggregate(x, PythonFunctionKind.GENERAL));
    boolean existPandasFunction = aggCalls.stream().anyMatch(x -> PythonUtil.isPythonAggregate(x, PythonFunctionKind.PANDAS));
    boolean existJavaFunction = aggCalls.stream().anyMatch(x -> !PythonUtil.isPythonAggregate(x, null));
    if (existPandasFunction || existGeneralPythonFunction) {
        if (existGeneralPythonFunction) {
            throw new TableException("Non-Pandas Python UDAFs are not supported in stream mode currently.");
        }
        if (existJavaFunction) {
            throw new TableException("Python UDAF and Java/Scala UDAF cannot be used together.");
        }
        return true;
    } else {
        return false;
    }
}

Example 54 with AggregateCall

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.AggregateCall in project flink by apache.

the class PushLocalAggIntoScanRuleBase method pushLocalAggregateIntoScan.

protected void pushLocalAggregateIntoScan(RelOptRuleCall call, BatchPhysicalGroupAggregateBase localAgg, BatchPhysicalTableSourceScan oldScan, int[] calcRefFields) {
    RowType inputType = FlinkTypeFactory.toLogicalRowType(oldScan.getRowType());
    List<int[]> groupingSets = Collections.singletonList(ArrayUtils.addAll(localAgg.grouping(), localAgg.auxGrouping()));
    List<AggregateCall> aggCallList = JavaScalaConversionUtil.toJava(localAgg.getAggCallList());
    // map arg index in aggregate to field index in scan through referred fields by calc.
    if (calcRefFields != null) {
        groupingSets = translateGroupingArgIndex(groupingSets, calcRefFields);
        aggCallList = translateAggCallArgIndex(aggCallList, calcRefFields);
    }
    RowType producedType = FlinkTypeFactory.toLogicalRowType(localAgg.getRowType());
    TableSourceTable oldTableSourceTable = oldScan.tableSourceTable();
    DynamicTableSource newTableSource = oldScan.tableSource().copy();
    boolean isPushDownSuccess = AggregatePushDownSpec.apply(inputType, groupingSets, aggCallList, producedType, newTableSource, SourceAbilityContext.from(oldScan));
    if (!isPushDownSuccess) {
        // aggregate push down failed, just return without changing any nodes.
        return;
    }
    // create new source table with new spec and statistic.
    AggregatePushDownSpec aggregatePushDownSpec = new AggregatePushDownSpec(inputType, groupingSets, aggCallList, producedType);
    TableSourceTable newTableSourceTable = oldTableSourceTable.copy(newTableSource, localAgg.getRowType(), new SourceAbilitySpec[] { aggregatePushDownSpec }).copy(FlinkStatistic.UNKNOWN());
    // transform to new nodes.
    BatchPhysicalTableSourceScan newScan = oldScan.copy(oldScan.getTraitSet(), newTableSourceTable);
    BatchPhysicalExchange oldExchange = call.rel(0);
    BatchPhysicalExchange newExchange = oldExchange.copy(oldExchange.getTraitSet(), newScan, oldExchange.getDistribution());
    call.transformTo(newExchange);
}

Example 55 with AggregateCall

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.AggregateCall in project flink by apache.

the class StreamPhysicalPythonGroupWindowAggregateRule method matches.

@Override
public boolean matches(RelOptRuleCall call) {
    FlinkLogicalWindowAggregate agg = call.rel(0);
    List<AggregateCall> aggCalls = agg.getAggCallList();
    // check if we have grouping sets
    if (agg.getGroupType() != Aggregate.Group.SIMPLE || agg.indicator) {
        throw new TableException("GROUPING SETS are currently not supported.");
    }
    boolean existGeneralPythonFunction = aggCalls.stream().anyMatch(x -> PythonUtil.isPythonAggregate(x, PythonFunctionKind.GENERAL));
    boolean existPandasFunction = aggCalls.stream().anyMatch(x -> PythonUtil.isPythonAggregate(x, PythonFunctionKind.PANDAS));
    boolean existJavaFunction = aggCalls.stream().anyMatch(x -> !PythonUtil.isPythonAggregate(x, null) && !PythonUtil.isBuiltInAggregate(x));
    if (existPandasFunction && existGeneralPythonFunction) {
        throw new TableException("Pandas UDAFs and General Python UDAFs are not supported in used together currently.");
    }
    if (existPandasFunction || existGeneralPythonFunction) {
        if (existJavaFunction) {
            throw new TableException("Python UDAF and Java/Scala UDAF cannot be used together.");
        }
        return true;
    } else {
        return false;
    }
}