Examples with SqlAggFunction org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlAggFunction used on opensource projects

Example 36 with SqlAggFunction

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlAggFunction in project flink by apache.

the class SqlFunctionConverter method visitOver.

@Override
public RexNode visitOver(RexOver over) {
    SqlOperator operator = convertOperator(over.getAggOperator());
    Preconditions.checkArgument(operator instanceof SqlAggFunction, "Expect converted operator to be an agg function, but got " + operator.toString());
    SqlAggFunction convertedAgg = (SqlAggFunction) operator;
    RexWindow window = over.getWindow();
    // let's not rely on the type of the RexOver created by Hive since it can be different from
    // what Flink expects
    RelDataType inferredType = builder.makeCall(convertedAgg, over.getOperands()).getType();
    // Hive may add literals to partition keys, remove them
    List<RexNode> partitionKeys = new ArrayList<>();
    for (RexNode hivePartitionKey : getPartKeys(window)) {
        if (!(hivePartitionKey instanceof RexLiteral)) {
            partitionKeys.add(hivePartitionKey);
        }
    }
    List<RexFieldCollation> convertedOrderKeys = new ArrayList<>(getOrderKeys(window).size());
    for (RexFieldCollation orderKey : getOrderKeys(window)) {
        convertedOrderKeys.add(new RexFieldCollation(orderKey.getKey().accept(this), orderKey.getValue()));
    }
    final boolean[] update = null;
    return HiveParserUtils.makeOver(builder, inferredType, convertedAgg, visitList(over.getOperands(), update), visitList(partitionKeys, update), convertedOrderKeys, window.getLowerBound(), window.getUpperBound(), window.isRows(), true, false, false, false);
}

Example 37 with SqlAggFunction

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlAggFunction in project flink by apache.

the class HiveParserCalcitePlanner method getWindowRexAndType.

private Pair<RexNode, TypeInfo> getWindowRexAndType(HiveParserWindowingSpec.WindowExpressionSpec winExprSpec, RelNode srcRel) throws SemanticException {
    RexNode window;
    if (winExprSpec instanceof HiveParserWindowingSpec.WindowFunctionSpec) {
        HiveParserWindowingSpec.WindowFunctionSpec wFnSpec = (HiveParserWindowingSpec.WindowFunctionSpec) winExprSpec;
        HiveParserASTNode windowProjAst = wFnSpec.getExpression();
        // TODO: do we need to get to child?
        int wndSpecASTIndx = getWindowSpecIndx(windowProjAst);
        // 2. Get Hive Aggregate Info
        AggInfo hiveAggInfo = getHiveAggInfo(windowProjAst, wndSpecASTIndx - 1, relToRowResolver.get(srcRel), (HiveParserWindowingSpec.WindowFunctionSpec) winExprSpec, semanticAnalyzer, frameworkConfig, cluster);
        // 3. Get Calcite Return type for Agg Fn
        RelDataType calciteAggFnRetType = HiveParserUtils.toRelDataType(hiveAggInfo.getReturnType(), cluster.getTypeFactory());
        // 4. Convert Agg Fn args to Calcite
        Map<String, Integer> posMap = relToHiveColNameCalcitePosMap.get(srcRel);
        HiveParserRexNodeConverter converter = new HiveParserRexNodeConverter(cluster, srcRel.getRowType(), posMap, 0, false, funcConverter);
        List<RexNode> calciteAggFnArgs = new ArrayList<>();
        List<RelDataType> calciteAggFnArgTypes = new ArrayList<>();
        for (int i = 0; i < hiveAggInfo.getAggParams().size(); i++) {
            calciteAggFnArgs.add(converter.convert(hiveAggInfo.getAggParams().get(i)));
            calciteAggFnArgTypes.add(HiveParserUtils.toRelDataType(hiveAggInfo.getAggParams().get(i).getTypeInfo(), cluster.getTypeFactory()));
        }
        // 5. Get Calcite Agg Fn
        final SqlAggFunction calciteAggFn = HiveParserSqlFunctionConverter.getCalciteAggFn(hiveAggInfo.getUdfName(), hiveAggInfo.isDistinct(), calciteAggFnArgTypes, calciteAggFnRetType);
        // 6. Translate Window spec
        HiveParserRowResolver inputRR = relToRowResolver.get(srcRel);
        HiveParserWindowingSpec.WindowSpec wndSpec = ((HiveParserWindowingSpec.WindowFunctionSpec) winExprSpec).getWindowSpec();
        List<RexNode> partitionKeys = getPartitionKeys(wndSpec.getPartition(), converter, inputRR, new HiveParserTypeCheckCtx(inputRR, frameworkConfig, cluster), semanticAnalyzer);
        List<RexFieldCollation> orderKeys = getOrderKeys(wndSpec.getOrder(), converter, inputRR, new HiveParserTypeCheckCtx(inputRR, frameworkConfig, cluster), semanticAnalyzer);
        RexWindowBound lowerBound = getBound(wndSpec.getWindowFrame().getStart(), cluster);
        RexWindowBound upperBound = getBound(wndSpec.getWindowFrame().getEnd(), cluster);
        boolean isRows = wndSpec.getWindowFrame().getWindowType() == HiveParserWindowingSpec.WindowType.ROWS;
        window = HiveParserUtils.makeOver(cluster.getRexBuilder(), calciteAggFnRetType, calciteAggFn, calciteAggFnArgs, partitionKeys, orderKeys, lowerBound, upperBound, isRows, true, false, false, false);
        window = window.accept(funcConverter);
    } else {
        throw new SemanticException("Unsupported window Spec");
    }
    return new Pair<>(window, HiveParserTypeConverter.convert(window.getType()));
}

Example 38 with SqlAggFunction

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlAggFunction in project calcite by apache.

the class AggregateStarTableRule method rollUp.

private static AggregateCall rollUp(int groupCount, RelBuilder relBuilder, AggregateCall aggregateCall, TileKey tileKey) {
    if (aggregateCall.isDistinct()) {
        return null;
    }
    final SqlAggFunction aggregation = aggregateCall.getAggregation();
    final Pair<SqlAggFunction, List<Integer>> seek = Pair.of(aggregation, aggregateCall.getArgList());
    final int offset = tileKey.dimensions.cardinality();
    final ImmutableList<Lattice.Measure> measures = tileKey.measures;
    // First, try to satisfy the aggregation by rolling up an aggregate in the
    // materialization.
    final int i = find(measures, seek);
    tryRoll: if (i >= 0) {
        final SqlAggFunction roll = SubstitutionVisitor.getRollup(aggregation);
        if (roll == null) {
            break tryRoll;
        }
        return AggregateCall.create(roll, false, aggregateCall.isApproximate(), ImmutableList.of(offset + i), -1, groupCount, relBuilder.peek(), null, aggregateCall.name);
    }
    // Second, try to satisfy the aggregation based on group set columns.
    tryGroup: {
        List<Integer> newArgs = Lists.newArrayList();
        for (Integer arg : aggregateCall.getArgList()) {
            int z = tileKey.dimensions.indexOf(arg);
            if (z < 0) {
                break tryGroup;
            }
            newArgs.add(z);
        }
        return AggregateCall.create(aggregation, false, aggregateCall.isApproximate(), newArgs, -1, groupCount, relBuilder.peek(), null, aggregateCall.name);
    }
    // No roll up possible.
    return null;
}

Example 39 with SqlAggFunction

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlAggFunction in project calcite by apache.

the class AggregateUnionTransposeRule method transformAggCalls.

private List<AggregateCall> transformAggCalls(RelNode input, int groupCount, List<AggregateCall> origCalls) {
    final List<AggregateCall> newCalls = Lists.newArrayList();
    for (Ord<AggregateCall> ord : Ord.zip(origCalls)) {
        final AggregateCall origCall = ord.e;
        if (origCall.isDistinct() || !SUPPORTED_AGGREGATES.containsKey(origCall.getAggregation().getClass())) {
            return null;
        }
        final SqlAggFunction aggFun;
        final RelDataType aggType;
        if (origCall.getAggregation() == SqlStdOperatorTable.COUNT) {
            aggFun = SqlStdOperatorTable.SUM0;
            // count(any) is always not null, however nullability of sum might
            // depend on the number of columns in GROUP BY.
            // Here we use SUM0 since we are sure we will not face nullable
            // inputs nor we'll face empty set.
            aggType = null;
        } else {
            aggFun = origCall.getAggregation();
            aggType = origCall.getType();
        }
        AggregateCall newCall = AggregateCall.create(aggFun, origCall.isDistinct(), origCall.isApproximate(), ImmutableList.of(groupCount + ord.i), -1, groupCount, input, aggType, origCall.getName());
        newCalls.add(newCall);
    }
    return newCalls;
}

Example 40 with SqlAggFunction

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlAggFunction in project calcite by apache.

the class AggregateExpandDistinctAggregatesRule method rewriteUsingGroupingSets.

private void rewriteUsingGroupingSets(RelOptRuleCall call, Aggregate aggregate) {
    final Set<ImmutableBitSet> groupSetTreeSet = new TreeSet<>(ImmutableBitSet.ORDERING);
    for (AggregateCall aggCall : aggregate.getAggCallList()) {
        if (!aggCall.isDistinct()) {
            groupSetTreeSet.add(aggregate.getGroupSet());
        } else {
            groupSetTreeSet.add(ImmutableBitSet.of(aggCall.getArgList()).setIf(aggCall.filterArg, aggCall.filterArg >= 0).union(aggregate.getGroupSet()));
        }
    }
    final ImmutableList<ImmutableBitSet> groupSets = ImmutableList.copyOf(groupSetTreeSet);
    final ImmutableBitSet fullGroupSet = ImmutableBitSet.union(groupSets);
    final List<AggregateCall> distinctAggCalls = new ArrayList<>();
    for (Pair<AggregateCall, String> aggCall : aggregate.getNamedAggCalls()) {
        if (!aggCall.left.isDistinct()) {
            AggregateCall newAggCall = aggCall.left.adaptTo(aggregate.getInput(), aggCall.left.getArgList(), aggCall.left.filterArg, aggregate.getGroupCount(), fullGroupSet.cardinality());
            distinctAggCalls.add(newAggCall.rename(aggCall.right));
        }
    }
    final RelBuilder relBuilder = call.builder();
    relBuilder.push(aggregate.getInput());
    final int groupCount = fullGroupSet.cardinality();
    final Map<ImmutableBitSet, Integer> filters = new LinkedHashMap<>();
    final int z = groupCount + distinctAggCalls.size();
    distinctAggCalls.add(AggregateCall.create(SqlStdOperatorTable.GROUPING, false, false, ImmutableIntList.copyOf(fullGroupSet), -1, groupSets.size(), relBuilder.peek(), null, "$g"));
    for (Ord<ImmutableBitSet> groupSet : Ord.zip(groupSets)) {
        filters.put(groupSet.e, z + groupSet.i);
    }
    relBuilder.aggregate(relBuilder.groupKey(fullGroupSet, groupSets), distinctAggCalls);
    final RelNode distinct = relBuilder.peek();
    // values to BOOLEAN.
    if (!filters.isEmpty()) {
        final List<RexNode> nodes = new ArrayList<>(relBuilder.fields());
        final RexNode nodeZ = nodes.remove(nodes.size() - 1);
        for (Map.Entry<ImmutableBitSet, Integer> entry : filters.entrySet()) {
            final long v = groupValue(fullGroupSet, entry.getKey());
            nodes.add(relBuilder.alias(relBuilder.equals(nodeZ, relBuilder.literal(v)), "$g_" + v));
        }
        relBuilder.project(nodes);
    }
    int x = groupCount;
    final List<AggregateCall> newCalls = new ArrayList<>();
    for (AggregateCall aggCall : aggregate.getAggCallList()) {
        final int newFilterArg;
        final List<Integer> newArgList;
        final SqlAggFunction aggregation;
        if (!aggCall.isDistinct()) {
            aggregation = SqlStdOperatorTable.MIN;
            newArgList = ImmutableIntList.of(x++);
            newFilterArg = filters.get(aggregate.getGroupSet());
        } else {
            aggregation = aggCall.getAggregation();
            newArgList = remap(fullGroupSet, aggCall.getArgList());
            newFilterArg = filters.get(ImmutableBitSet.of(aggCall.getArgList()).setIf(aggCall.filterArg, aggCall.filterArg >= 0).union(aggregate.getGroupSet()));
        }
        final AggregateCall newCall = AggregateCall.create(aggregation, false, aggCall.isApproximate(), newArgList, newFilterArg, aggregate.getGroupCount(), distinct, null, aggCall.name);
        newCalls.add(newCall);
    }
    relBuilder.aggregate(relBuilder.groupKey(remap(fullGroupSet, aggregate.getGroupSet()), remap(fullGroupSet, aggregate.getGroupSets())), newCalls);
    relBuilder.convert(aggregate.getRowType(), true);
    call.transformTo(relBuilder.build());
}