Examples with Aggregate org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Aggregate used on opensource projects

Example 21 with Aggregate

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Aggregate in project flink by apache.

the class FlinkAggregateJoinTransposeRule method toRegularAggregate.

/**
 * Convert aggregate with AUXILIARY_GROUP to regular aggregate. Return original aggregate and
 * null project if the given aggregate does not contain AUXILIARY_GROUP, else new aggregate
 * without AUXILIARY_GROUP and a project to permute output columns if needed.
 */
private Pair<Aggregate, List<RexNode>> toRegularAggregate(Aggregate aggregate) {
    Tuple2<int[], Seq<AggregateCall>> auxGroupAndRegularAggCalls = AggregateUtil.checkAndSplitAggCalls(aggregate);
    final int[] auxGroup = auxGroupAndRegularAggCalls._1;
    final Seq<AggregateCall> regularAggCalls = auxGroupAndRegularAggCalls._2;
    if (auxGroup.length != 0) {
        int[] fullGroupSet = AggregateUtil.checkAndGetFullGroupSet(aggregate);
        ImmutableBitSet newGroupSet = ImmutableBitSet.of(fullGroupSet);
        List<AggregateCall> aggCalls = JavaConverters.seqAsJavaListConverter(regularAggCalls).asJava();
        final Aggregate newAgg = aggregate.copy(aggregate.getTraitSet(), aggregate.getInput(), aggregate.indicator, newGroupSet, com.google.common.collect.ImmutableList.of(newGroupSet), aggCalls);
        final List<RelDataTypeField> aggFields = aggregate.getRowType().getFieldList();
        final List<RexNode> projectAfterAgg = new ArrayList<>();
        for (int i = 0; i < fullGroupSet.length; ++i) {
            int group = fullGroupSet[i];
            int index = newGroupSet.indexOf(group);
            projectAfterAgg.add(new RexInputRef(index, aggFields.get(i).getType()));
        }
        int fieldCntOfAgg = aggFields.size();
        for (int i = fullGroupSet.length; i < fieldCntOfAgg; ++i) {
            projectAfterAgg.add(new RexInputRef(i, aggFields.get(i).getType()));
        }
        Preconditions.checkArgument(projectAfterAgg.size() == fieldCntOfAgg);
        return new Pair<>(newAgg, projectAfterAgg);
    } else {
        return new Pair<>(aggregate, null);
    }
}

Example 22 with Aggregate

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Aggregate in project flink by apache.

the class FlinkAggregateExpandDistinctAggregatesRule method onMatch.

// ~ Methods ----------------------------------------------------------------
public void onMatch(RelOptRuleCall call) {
    final Aggregate aggregate = call.rel(0);
    if (!AggregateUtil.containsAccurateDistinctCall(aggregate.getAggCallList())) {
        return;
    }
    // accurate distinct call.
    if (AggregateUtil.containsApproximateDistinctCall(aggregate.getAggCallList())) {
        throw new TableException("There are both Distinct AggCall and Approximate Distinct AggCall in one sql statement, " + "it is not supported yet.\nPlease choose one of them.");
    }
    // by DecomposeGroupingSetsRule. Then this rule expands it's distinct aggregates.
    if (aggregate.getGroupSets().size() > 1) {
        return;
    }
    // Find all of the agg expressions. We use a LinkedHashSet to ensure determinism.
    // Find all aggregate calls without distinct
    int nonDistinctAggCallCount = 0;
    // Find all aggregate calls without distinct but ignore MAX, MIN, BIT_AND, BIT_OR
    int nonDistinctAggCallExcludingIgnoredCount = 0;
    int filterCount = 0;
    int unsupportedNonDistinctAggCallCount = 0;
    final Set<Pair<List<Integer>, Integer>> argLists = new LinkedHashSet<>();
    for (AggregateCall aggCall : aggregate.getAggCallList()) {
        if (aggCall.filterArg >= 0) {
            ++filterCount;
        }
        if (!aggCall.isDistinct()) {
            ++nonDistinctAggCallCount;
            final SqlKind aggCallKind = aggCall.getAggregation().getKind();
            // We only support COUNT/SUM/MIN/MAX for the "single" count distinct optimization
            switch(aggCallKind) {
                case COUNT:
                case SUM:
                case SUM0:
                case MIN:
                case MAX:
                    break;
                default:
                    ++unsupportedNonDistinctAggCallCount;
            }
            if (aggCall.getAggregation().getDistinctOptionality() == Optionality.IGNORED) {
                argLists.add(Pair.of(aggCall.getArgList(), aggCall.filterArg));
            } else {
                ++nonDistinctAggCallExcludingIgnoredCount;
            }
        } else {
            argLists.add(Pair.of(aggCall.getArgList(), aggCall.filterArg));
        }
    }
    final int distinctAggCallCount = aggregate.getAggCallList().size() - nonDistinctAggCallCount;
    Preconditions.checkState(argLists.size() > 0, "containsDistinctCall lied");
    // we can still use this promotion.
    if (nonDistinctAggCallExcludingIgnoredCount == 0 && argLists.size() == 1 && aggregate.getGroupType() == Group.SIMPLE) {
        final Pair<List<Integer>, Integer> pair = com.google.common.collect.Iterables.getOnlyElement(argLists);
        final RelBuilder relBuilder = call.builder();
        convertMonopole(relBuilder, aggregate, pair.left, pair.right);
        call.transformTo(relBuilder.build());
        return;
    }
    if (useGroupingSets) {
        rewriteUsingGroupingSets(call, aggregate);
        return;
    }
    // we can generate multi-phase aggregates
    if (// one distinct aggregate
    distinctAggCallCount == 1 && // no filter
    filterCount == 0 && unsupportedNonDistinctAggCallCount == // sum/min/max/count in non-distinct aggregate
    0 && nonDistinctAggCallCount > 0) {
        // one or more non-distinct aggregates
        final RelBuilder relBuilder = call.builder();
        convertSingletonDistinct(relBuilder, aggregate, argLists);
        call.transformTo(relBuilder.build());
        return;
    }
    // Create a list of the expressions which will yield the final result.
    // Initially, the expressions point to the input field.
    final List<RelDataTypeField> aggFields = aggregate.getRowType().getFieldList();
    final List<RexInputRef> refs = new ArrayList<>();
    final List<String> fieldNames = aggregate.getRowType().getFieldNames();
    final ImmutableBitSet groupSet = aggregate.getGroupSet();
    final int groupCount = aggregate.getGroupCount();
    for (int i : Util.range(groupCount)) {
        refs.add(RexInputRef.of(i, aggFields));
    }
    // Aggregate the original relation, including any non-distinct aggregates.
    final List<AggregateCall> newAggCallList = new ArrayList<>();
    int i = -1;
    for (AggregateCall aggCall : aggregate.getAggCallList()) {
        ++i;
        if (aggCall.isDistinct()) {
            refs.add(null);
            continue;
        }
        refs.add(new RexInputRef(groupCount + newAggCallList.size(), aggFields.get(groupCount + i).getType()));
        newAggCallList.add(aggCall);
    }
    // In the case where there are no non-distinct aggregates (regardless of
    // whether there are group bys), there's no need to generate the
    // extra aggregate and join.
    final RelBuilder relBuilder = call.builder();
    relBuilder.push(aggregate.getInput());
    int n = 0;
    if (!newAggCallList.isEmpty()) {
        final RelBuilder.GroupKey groupKey = relBuilder.groupKey(groupSet, aggregate.getGroupSets());
        relBuilder.aggregate(groupKey, newAggCallList);
        ++n;
    }
    // set of operands.
    for (Pair<List<Integer>, Integer> argList : argLists) {
        doRewrite(relBuilder, aggregate, n++, argList.left, argList.right, refs);
    }
    relBuilder.project(refs, fieldNames);
    call.transformTo(relBuilder.build());
}

Example 23 with Aggregate

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Aggregate in project flink by apache.

the class FlinkAggregateRemoveRule method matches.

@Override
public boolean matches(RelOptRuleCall call) {
    final Aggregate aggregate = call.rel(0);
    final RelNode input = call.rel(1);
    if (aggregate.getGroupCount() == 0 || aggregate.indicator || aggregate.getGroupType() != Aggregate.Group.SIMPLE) {
        return false;
    }
    for (AggregateCall aggCall : aggregate.getAggCallList()) {
        SqlKind aggCallKind = aggCall.getAggregation().getKind();
        // TODO supports more AggregateCalls
        boolean isAllowAggCall = aggCallKind == SqlKind.SUM || aggCallKind == SqlKind.MIN || aggCallKind == SqlKind.MAX || aggCall.getAggregation() instanceof SqlAuxiliaryGroupAggFunction;
        if (!isAllowAggCall || aggCall.filterArg >= 0 || aggCall.getArgList().size() != 1) {
            return false;
        }
    }
    final RelMetadataQuery mq = call.getMetadataQuery();
    return SqlFunctions.isTrue(mq.areColumnsUnique(input, aggregate.getGroupSet()));
}

Example 24 with Aggregate

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Aggregate in project flink by apache.

the class FlinkAggregateRemoveRule method onMatch.

// ~ Methods ----------------------------------------------------------------
public void onMatch(RelOptRuleCall call) {
    final Aggregate aggregate = call.rel(0);
    final RelNode input = call.rel(1);
    // Distinct is "GROUP BY c1, c2" (where c1, c2 are a set of columns on
    // which the input is unique, i.e. contain a key) and has no aggregate
    // functions or the functions we enumerated. It can be removed.
    final RelNode newInput = convert(input, aggregate.getTraitSet().simplify());
    // If aggregate was projecting a subset of columns, add a project for the
    // same effect.
    final RelBuilder relBuilder = call.builder();
    relBuilder.push(newInput);
    List<Integer> projectIndices = new ArrayList<>(aggregate.getGroupSet().asList());
    for (AggregateCall aggCall : aggregate.getAggCallList()) {
        projectIndices.addAll(aggCall.getArgList());
    }
    relBuilder.project(relBuilder.fields(projectIndices));
    // Create a project if some of the columns have become
    // NOT NULL due to aggregate functions are removed
    relBuilder.convert(aggregate.getRowType(), true);
    call.transformTo(relBuilder.build());
}

Example 25 with Aggregate

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Aggregate in project beam by apache.

the class AggregateScanConverter method convert.

@Override
public RelNode convert(ResolvedAggregateScan zetaNode, List<RelNode> inputs) {
    LogicalProject input = convertAggregateScanInputScanToLogicalProject(zetaNode, inputs.get(0));
    // Calcite LogicalAggregate's GroupSet is indexes of group fields starting from 0.
    int groupFieldsListSize = zetaNode.getGroupByList().size();
    ImmutableBitSet groupSet;
    if (groupFieldsListSize != 0) {
        groupSet = ImmutableBitSet.of(IntStream.rangeClosed(0, groupFieldsListSize - 1).boxed().collect(Collectors.toList()));
    } else {
        groupSet = ImmutableBitSet.of();
    }
    // TODO: add support for indicator
    List<AggregateCall> aggregateCalls;
    if (zetaNode.getAggregateList().isEmpty()) {
        aggregateCalls = ImmutableList.of();
    } else {
        aggregateCalls = new ArrayList<>();
        // For aggregate calls, their input ref follow after GROUP BY input ref.
        int columnRefoff = groupFieldsListSize;
        for (ResolvedComputedColumn computedColumn : zetaNode.getAggregateList()) {
            AggregateCall aggCall = convertAggCall(computedColumn, columnRefoff, groupSet.size(), input);
            aggregateCalls.add(aggCall);
            if (!aggCall.getArgList().isEmpty()) {
                // Only increment column reference offset when aggregates use them (BEAM-8042).
                // Ex: COUNT(*) does not have arguments, while COUNT(`field`) does.
                columnRefoff++;
            }
        }
    }
    LogicalAggregate logicalAggregate = new LogicalAggregate(getCluster(), input.getTraitSet(), input, groupSet, ImmutableList.of(groupSet), aggregateCalls);
    return logicalAggregate;
}