Examples with Aggregate org.apache.calcite.rel.core.Aggregate used on opensource projects

Example 41 with Aggregate

use of org.apache.calcite.rel.core.Aggregate in project flink by apache.

the class FlinkAggregateJoinTransposeRule method onMatch.

public void onMatch(RelOptRuleCall call) {
    final Aggregate aggregate = call.rel(0);
    final Join join = call.rel(1);
    final RexBuilder rexBuilder = aggregate.getCluster().getRexBuilder();
    final RelBuilder relBuilder = call.builder();
    // If any aggregate call has a filter, bail out
    for (AggregateCall aggregateCall : aggregate.getAggCallList()) {
        if (aggregateCall.getAggregation().unwrap(SqlSplittableAggFunction.class) == null) {
            return;
        }
        if (aggregateCall.filterArg >= 0) {
            return;
        }
    }
    // aggregate operator
    if (join.getJoinType() != JoinRelType.INNER) {
        return;
    }
    if (!allowFunctions && !aggregate.getAggCallList().isEmpty()) {
        return;
    }
    // Do the columns used by the join appear in the output of the aggregate?
    final ImmutableBitSet aggregateColumns = aggregate.getGroupSet();
    final RelMetadataQuery mq = RelMetadataQuery.instance();
    final ImmutableBitSet keyColumns = keyColumns(aggregateColumns, mq.getPulledUpPredicates(join).pulledUpPredicates);
    final ImmutableBitSet joinColumns = RelOptUtil.InputFinder.bits(join.getCondition());
    final boolean allColumnsInAggregate = keyColumns.contains(joinColumns);
    final ImmutableBitSet belowAggregateColumns = aggregateColumns.union(joinColumns);
    // Split join condition
    final List<Integer> leftKeys = Lists.newArrayList();
    final List<Integer> rightKeys = Lists.newArrayList();
    final List<Boolean> filterNulls = Lists.newArrayList();
    RexNode nonEquiConj = RelOptUtil.splitJoinCondition(join.getLeft(), join.getRight(), join.getCondition(), leftKeys, rightKeys, filterNulls);
    // If it contains non-equi join conditions, we bail out
    if (!nonEquiConj.isAlwaysTrue()) {
        return;
    }
    // Push each aggregate function down to each side that contains all of its
    // arguments. Note that COUNT(*), because it has no arguments, can go to
    // both sides.
    final Map<Integer, Integer> map = new HashMap<>();
    final List<Side> sides = new ArrayList<>();
    int uniqueCount = 0;
    int offset = 0;
    int belowOffset = 0;
    for (int s = 0; s < 2; s++) {
        final Side side = new Side();
        final RelNode joinInput = join.getInput(s);
        int fieldCount = joinInput.getRowType().getFieldCount();
        final ImmutableBitSet fieldSet = ImmutableBitSet.range(offset, offset + fieldCount);
        final ImmutableBitSet belowAggregateKeyNotShifted = belowAggregateColumns.intersect(fieldSet);
        for (Ord<Integer> c : Ord.zip(belowAggregateKeyNotShifted)) {
            map.put(c.e, belowOffset + c.i);
        }
        final ImmutableBitSet belowAggregateKey = belowAggregateKeyNotShifted.shift(-offset);
        final boolean unique;
        if (!allowFunctions) {
            assert aggregate.getAggCallList().isEmpty();
            // If there are no functions, it doesn't matter as much whether we
            // aggregate the inputs before the join, because there will not be
            // any functions experiencing a cartesian product effect.
            //
            // But finding out whether the input is already unique requires a call
            // to areColumnsUnique that currently (until [CALCITE-1048] "Make
            // metadata more robust" is fixed) places a heavy load on
            // the metadata system.
            //
            // So we choose to imagine the the input is already unique, which is
            // untrue but harmless.
            //
            Util.discard(Bug.CALCITE_1048_FIXED);
            unique = true;
        } else {
            final Boolean unique0 = mq.areColumnsUnique(joinInput, belowAggregateKey);
            unique = unique0 != null && unique0;
        }
        if (unique) {
            ++uniqueCount;
            side.aggregate = false;
            side.newInput = joinInput;
        } else {
            side.aggregate = true;
            List<AggregateCall> belowAggCalls = new ArrayList<>();
            final SqlSplittableAggFunction.Registry<AggregateCall> belowAggCallRegistry = registry(belowAggCalls);
            final Mappings.TargetMapping mapping = s == 0 ? Mappings.createIdentity(fieldCount) : Mappings.createShiftMapping(fieldCount + offset, 0, offset, fieldCount);
            for (Ord<AggregateCall> aggCall : Ord.zip(aggregate.getAggCallList())) {
                final SqlAggFunction aggregation = aggCall.e.getAggregation();
                final SqlSplittableAggFunction splitter = Preconditions.checkNotNull(aggregation.unwrap(SqlSplittableAggFunction.class));
                final AggregateCall call1;
                if (fieldSet.contains(ImmutableBitSet.of(aggCall.e.getArgList()))) {
                    call1 = splitter.split(aggCall.e, mapping);
                } else {
                    call1 = splitter.other(rexBuilder.getTypeFactory(), aggCall.e);
                }
                if (call1 != null) {
                    side.split.put(aggCall.i, belowAggregateKey.cardinality() + belowAggCallRegistry.register(call1));
                }
            }
            side.newInput = relBuilder.push(joinInput).aggregate(relBuilder.groupKey(belowAggregateKey, false, null), belowAggCalls).build();
        }
        offset += fieldCount;
        belowOffset += side.newInput.getRowType().getFieldCount();
        sides.add(side);
    }
    if (uniqueCount == 2) {
        // invocation of this rule; if we continue we might loop forever.
        return;
    }
    // Update condition
    final Mapping mapping = (Mapping) Mappings.target(new Function<Integer, Integer>() {

        public Integer apply(Integer a0) {
            return map.get(a0);
        }
    }, join.getRowType().getFieldCount(), belowOffset);
    final RexNode newCondition = RexUtil.apply(mapping, join.getCondition());
    // Create new join
    relBuilder.push(sides.get(0).newInput).push(sides.get(1).newInput).join(join.getJoinType(), newCondition);
    // Aggregate above to sum up the sub-totals
    final List<AggregateCall> newAggCalls = new ArrayList<>();
    final int groupIndicatorCount = aggregate.getGroupCount() + aggregate.getIndicatorCount();
    final int newLeftWidth = sides.get(0).newInput.getRowType().getFieldCount();
    final List<RexNode> projects = new ArrayList<>(rexBuilder.identityProjects(relBuilder.peek().getRowType()));
    for (Ord<AggregateCall> aggCall : Ord.zip(aggregate.getAggCallList())) {
        final SqlAggFunction aggregation = aggCall.e.getAggregation();
        final SqlSplittableAggFunction splitter = Preconditions.checkNotNull(aggregation.unwrap(SqlSplittableAggFunction.class));
        final Integer leftSubTotal = sides.get(0).split.get(aggCall.i);
        final Integer rightSubTotal = sides.get(1).split.get(aggCall.i);
        newAggCalls.add(splitter.topSplit(rexBuilder, registry(projects), groupIndicatorCount, relBuilder.peek().getRowType(), aggCall.e, leftSubTotal == null ? -1 : leftSubTotal, rightSubTotal == null ? -1 : rightSubTotal + newLeftWidth));
    }
    relBuilder.project(projects);
    boolean aggConvertedToProjects = false;
    if (allColumnsInAggregate) {
        // let's see if we can convert aggregate into projects
        List<RexNode> projects2 = new ArrayList<>();
        for (int key : Mappings.apply(mapping, aggregate.getGroupSet())) {
            projects2.add(relBuilder.field(key));
        }
        for (AggregateCall newAggCall : newAggCalls) {
            final SqlSplittableAggFunction splitter = newAggCall.getAggregation().unwrap(SqlSplittableAggFunction.class);
            if (splitter != null) {
                projects2.add(splitter.singleton(rexBuilder, relBuilder.peek().getRowType(), newAggCall));
            }
        }
        if (projects2.size() == aggregate.getGroupSet().cardinality() + newAggCalls.size()) {
            // We successfully converted agg calls into projects.
            relBuilder.project(projects2);
            aggConvertedToProjects = true;
        }
    }
    if (!aggConvertedToProjects) {
        relBuilder.aggregate(relBuilder.groupKey(Mappings.apply(mapping, aggregate.getGroupSet()), aggregate.indicator, Mappings.apply2(mapping, aggregate.getGroupSets())), newAggCalls);
    }
    call.transformTo(relBuilder.build());
}

Example 42 with Aggregate

use of org.apache.calcite.rel.core.Aggregate in project hive by apache.

the class HiveExpandDistinctAggregatesRule method createCount.

/**
 * @param aggr: the original aggregate
 * @param argList: the original argList in aggregate
 * @param cleanArgList: the new argList without duplicates
 * @param map: the mapping from the original argList to the new argList
 * @param sourceOfForCountDistinct: the sorted positions of groupset
 * @return
 * @throws CalciteSemanticException
 */
private RelNode createCount(Aggregate aggr, List<List<Integer>> argList, List<List<Integer>> cleanArgList, Map<Integer, Integer> map, List<Integer> sourceOfForCountDistinct) throws CalciteSemanticException {
    List<RexNode> originalInputRefs = Lists.transform(aggr.getRowType().getFieldList(), new Function<RelDataTypeField, RexNode>() {

        @Override
        public RexNode apply(RelDataTypeField input) {
            return new RexInputRef(input.getIndex(), input.getType());
        }
    });
    final List<RexNode> gbChildProjLst = Lists.newArrayList();
    // for non-singular args, count can include null, i.e. (,) is counted as 1
    for (List<Integer> list : cleanArgList) {
        RexNode condition = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, originalInputRefs.get(originalInputRefs.size() - 1), rexBuilder.makeExactLiteral(new BigDecimal(getGroupingIdValue(list, sourceOfForCountDistinct, aggr.getGroupCount()))));
        if (list.size() == 1) {
            int pos = list.get(0);
            RexNode notNull = rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, originalInputRefs.get(pos));
            condition = rexBuilder.makeCall(SqlStdOperatorTable.AND, condition, notNull);
        }
        RexNode when = rexBuilder.makeCall(SqlStdOperatorTable.CASE, condition, rexBuilder.makeExactLiteral(BigDecimal.ONE), rexBuilder.constantNull());
        gbChildProjLst.add(when);
    }
    // create the project before GB
    RelNode gbInputRel = HiveProject.create(aggr, gbChildProjLst, null);
    // create the aggregate
    List<AggregateCall> aggregateCalls = Lists.newArrayList();
    RelDataType aggFnRetType = TypeConverter.convert(TypeInfoFactory.longTypeInfo, cluster.getTypeFactory());
    for (int i = 0; i < cleanArgList.size(); i++) {
        AggregateCall aggregateCall = HiveCalciteUtil.createSingleArgAggCall("count", cluster, TypeInfoFactory.longTypeInfo, i, aggFnRetType);
        aggregateCalls.add(aggregateCall);
    }
    Aggregate aggregate = new HiveAggregate(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), gbInputRel, ImmutableBitSet.of(), null, aggregateCalls);
    // count(distinct x, y), count(distinct y, x), we find the correct mapping.
    if (map.isEmpty()) {
        return aggregate;
    } else {
        List<RexNode> originalAggrRefs = Lists.transform(aggregate.getRowType().getFieldList(), new Function<RelDataTypeField, RexNode>() {

            @Override
            public RexNode apply(RelDataTypeField input) {
                return new RexInputRef(input.getIndex(), input.getType());
            }
        });
        final List<RexNode> projLst = Lists.newArrayList();
        int index = 0;
        for (int i = 0; i < argList.size(); i++) {
            if (map.containsKey(i)) {
                projLst.add(originalAggrRefs.get(map.get(i)));
            } else {
                projLst.add(originalAggrRefs.get(index++));
            }
        }
        return HiveProject.create(aggregate, projLst, null);
    }
}

Example 43 with Aggregate

use of org.apache.calcite.rel.core.Aggregate in project drill by axbaretto.

the class DrillReduceAggregatesRule method onMatch.

@Override
public void onMatch(RelOptRuleCall ruleCall) {
    Aggregate oldAggRel = (Aggregate) ruleCall.rels[0];
    reduceAggs(ruleCall, oldAggRel);
}

Example 44 with Aggregate

use of org.apache.calcite.rel.core.Aggregate in project drill by axbaretto.

the class DrillFilterAggregateTransposeRule method matches.

@Override
public boolean matches(RelOptRuleCall call) {
    final Filter filter = call.rel(0);
    final Aggregate aggregate = call.rel(1);
    return filter.getTraitSet().getTrait(ConventionTraitDef.INSTANCE) == aggregate.getTraitSet().getTrait(ConventionTraitDef.INSTANCE);
}

Example 45 with Aggregate

use of org.apache.calcite.rel.core.Aggregate in project hive by apache.

the class HiveMaterializedViewUtils method deriveGroupingSetsMaterializedViews.

/**
 * If a materialization does not contain grouping sets, it returns the materialization
 * itself. Otherwise, it will create one materialization for each grouping set.
 * For each grouping set, the query for the materialization will consist of the group by
 * columns in the grouping set, followed by a projection to recreate the literal null
 * values. The view scan will consist of the scan over the materialization followed by a
 * filter on the grouping id value corresponding to that grouping set.
 */
public static List<HiveRelOptMaterialization> deriveGroupingSetsMaterializedViews(HiveRelOptMaterialization materialization) {
    final RelNode query = materialization.queryRel;
    final Project project;
    final Aggregate aggregate;
    if (query instanceof Aggregate) {
        project = null;
        aggregate = (Aggregate) query;
    } else if (query instanceof Project && query.getInput(0) instanceof Aggregate) {
        project = (Project) query;
        aggregate = (Aggregate) query.getInput(0);
    } else {
        project = null;
        aggregate = null;
    }
    if (aggregate == null) {
        // Not an aggregate materialized view, return original materialization
        return Collections.singletonList(materialization);
    }
    if (aggregate.getGroupType() == Group.SIMPLE) {
        // Not a grouping sets materialized view, return original materialization
        return Collections.singletonList(materialization);
    }
    int aggregateGroupingIdIndex = -1;
    for (int i = 0; i < aggregate.getAggCallList().size(); i++) {
        if (aggregate.getAggCallList().get(i).getAggregation() == HiveGroupingID.INSTANCE) {
            aggregateGroupingIdIndex = aggregate.getGroupCount() + i;
            break;
        }
    }
    Preconditions.checkState(aggregateGroupingIdIndex != -1);
    int projectGroupingIdIndex = -1;
    if (project != null) {
        for (int i = 0; i < project.getProjects().size(); i++) {
            RexNode expr = project.getProjects().get(i);
            if (expr instanceof RexInputRef) {
                RexInputRef ref = (RexInputRef) expr;
                if (ref.getIndex() == aggregateGroupingIdIndex) {
                    // Grouping id is present
                    projectGroupingIdIndex = i;
                    break;
                }
            }
        }
        if (projectGroupingIdIndex == -1) {
            // Grouping id is not present, return original materialization
            return Collections.singletonList(materialization);
        }
    }
    // Create multiple materializations
    final List<HiveRelOptMaterialization> materializationList = new ArrayList<>();
    final RelBuilder builder = HiveRelFactories.HIVE_BUILDER.create(aggregate.getCluster(), null);
    final RexBuilder rexBuilder = aggregate.getCluster().getRexBuilder();
    final List<AggregateCall> aggregateCalls = new ArrayList<>(aggregate.getAggCallList());
    aggregateCalls.remove(aggregateGroupingIdIndex - aggregate.getGroupCount());
    for (ImmutableBitSet targetGroupSet : aggregate.getGroupSets()) {
        // Compute the grouping id value
        long groupingIdValue = convert(targetGroupSet, aggregate.getGroupSet());
        // First we modify the MV query
        Aggregate newAggregate = aggregate.copy(aggregate.getTraitSet(), aggregate.getInput(), targetGroupSet, null, aggregateCalls);
        builder.push(newAggregate);
        List<RexNode> exprs = new ArrayList<>();
        for (int pos = 0; pos < aggregate.getGroupCount(); pos++) {
            int ref = aggregate.getGroupSet().nth(pos);
            if (targetGroupSet.get(ref)) {
                exprs.add(rexBuilder.makeInputRef(newAggregate, targetGroupSet.indexOf(ref)));
            } else {
                exprs.add(rexBuilder.makeNullLiteral(aggregate.getRowType().getFieldList().get(pos).getType()));
            }
        }
        int pos = targetGroupSet.cardinality();
        for (AggregateCall aggregateCall : aggregate.getAggCallList()) {
            if (aggregateCall.getAggregation() == HiveGroupingID.INSTANCE) {
                exprs.add(rexBuilder.makeBigintLiteral(new BigDecimal(groupingIdValue)));
            } else {
                exprs.add(rexBuilder.makeInputRef(newAggregate, pos++));
            }
        }
        if (project != null) {
            // Include projections from top operator
            Project bottomProject = (Project) builder.project(exprs, ImmutableList.of(), true).build();
            List<RexNode> newNodes = RelOptUtil.pushPastProject(project.getProjects(), bottomProject);
            builder.push(bottomProject.getInput()).project(newNodes);
        } else {
            builder.project(exprs);
        }
        final RelNode newQueryRel = builder.build();
        // Second we modify the MV scan
        builder.push(materialization.tableRel);
        RexNode condition = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, rexBuilder.makeInputRef(materialization.tableRel, project != null ? projectGroupingIdIndex : aggregateGroupingIdIndex), rexBuilder.makeBigintLiteral(new BigDecimal(groupingIdValue)));
        builder.filter(condition);
        final RelNode newTableRel = builder.build();
        final Table scanTable = extractTable(materialization);
        materializationList.add(new HiveRelOptMaterialization(newTableRel, newQueryRel, null, ImmutableList.of(scanTable.getDbName(), scanTable.getTableName(), "#" + materializationList.size()), materialization.getScope(), materialization.getRebuildMode()));
    }
    return materializationList;
}