Examples with AggregateCall org.apache.calcite.rel.core.AggregateCall used on opensource projects

Example 26 with AggregateCall

use of org.apache.calcite.rel.core.AggregateCall in project hive by apache.

the class HiveAggregateProjectMergeRule method apply.

public static RelNode apply(HiveAggregate aggregate, HiveProject project) {
    final List<Integer> newKeys = Lists.newArrayList();
    final Map<Integer, Integer> map = new HashMap<>();
    for (int key : aggregate.getGroupSet()) {
        final RexNode rex = project.getProjects().get(key);
        if (rex instanceof RexInputRef) {
            final int newKey = ((RexInputRef) rex).getIndex();
            newKeys.add(newKey);
            map.put(key, newKey);
        } else {
            // Cannot handle "GROUP BY expression"
            return null;
        }
    }
    final ImmutableBitSet newGroupSet = aggregate.getGroupSet().permute(map);
    ImmutableList<ImmutableBitSet> newGroupingSets = null;
    if (aggregate.indicator) {
        newGroupingSets = ImmutableBitSet.ORDERING.immutableSortedCopy(ImmutableBitSet.permute(aggregate.getGroupSets(), map));
    }
    final ImmutableList.Builder<AggregateCall> aggCalls = ImmutableList.builder();
    for (AggregateCall aggregateCall : aggregate.getAggCallList()) {
        final ImmutableList.Builder<Integer> newArgs = ImmutableList.builder();
        for (int arg : aggregateCall.getArgList()) {
            final RexNode rex = project.getProjects().get(arg);
            if (rex instanceof RexInputRef) {
                newArgs.add(((RexInputRef) rex).getIndex());
            } else {
                // Cannot handle "AGG(expression)"
                return null;
            }
        }
        final int newFilterArg;
        if (aggregateCall.filterArg >= 0) {
            final RexNode rex = project.getProjects().get(aggregateCall.filterArg);
            if (!(rex instanceof RexInputRef)) {
                return null;
            }
            newFilterArg = ((RexInputRef) rex).getIndex();
        } else {
            newFilterArg = -1;
        }
        aggCalls.add(aggregateCall.copy(newArgs.build(), newFilterArg));
    }
    final Aggregate newAggregate = aggregate.copy(aggregate.getTraitSet(), project.getInput(), aggregate.indicator, newGroupSet, newGroupingSets, aggCalls.build());
    // Add a project if the group set is not in the same order or
    // contains duplicates.
    RelNode rel = newAggregate;
    if (!newKeys.equals(newGroupSet.asList())) {
        final List<Integer> posList = Lists.newArrayList();
        for (int newKey : newKeys) {
            posList.add(newGroupSet.indexOf(newKey));
        }
        if (aggregate.indicator) {
            for (int newKey : newKeys) {
                posList.add(aggregate.getGroupCount() + newGroupSet.indexOf(newKey));
            }
        }
        for (int i = newAggregate.getGroupCount() + newAggregate.getIndicatorCount(); i < newAggregate.getRowType().getFieldCount(); i++) {
            posList.add(i);
        }
        rel = HiveRelOptUtil.createProject(HiveRelFactories.HIVE_BUILDER.create(aggregate.getCluster(), null), rel, posList);
    }
    return rel;
}

Example 27 with AggregateCall

use of org.apache.calcite.rel.core.AggregateCall in project hive by apache.

the class HiveExceptRewriteRule method createFirstGB.

private RelNode createFirstGB(RelNode input, boolean left, RelOptCluster cluster, RexBuilder rexBuilder) throws CalciteSemanticException {
    final List<RexNode> gbChildProjLst = Lists.newArrayList();
    final List<Integer> groupSetPositions = Lists.newArrayList();
    for (int cInd = 0; cInd < input.getRowType().getFieldList().size(); cInd++) {
        gbChildProjLst.add(rexBuilder.makeInputRef(input, cInd));
        groupSetPositions.add(cInd);
    }
    if (left) {
        gbChildProjLst.add(rexBuilder.makeBigintLiteral(new BigDecimal(2)));
    } else {
        gbChildProjLst.add(rexBuilder.makeBigintLiteral(new BigDecimal(1)));
    }
    // also add the last VCol
    groupSetPositions.add(input.getRowType().getFieldList().size());
    // create the project before GB
    RelNode gbInputRel = HiveProject.create(input, gbChildProjLst, null);
    // groupSetPosition includes all the positions
    final ImmutableBitSet groupSet = ImmutableBitSet.of(groupSetPositions);
    List<AggregateCall> aggregateCalls = Lists.newArrayList();
    RelDataType aggFnRetType = TypeConverter.convert(TypeInfoFactory.longTypeInfo, cluster.getTypeFactory());
    AggregateCall aggregateCall = HiveCalciteUtil.createSingleArgAggCall("count", cluster, TypeInfoFactory.longTypeInfo, input.getRowType().getFieldList().size(), aggFnRetType);
    aggregateCalls.add(aggregateCall);
    return new HiveAggregate(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), gbInputRel, false, groupSet, null, aggregateCalls);
}

Example 28 with AggregateCall

use of org.apache.calcite.rel.core.AggregateCall in project hive by apache.

the class HiveExceptRewriteRule method onMatch.

// ~ Methods ----------------------------------------------------------------
public void onMatch(RelOptRuleCall call) {
    final HiveExcept hiveExcept = call.rel(0);
    final RelOptCluster cluster = hiveExcept.getCluster();
    final RexBuilder rexBuilder = cluster.getRexBuilder();
    Builder<RelNode> bldr = new ImmutableList.Builder<RelNode>();
    // branch
    try {
        bldr.add(createFirstGB(hiveExcept.getInputs().get(0), true, cluster, rexBuilder));
        bldr.add(createFirstGB(hiveExcept.getInputs().get(1), false, cluster, rexBuilder));
    } catch (CalciteSemanticException e) {
        LOG.debug(e.toString());
        throw new RuntimeException(e);
    }
    // create a union above all the branches
    // the schema of union looks like this
    // all keys + VCol + c
    HiveRelNode union = new HiveUnion(cluster, TraitsUtil.getDefaultTraitSet(cluster), bldr.build());
    // 2nd level GB: create a GB (all keys + sum(c) as a + sum(VCol*c) as b) for
    // each branch
    final List<RexNode> gbChildProjLst = Lists.newArrayList();
    final List<Integer> groupSetPositions = Lists.newArrayList();
    int unionColumnSize = union.getRowType().getFieldList().size();
    for (int cInd = 0; cInd < unionColumnSize; cInd++) {
        gbChildProjLst.add(rexBuilder.makeInputRef(union, cInd));
        // the last 2 columns are VCol and c
        if (cInd < unionColumnSize - 2) {
            groupSetPositions.add(cInd);
        }
    }
    try {
        gbChildProjLst.add(multiply(rexBuilder.makeInputRef(union, unionColumnSize - 2), rexBuilder.makeInputRef(union, unionColumnSize - 1), cluster, rexBuilder));
    } catch (CalciteSemanticException e) {
        LOG.debug(e.toString());
        throw new RuntimeException(e);
    }
    RelNode gbInputRel = null;
    try {
        // Here we create a project for the following reasons:
        // (1) GBy only accepts arg as a position of the input, however, we need to sum on VCol*c
        // (2) This can better reuse the function createSingleArgAggCall.
        gbInputRel = HiveProject.create(union, gbChildProjLst, null);
    } catch (CalciteSemanticException e) {
        LOG.debug(e.toString());
        throw new RuntimeException(e);
    }
    // gbInputRel's schema is like this
    // all keys + VCol + c + VCol*c
    List<AggregateCall> aggregateCalls = Lists.newArrayList();
    RelDataType aggFnRetType = TypeConverter.convert(TypeInfoFactory.longTypeInfo, cluster.getTypeFactory());
    // sum(c)
    AggregateCall aggregateCall = HiveCalciteUtil.createSingleArgAggCall("sum", cluster, TypeInfoFactory.longTypeInfo, unionColumnSize - 1, aggFnRetType);
    aggregateCalls.add(aggregateCall);
    // sum(VCol*c)
    aggregateCall = HiveCalciteUtil.createSingleArgAggCall("sum", cluster, TypeInfoFactory.longTypeInfo, unionColumnSize, aggFnRetType);
    aggregateCalls.add(aggregateCall);
    final ImmutableBitSet groupSet = ImmutableBitSet.of(groupSetPositions);
    HiveRelNode aggregateRel = new HiveAggregate(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), gbInputRel, false, groupSet, null, aggregateCalls);
    if (!hiveExcept.all) {
        RelNode filterRel = null;
        try {
            filterRel = new HiveFilter(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), aggregateRel, makeFilterExprForExceptDistinct(aggregateRel, unionColumnSize, cluster, rexBuilder));
        } catch (CalciteSemanticException e) {
            LOG.debug(e.toString());
            throw new RuntimeException(e);
        }
        // finally add a project to project out the last 2 columns
        Set<Integer> projectOutColumnPositions = new HashSet<>();
        projectOutColumnPositions.add(filterRel.getRowType().getFieldList().size() - 2);
        projectOutColumnPositions.add(filterRel.getRowType().getFieldList().size() - 1);
        try {
            call.transformTo(HiveCalciteUtil.createProjectWithoutColumn(filterRel, projectOutColumnPositions));
        } catch (CalciteSemanticException e) {
            LOG.debug(e.toString());
            throw new RuntimeException(e);
        }
    } else {
        List<RexNode> originalInputRefs = Lists.transform(aggregateRel.getRowType().getFieldList(), new Function<RelDataTypeField, RexNode>() {

            @Override
            public RexNode apply(RelDataTypeField input) {
                return new RexInputRef(input.getIndex(), input.getType());
            }
        });
        List<RexNode> copyInputRefs = new ArrayList<>();
        try {
            copyInputRefs.add(makeExprForExceptAll(aggregateRel, unionColumnSize, cluster, rexBuilder));
        } catch (CalciteSemanticException e) {
            LOG.debug(e.toString());
            throw new RuntimeException(e);
        }
        for (int i = 0; i < originalInputRefs.size() - 2; i++) {
            copyInputRefs.add(originalInputRefs.get(i));
        }
        RelNode srcRel = null;
        try {
            srcRel = HiveProject.create(aggregateRel, copyInputRefs, null);
            HiveTableFunctionScan udtf = HiveCalciteUtil.createUDTFForSetOp(cluster, srcRel);
            // finally add a project to project out the 1st columns
            Set<Integer> projectOutColumnPositions = new HashSet<>();
            projectOutColumnPositions.add(0);
            call.transformTo(HiveCalciteUtil.createProjectWithoutColumn(udtf, projectOutColumnPositions));
        } catch (SemanticException e) {
            LOG.debug(e.toString());
            throw new RuntimeException(e);
        }
    }
}

Example 29 with AggregateCall

use of org.apache.calcite.rel.core.AggregateCall in project hive by apache.

the class HiveExpandDistinctAggregatesRule method createCount.

/**
   * @param aggr: the original aggregate
   * @param argList: the original argList in aggregate
   * @param cleanArgList: the new argList without duplicates
   * @param map: the mapping from the original argList to the new argList
   * @param sourceOfForCountDistinct: the sorted positions of groupset
   * @return
   * @throws CalciteSemanticException
   */
private RelNode createCount(Aggregate aggr, List<List<Integer>> argList, List<List<Integer>> cleanArgList, Map<Integer, Integer> map, List<Integer> sourceOfForCountDistinct) throws CalciteSemanticException {
    List<RexNode> originalInputRefs = Lists.transform(aggr.getRowType().getFieldList(), new Function<RelDataTypeField, RexNode>() {

        @Override
        public RexNode apply(RelDataTypeField input) {
            return new RexInputRef(input.getIndex(), input.getType());
        }
    });
    final List<RexNode> gbChildProjLst = Lists.newArrayList();
    // for non-singular args, count can include null, i.e. (,) is counted as 1
    for (List<Integer> list : cleanArgList) {
        RexNode condition = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, originalInputRefs.get(originalInputRefs.size() - 1), rexBuilder.makeExactLiteral(new BigDecimal(getGroupingIdValue(list, sourceOfForCountDistinct, aggr.getGroupCount()))));
        if (list.size() == 1) {
            int pos = list.get(0);
            RexNode notNull = rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, originalInputRefs.get(pos));
            condition = rexBuilder.makeCall(SqlStdOperatorTable.AND, condition, notNull);
        }
        RexNode when = rexBuilder.makeCall(SqlStdOperatorTable.CASE, condition, rexBuilder.makeExactLiteral(BigDecimal.ONE), rexBuilder.constantNull());
        gbChildProjLst.add(when);
    }
    // create the project before GB
    RelNode gbInputRel = HiveProject.create(aggr, gbChildProjLst, null);
    // create the aggregate
    List<AggregateCall> aggregateCalls = Lists.newArrayList();
    RelDataType aggFnRetType = TypeConverter.convert(TypeInfoFactory.longTypeInfo, cluster.getTypeFactory());
    for (int i = 0; i < cleanArgList.size(); i++) {
        AggregateCall aggregateCall = HiveCalciteUtil.createSingleArgAggCall("count", cluster, TypeInfoFactory.longTypeInfo, i, aggFnRetType);
        aggregateCalls.add(aggregateCall);
    }
    Aggregate aggregate = new HiveAggregate(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), gbInputRel, false, ImmutableBitSet.of(), null, aggregateCalls);
    // count(distinct x, y), count(distinct y, x), we find the correct mapping.
    if (map.isEmpty()) {
        return aggregate;
    } else {
        List<RexNode> originalAggrRefs = Lists.transform(aggregate.getRowType().getFieldList(), new Function<RelDataTypeField, RexNode>() {

            @Override
            public RexNode apply(RelDataTypeField input) {
                return new RexInputRef(input.getIndex(), input.getType());
            }
        });
        final List<RexNode> projLst = Lists.newArrayList();
        int index = 0;
        for (int i = 0; i < argList.size(); i++) {
            if (map.containsKey(i)) {
                projLst.add(originalAggrRefs.get(map.get(i)));
            } else {
                projLst.add(originalAggrRefs.get(index++));
            }
        }
        return HiveProject.create(aggregate, projLst, null);
    }
}

Example 30 with AggregateCall

use of org.apache.calcite.rel.core.AggregateCall in project hive by apache.

the class HiveExpandDistinctAggregatesRule method onMatch.

//~ Methods ----------------------------------------------------------------
@Override
public void onMatch(RelOptRuleCall call) {
    final Aggregate aggregate = call.rel(0);
    int numCountDistinct = getNumCountDistinctCall(aggregate);
    if (numCountDistinct == 0) {
        return;
    }
    // Find all of the agg expressions. We use a List (for all count(distinct))
    // as well as a Set (for all others) to ensure determinism.
    int nonDistinctCount = 0;
    List<List<Integer>> argListList = new ArrayList<List<Integer>>();
    Set<List<Integer>> argListSets = new LinkedHashSet<List<Integer>>();
    Set<Integer> positions = new HashSet<>();
    for (AggregateCall aggCall : aggregate.getAggCallList()) {
        if (!aggCall.isDistinct()) {
            ++nonDistinctCount;
            continue;
        }
        ArrayList<Integer> argList = new ArrayList<Integer>();
        for (Integer arg : aggCall.getArgList()) {
            argList.add(arg);
            positions.add(arg);
        }
        // Aggr checks for sorted argList.
        argListList.add(argList);
        argListSets.add(argList);
    }
    Util.permAssert(argListSets.size() > 0, "containsDistinctCall lied");
    if (numCountDistinct > 1 && numCountDistinct == aggregate.getAggCallList().size() && aggregate.getGroupSet().isEmpty()) {
        LOG.debug("Trigger countDistinct rewrite. numCountDistinct is " + numCountDistinct);
        // now positions contains all the distinct positions, i.e., $5, $4, $6
        // we need to first sort them as group by set
        // and then get their position later, i.e., $4->1, $5->2, $6->3
        cluster = aggregate.getCluster();
        rexBuilder = cluster.getRexBuilder();
        RelNode converted = null;
        List<Integer> sourceOfForCountDistinct = new ArrayList<>();
        sourceOfForCountDistinct.addAll(positions);
        Collections.sort(sourceOfForCountDistinct);
        try {
            converted = convert(aggregate, argListList, sourceOfForCountDistinct);
        } catch (CalciteSemanticException e) {
            LOG.debug(e.toString());
            throw new RuntimeException(e);
        }
        call.transformTo(converted);
        return;
    }
    // arguments then we can use a more efficient form.
    if ((nonDistinctCount == 0) && (argListSets.size() == 1)) {
        for (Integer arg : argListSets.iterator().next()) {
            Set<RelColumnOrigin> colOrigs = RelMetadataQuery.instance().getColumnOrigins(aggregate, arg);
            if (null != colOrigs) {
                for (RelColumnOrigin colOrig : colOrigs) {
                    RelOptHiveTable hiveTbl = (RelOptHiveTable) colOrig.getOriginTable();
                    if (hiveTbl.getPartColInfoMap().containsKey(colOrig.getOriginColumnOrdinal())) {
                        // Encountered partitioning column, this will be better handled by MetadataOnly optimizer.
                        return;
                    }
                }
            }
        }
        RelNode converted = convertMonopole(aggregate, argListSets.iterator().next());
        call.transformTo(converted);
        return;
    }
}