Examples with Aggregate org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Aggregate used on opensource projects

Example 46 with Aggregate

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Aggregate in project drill by axbaretto.

the class DrillReduceAggregatesRule method onMatch.

@Override
public void onMatch(RelOptRuleCall ruleCall) {
    Aggregate oldAggRel = (Aggregate) ruleCall.rels[0];
    reduceAggs(ruleCall, oldAggRel);
}

Example 47 with Aggregate

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Aggregate in project drill by axbaretto.

the class DrillFilterAggregateTransposeRule method matches.

@Override
public boolean matches(RelOptRuleCall call) {
    final Filter filter = call.rel(0);
    final Aggregate aggregate = call.rel(1);
    return filter.getTraitSet().getTrait(ConventionTraitDef.INSTANCE) == aggregate.getTraitSet().getTrait(ConventionTraitDef.INSTANCE);
}

Example 48 with Aggregate

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Aggregate in project hive by apache.

the class HiveMaterializedViewUtils method deriveGroupingSetsMaterializedViews.

/**
 * If a materialization does not contain grouping sets, it returns the materialization
 * itself. Otherwise, it will create one materialization for each grouping set.
 * For each grouping set, the query for the materialization will consist of the group by
 * columns in the grouping set, followed by a projection to recreate the literal null
 * values. The view scan will consist of the scan over the materialization followed by a
 * filter on the grouping id value corresponding to that grouping set.
 */
public static List<HiveRelOptMaterialization> deriveGroupingSetsMaterializedViews(HiveRelOptMaterialization materialization) {
    final RelNode query = materialization.queryRel;
    final Project project;
    final Aggregate aggregate;
    if (query instanceof Aggregate) {
        project = null;
        aggregate = (Aggregate) query;
    } else if (query instanceof Project && query.getInput(0) instanceof Aggregate) {
        project = (Project) query;
        aggregate = (Aggregate) query.getInput(0);
    } else {
        project = null;
        aggregate = null;
    }
    if (aggregate == null) {
        // Not an aggregate materialized view, return original materialization
        return Collections.singletonList(materialization);
    }
    if (aggregate.getGroupType() == Group.SIMPLE) {
        // Not a grouping sets materialized view, return original materialization
        return Collections.singletonList(materialization);
    }
    int aggregateGroupingIdIndex = -1;
    for (int i = 0; i < aggregate.getAggCallList().size(); i++) {
        if (aggregate.getAggCallList().get(i).getAggregation() == HiveGroupingID.INSTANCE) {
            aggregateGroupingIdIndex = aggregate.getGroupCount() + i;
            break;
        }
    }
    Preconditions.checkState(aggregateGroupingIdIndex != -1);
    int projectGroupingIdIndex = -1;
    if (project != null) {
        for (int i = 0; i < project.getProjects().size(); i++) {
            RexNode expr = project.getProjects().get(i);
            if (expr instanceof RexInputRef) {
                RexInputRef ref = (RexInputRef) expr;
                if (ref.getIndex() == aggregateGroupingIdIndex) {
                    // Grouping id is present
                    projectGroupingIdIndex = i;
                    break;
                }
            }
        }
        if (projectGroupingIdIndex == -1) {
            // Grouping id is not present, return original materialization
            return Collections.singletonList(materialization);
        }
    }
    // Create multiple materializations
    final List<HiveRelOptMaterialization> materializationList = new ArrayList<>();
    final RelBuilder builder = HiveRelFactories.HIVE_BUILDER.create(aggregate.getCluster(), null);
    final RexBuilder rexBuilder = aggregate.getCluster().getRexBuilder();
    final List<AggregateCall> aggregateCalls = new ArrayList<>(aggregate.getAggCallList());
    aggregateCalls.remove(aggregateGroupingIdIndex - aggregate.getGroupCount());
    for (ImmutableBitSet targetGroupSet : aggregate.getGroupSets()) {
        // Compute the grouping id value
        long groupingIdValue = convert(targetGroupSet, aggregate.getGroupSet());
        // First we modify the MV query
        Aggregate newAggregate = aggregate.copy(aggregate.getTraitSet(), aggregate.getInput(), targetGroupSet, null, aggregateCalls);
        builder.push(newAggregate);
        List<RexNode> exprs = new ArrayList<>();
        for (int pos = 0; pos < aggregate.getGroupCount(); pos++) {
            int ref = aggregate.getGroupSet().nth(pos);
            if (targetGroupSet.get(ref)) {
                exprs.add(rexBuilder.makeInputRef(newAggregate, targetGroupSet.indexOf(ref)));
            } else {
                exprs.add(rexBuilder.makeNullLiteral(aggregate.getRowType().getFieldList().get(pos).getType()));
            }
        }
        int pos = targetGroupSet.cardinality();
        for (AggregateCall aggregateCall : aggregate.getAggCallList()) {
            if (aggregateCall.getAggregation() == HiveGroupingID.INSTANCE) {
                exprs.add(rexBuilder.makeBigintLiteral(new BigDecimal(groupingIdValue)));
            } else {
                exprs.add(rexBuilder.makeInputRef(newAggregate, pos++));
            }
        }
        if (project != null) {
            // Include projections from top operator
            Project bottomProject = (Project) builder.project(exprs, ImmutableList.of(), true).build();
            List<RexNode> newNodes = RelOptUtil.pushPastProject(project.getProjects(), bottomProject);
            builder.push(bottomProject.getInput()).project(newNodes);
        } else {
            builder.project(exprs);
        }
        final RelNode newQueryRel = builder.build();
        // Second we modify the MV scan
        builder.push(materialization.tableRel);
        RexNode condition = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, rexBuilder.makeInputRef(materialization.tableRel, project != null ? projectGroupingIdIndex : aggregateGroupingIdIndex), rexBuilder.makeBigintLiteral(new BigDecimal(groupingIdValue)));
        builder.filter(condition);
        final RelNode newTableRel = builder.build();
        final Table scanTable = extractTable(materialization);
        materializationList.add(new HiveRelOptMaterialization(newTableRel, newQueryRel, null, ImmutableList.of(scanTable.getDbName(), scanTable.getTableName(), "#" + materializationList.size()), materialization.getScope(), materialization.getRebuildMode()));
    }
    return materializationList;
}

Example 49 with Aggregate

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Aggregate in project hive by apache.

the class HiveAggregatePartitionIncrementalRewritingRule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    RexBuilder rexBuilder = call.builder().getRexBuilder();
    final Aggregate aggregate = call.rel(0);
    final Union union = call.rel(1);
    final RelNode queryBranch = union.getInput(0);
    final RelNode mvBranch = union.getInput(1);
    // find Partition col indexes in mvBranch top operator row schema
    // mvBranch can be more complex than just a TS on the MV and the partition columns indexes in the top Operator's
    // row schema may differ from the one in the TS row schema. Example:
    // Project($2, $0, $1)
    // TableScan(table=materialized_view1, schema=a, b, part_col)
    RelMetadataQuery relMetadataQuery = RelMetadataQuery.instance();
    int partitionColumnCount = -1;
    List<Integer> partitionColumnIndexes = new ArrayList<>();
    for (int i = 0; i < mvBranch.getRowType().getFieldList().size(); ++i) {
        RelDataTypeField relDataTypeField = mvBranch.getRowType().getFieldList().get(i);
        RexInputRef inputRef = rexBuilder.makeInputRef(relDataTypeField.getType(), i);
        Set<RexNode> expressionLineage = relMetadataQuery.getExpressionLineage(mvBranch, inputRef);
        if (expressionLineage == null || expressionLineage.size() != 1) {
            continue;
        }
        Set<RexTableInputRef> tableInputRefs = findRexTableInputRefs(expressionLineage.iterator().next());
        if (tableInputRefs.size() != 1) {
            continue;
        }
        RexTableInputRef tableInputRef = tableInputRefs.iterator().next();
        RelOptHiveTable relOptHiveTable = (RelOptHiveTable) tableInputRef.getTableRef().getTable();
        if (!(relOptHiveTable.getHiveTableMD().isMaterializedView())) {
            LOG.warn("{} is not a materialized view, bail out.", relOptHiveTable.getQualifiedName());
            return;
        }
        partitionColumnCount = relOptHiveTable.getPartColInfoMap().size();
        if (relOptHiveTable.getPartColInfoMap().containsKey(tableInputRef.getIndex())) {
            partitionColumnIndexes.add(i);
        }
    }
    if (partitionColumnCount <= 0 || partitionColumnIndexes.size() != partitionColumnCount) {
        LOG.debug("Could not find all partition column lineages, bail out.");
        return;
    }
    List<RexNode> joinConjs = new ArrayList<>();
    for (int partColIndex : partitionColumnIndexes) {
        RexNode leftRef = rexBuilder.makeInputRef(mvBranch.getRowType().getFieldList().get(partColIndex).getType(), partColIndex);
        RexNode rightRef = rexBuilder.makeInputRef(queryBranch.getRowType().getFieldList().get(partColIndex).getType(), partColIndex + mvBranch.getRowType().getFieldCount());
        joinConjs.add(rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_DISTINCT_FROM, leftRef, rightRef));
    }
    RexNode joinCond = RexUtil.composeConjunction(rexBuilder, joinConjs);
    RelBuilder builder = call.builder();
    RelNode newNode = builder.push(mvBranch).push(queryBranch).join(JoinRelType.SEMI, joinCond).push(queryBranch).union(union.all).aggregate(builder.groupKey(aggregate.getGroupSet()), aggregate.getAggCallList()).build();
    call.transformTo(newNode);
}

Example 50 with Aggregate

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Aggregate in project hive by apache.

the class HiveAggregateReduceRule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    final RelBuilder relBuilder = call.builder();
    final Aggregate aggRel = (Aggregate) call.rel(0);
    final RexBuilder rexBuilder = aggRel.getCluster().getRexBuilder();
    // We try to rewrite COUNT(x) into COUNT(*) if x is not nullable.
    // We remove duplicate aggregate calls as well.
    boolean rewrite = false;
    boolean identity = true;
    final Map<AggregateCall, Integer> mapping = new HashMap<>();
    final List<Integer> indexes = new ArrayList<>();
    final List<AggregateCall> aggCalls = aggRel.getAggCallList();
    final List<AggregateCall> newAggCalls = new ArrayList<>(aggCalls.size());
    int nextIdx = aggRel.getGroupCount() + aggRel.getIndicatorCount();
    for (int i = 0; i < aggCalls.size(); i++) {
        AggregateCall aggCall = aggCalls.get(i);
        if (aggCall.getAggregation().getKind() == SqlKind.COUNT && !aggCall.isDistinct()) {
            final List<Integer> args = aggCall.getArgList();
            final List<Integer> nullableArgs = new ArrayList<>(args.size());
            for (int arg : args) {
                if (aggRel.getInput().getRowType().getFieldList().get(arg).getType().isNullable()) {
                    nullableArgs.add(arg);
                }
            }
            if (nullableArgs.size() != args.size()) {
                aggCall = aggCall.copy(nullableArgs, aggCall.filterArg);
                rewrite = true;
            }
        }
        Integer idx = mapping.get(aggCall);
        if (idx == null) {
            newAggCalls.add(aggCall);
            idx = nextIdx++;
            mapping.put(aggCall, idx);
        } else {
            rewrite = true;
            identity = false;
        }
        indexes.add(idx);
    }
    if (rewrite) {
        // We trigger the transform
        final Aggregate newAggregate = aggRel.copy(aggRel.getTraitSet(), aggRel.getInput(), aggRel.indicator, aggRel.getGroupSet(), aggRel.getGroupSets(), newAggCalls);
        if (identity) {
            call.transformTo(newAggregate);
        } else {
            final int offset = aggRel.getGroupCount() + aggRel.getIndicatorCount();
            final List<RexNode> projList = Lists.newArrayList();
            for (int i = 0; i < offset; ++i) {
                projList.add(rexBuilder.makeInputRef(aggRel.getRowType().getFieldList().get(i).getType(), i));
            }
            for (int i = offset; i < aggRel.getRowType().getFieldCount(); ++i) {
                projList.add(rexBuilder.makeInputRef(aggRel.getRowType().getFieldList().get(i).getType(), indexes.get(i - offset)));
            }
            call.transformTo(relBuilder.push(newAggregate).project(projList).build());
        }
    }
}