Examples with Aggregate org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Aggregate used on opensource projects

Example 31 with Aggregate

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Aggregate in project hive by apache.

the class PlanModifierForASTConv method replaceEmptyGroupAggr.

private static void replaceEmptyGroupAggr(final RelNode rel, RelNode parent) {
    // If this function is called, the parent should only include constant
    List<RexNode> exps = parent instanceof Project ? ((Project) parent).getProjects() : Collections.emptyList();
    for (RexNode rexNode : exps) {
        if (!rexNode.accept(new HiveCalciteUtil.ConstantFinder())) {
            throw new RuntimeException("We expect " + parent.toString() + " to contain only constants. However, " + rexNode.toString() + " is " + rexNode.getKind());
        }
    }
    HiveAggregate oldAggRel = (HiveAggregate) rel;
    RelDataTypeFactory typeFactory = oldAggRel.getCluster().getTypeFactory();
    RelDataType longType = TypeConverter.convert(TypeInfoFactory.longTypeInfo, typeFactory);
    RelDataType intType = TypeConverter.convert(TypeInfoFactory.intTypeInfo, typeFactory);
    // Create the dummy aggregation.
    SqlAggFunction countFn = SqlFunctionConverter.getCalciteAggFn("count", false, ImmutableList.of(intType), longType);
    // TODO: Using 0 might be wrong; might need to walk down to find the
    // proper index of a dummy.
    List<Integer> argList = ImmutableList.of(0);
    AggregateCall dummyCall = new AggregateCall(countFn, false, argList, longType, null);
    Aggregate newAggRel = oldAggRel.copy(oldAggRel.getTraitSet(), oldAggRel.getInput(), oldAggRel.indicator, oldAggRel.getGroupSet(), oldAggRel.getGroupSets(), ImmutableList.of(dummyCall));
    RelNode select = introduceDerivedTable(newAggRel);
    parent.replaceInput(0, select);
}

Example 32 with Aggregate

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Aggregate in project hive by apache.

the class MaterializedViewRewritingRelVisitor method visit.

@Override
public void visit(RelNode node, int ordinal, RelNode parent) {
    if (node instanceof Aggregate) {
        this.containsAggregate = true;
        // Aggregate mode - it should be followed by union
        // that we need to analyze
        RelNode input = node.getInput(0);
        if (input instanceof Union) {
            check((Union) input);
        }
    } else if (node instanceof Union) {
        // Non aggregate mode - analyze union operator
        check((Union) node);
    } else if (node instanceof Project) {
        // Project operator, we can continue
        super.visit(node, ordinal, parent);
    }
    throw new ReturnedValue(false);
}

Example 33 with Aggregate

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Aggregate in project hive by apache.

the class MaterializedViewRewritingRelVisitor method check.

private void check(Union union) {
    // We found the Union
    if (union.getInputs().size() != 2) {
        // Bail out
        throw new ReturnedValue(false);
    }
    // First branch should have the query (with write ID filter conditions)
    new RelVisitor() {

        @Override
        public void visit(RelNode node, int ordinal, RelNode parent) {
            if (node instanceof TableScan || node instanceof Filter || node instanceof Project || node instanceof Join) {
                // We can continue
                super.visit(node, ordinal, parent);
            } else if (node instanceof Aggregate && containsAggregate) {
                Aggregate aggregate = (Aggregate) node;
                for (int i = 0; i < aggregate.getAggCallList().size(); ++i) {
                    AggregateCall aggregateCall = aggregate.getAggCallList().get(i);
                    if (aggregateCall.getAggregation().getKind() == SqlKind.COUNT && aggregateCall.getArgList().size() == 0) {
                        countIndex = i + aggregate.getGroupCount();
                        break;
                    }
                }
                // We can continue
                super.visit(node, ordinal, parent);
            } else {
                throw new ReturnedValue(false);
            }
        }
    }.go(union.getInput(0));
    // Second branch should only have the MV
    new RelVisitor() {

        @Override
        public void visit(RelNode node, int ordinal, RelNode parent) {
            if (node instanceof TableScan) {
                // We can continue
                // TODO: Need to check that this is the same MV that we are rebuilding
                RelOptHiveTable hiveTable = (RelOptHiveTable) node.getTable();
                if (!hiveTable.getHiveTableMD().isMaterializedView()) {
                    // If it is not a materialized view, we do not rewrite it
                    throw new ReturnedValue(false);
                }
                if (containsAggregate && !AcidUtils.isFullAcidTable(hiveTable.getHiveTableMD())) {
                    // we do not rewrite it (we need MERGE support)
                    throw new ReturnedValue(false);
                }
            } else if (node instanceof Project) {
                // We can continue
                super.visit(node, ordinal, parent);
            } else {
                throw new ReturnedValue(false);
            }
        }
    }.go(union.getInput(1));
    // We pass all the checks, we can rewrite
    throw new ReturnedValue(true);
}

Example 34 with Aggregate

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Aggregate in project hive by apache.

the class RelFieldTrimmer method trimFields.

/**
 * Variant of {@link #trimFields(RelNode, ImmutableBitSet, Set)} for
 * {@link org.apache.calcite.rel.logical.LogicalAggregate}.
 */
public TrimResult trimFields(Aggregate aggregate, ImmutableBitSet fieldsUsed, Set<RelDataTypeField> extraFields) {
    // Fields:
    // 
    // | sys fields | group fields | indicator fields | agg functions |
    // 
    // Two kinds of trimming:
    // 
    // 1. If agg rel has system fields but none of these are used, create an
    // agg rel with no system fields.
    // 
    // 2. If aggregate functions are not used, remove them.
    // 
    // But group and indicator fields stay, even if they are not used.
    final RelDataType rowType = aggregate.getRowType();
    // Compute which input fields are used.
    // 1. group fields are always used
    final ImmutableBitSet.Builder inputFieldsUsed = aggregate.getGroupSet().rebuild();
    // 2. agg functions
    for (AggregateCall aggCall : aggregate.getAggCallList()) {
        inputFieldsUsed.addAll(aggCall.getArgList());
        if (aggCall.filterArg >= 0) {
            inputFieldsUsed.set(aggCall.filterArg);
        }
        inputFieldsUsed.addAll(RelCollations.ordinals(aggCall.collation));
    }
    // Create input with trimmed columns.
    final RelNode input = aggregate.getInput();
    final Set<RelDataTypeField> inputExtraFields = Collections.emptySet();
    final TrimResult trimResult = trimChild(aggregate, input, inputFieldsUsed.build(), inputExtraFields);
    final RelNode newInput = trimResult.left;
    final Mapping inputMapping = trimResult.right;
    // We have to return group keys and (if present) indicators.
    // So, pretend that the consumer asked for them.
    final int groupCount = aggregate.getGroupSet().cardinality();
    fieldsUsed = fieldsUsed.union(ImmutableBitSet.range(groupCount));
    // there's nothing to do.
    if (input == newInput && fieldsUsed.equals(ImmutableBitSet.range(rowType.getFieldCount()))) {
        return result(aggregate, Mappings.createIdentity(rowType.getFieldCount()));
    }
    // Which agg calls are used by our consumer?
    int j = groupCount;
    int usedAggCallCount = 0;
    for (int i = 0; i < aggregate.getAggCallList().size(); i++) {
        if (fieldsUsed.get(j++)) {
            ++usedAggCallCount;
        }
    }
    // Offset due to the number of system fields having changed.
    Mapping mapping = Mappings.create(MappingType.INVERSE_SURJECTION, rowType.getFieldCount(), groupCount + usedAggCallCount);
    final ImmutableBitSet newGroupSet = Mappings.apply(inputMapping, aggregate.getGroupSet());
    final ImmutableList<ImmutableBitSet> newGroupSets = ImmutableList.copyOf(Iterables.transform(aggregate.getGroupSets(), input1 -> Mappings.apply(inputMapping, input1)));
    // indicator fields first.
    for (j = 0; j < groupCount; j++) {
        mapping.set(j, j);
    }
    // Now create new agg calls, and populate mapping for them.
    final RelBuilder relBuilder = REL_BUILDER.get();
    relBuilder.push(newInput);
    final List<RelBuilder.AggCall> newAggCallList = new ArrayList<>();
    j = groupCount;
    for (AggregateCall aggCall : aggregate.getAggCallList()) {
        if (fieldsUsed.get(j)) {
            final ImmutableList<RexNode> args = relBuilder.fields(Mappings.apply2(inputMapping, aggCall.getArgList()));
            final RexNode filterArg = aggCall.filterArg < 0 ? null : relBuilder.field(Mappings.apply(inputMapping, aggCall.filterArg));
            RelBuilder.AggCall newAggCall = relBuilder.aggregateCall(aggCall.getAggregation(), args).distinct(aggCall.isDistinct()).filter(filterArg).approximate(aggCall.isApproximate()).sort(relBuilder.fields(aggCall.collation)).as(aggCall.name);
            mapping.set(j, groupCount + newAggCallList.size());
            newAggCallList.add(newAggCall);
        }
        ++j;
    }
    final RelBuilder.GroupKey groupKey = relBuilder.groupKey(newGroupSet, newGroupSets);
    relBuilder.aggregate(groupKey, newAggCallList);
    return result(relBuilder.build(), mapping);
}

Example 35 with Aggregate

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Aggregate in project hive by apache.

the class HiveRemoveSqCountCheck method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    final Join topJoin = call.rel(0);
    final Join join = call.rel(2);
    final Aggregate aggregate = call.rel(6);
    // in presence of grouping sets we can't remove sq_count_check
    if (aggregate.indicator) {
        return;
    }
    if (isAggregateWithoutGbyKeys(aggregate) || isAggWithConstantGbyKeys(aggregate, call)) {
        // join(left, join.getRight)
        RelNode newJoin = HiveJoin.getJoin(topJoin.getCluster(), join.getLeft(), topJoin.getRight(), topJoin.getCondition(), topJoin.getJoinType());
        call.transformTo(newJoin);
    }
}