Examples with ImmutableBitSet org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.ImmutableBitSet used on opensource projects

Example 16 with ImmutableBitSet

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.ImmutableBitSet in project calcite by apache.

the class RelMdAllPredicates method getAllPredicates.

/**
 * Add the Join condition to the list obtained from the input.
 */
public RelOptPredicateList getAllPredicates(Join join, RelMetadataQuery mq) {
    if (join.getJoinType() != JoinRelType.INNER) {
        // We cannot map origin of this expression.
        return null;
    }
    final RexBuilder rexBuilder = join.getCluster().getRexBuilder();
    final RexNode pred = join.getCondition();
    final Multimap<List<String>, RelTableRef> qualifiedNamesToRefs = HashMultimap.create();
    RelOptPredicateList newPreds = RelOptPredicateList.EMPTY;
    for (RelNode input : join.getInputs()) {
        final RelOptPredicateList inputPreds = mq.getAllPredicates(input);
        if (inputPreds == null) {
            // Bail out
            return null;
        }
        // Gather table references
        final Set<RelTableRef> tableRefs = mq.getTableReferences(input);
        if (input == join.getLeft()) {
            // Left input references remain unchanged
            for (RelTableRef leftRef : tableRefs) {
                qualifiedNamesToRefs.put(leftRef.getQualifiedName(), leftRef);
            }
            newPreds = newPreds.union(rexBuilder, inputPreds);
        } else {
            // Right input references might need to be updated if there are table name
            // clashes with left input
            final Map<RelTableRef, RelTableRef> currentTablesMapping = new HashMap<>();
            for (RelTableRef rightRef : tableRefs) {
                int shift = 0;
                Collection<RelTableRef> lRefs = qualifiedNamesToRefs.get(rightRef.getQualifiedName());
                if (lRefs != null) {
                    shift = lRefs.size();
                }
                currentTablesMapping.put(rightRef, RelTableRef.of(rightRef.getTable(), shift + rightRef.getEntityNumber()));
            }
            final List<RexNode> updatedPreds = Lists.newArrayList(Iterables.transform(inputPreds.pulledUpPredicates, new Function<RexNode, RexNode>() {

                @Override
                public RexNode apply(RexNode e) {
                    return RexUtil.swapTableReferences(rexBuilder, e, currentTablesMapping);
                }
            }));
            newPreds = newPreds.union(rexBuilder, RelOptPredicateList.of(rexBuilder, updatedPreds));
        }
    }
    // Extract input fields referenced by Join condition
    final Set<RelDataTypeField> inputExtraFields = new LinkedHashSet<>();
    final RelOptUtil.InputFinder inputFinder = new RelOptUtil.InputFinder(inputExtraFields);
    pred.accept(inputFinder);
    final ImmutableBitSet inputFieldsUsed = inputFinder.inputBitSet.build();
    // Infer column origin expressions for given references
    final Map<RexInputRef, Set<RexNode>> mapping = new LinkedHashMap<>();
    for (int idx : inputFieldsUsed) {
        final RexInputRef inputRef = RexInputRef.of(idx, join.getRowType().getFieldList());
        final Set<RexNode> originalExprs = mq.getExpressionLineage(join, inputRef);
        if (originalExprs == null) {
            // Bail out
            return null;
        }
        final RexInputRef ref = RexInputRef.of(idx, join.getRowType().getFieldList());
        mapping.put(ref, originalExprs);
    }
    // Replace with new expressions and return union of predicates
    return newPreds.union(rexBuilder, RelOptPredicateList.of(rexBuilder, RelMdExpressionLineage.createAllPossibleExpressions(rexBuilder, pred, mapping)));
}

Example 17 with ImmutableBitSet

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.ImmutableBitSet in project calcite by apache.

the class RelMdAllPredicates method getAllPredicates.

/**
 * Add the Filter condition to the list obtained from the input.
 */
public RelOptPredicateList getAllPredicates(Filter filter, RelMetadataQuery mq) {
    final RelNode input = filter.getInput();
    final RexBuilder rexBuilder = filter.getCluster().getRexBuilder();
    final RexNode pred = filter.getCondition();
    final RelOptPredicateList predsBelow = mq.getAllPredicates(input);
    if (predsBelow == null) {
        // Safety check
        return null;
    }
    // Extract input fields referenced by Filter condition
    final Set<RelDataTypeField> inputExtraFields = new LinkedHashSet<>();
    final RelOptUtil.InputFinder inputFinder = new RelOptUtil.InputFinder(inputExtraFields);
    pred.accept(inputFinder);
    final ImmutableBitSet inputFieldsUsed = inputFinder.inputBitSet.build();
    // Infer column origin expressions for given references
    final Map<RexInputRef, Set<RexNode>> mapping = new LinkedHashMap<>();
    for (int idx : inputFieldsUsed) {
        final RexInputRef ref = RexInputRef.of(idx, filter.getRowType().getFieldList());
        final Set<RexNode> originalExprs = mq.getExpressionLineage(filter, ref);
        if (originalExprs == null) {
            // Bail out
            return null;
        }
        mapping.put(ref, originalExprs);
    }
    // Replace with new expressions and return union of predicates
    return predsBelow.union(rexBuilder, RelOptPredicateList.of(rexBuilder, RelMdExpressionLineage.createAllPossibleExpressions(rexBuilder, pred, mapping)));
}

Example 18 with ImmutableBitSet

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.ImmutableBitSet in project calcite by apache.

the class InduceGroupingTypeTest method testInduceGroupingType0.

@Test
public void testInduceGroupingType0() {
    final ImmutableBitSet groupSet = ImmutableBitSet.of();
    // Could be CUBE or ROLLUP but we choose SIMPLE
    List<ImmutableBitSet> groupSets = Lists.newArrayList();
    groupSets.add(groupSet);
    assertEquals(Aggregate.Group.SIMPLE, Aggregate.Group.induce(groupSet, groupSets));
    groupSets = Lists.newArrayList();
    assertEquals(Aggregate.Group.OTHER, Aggregate.Group.induce(groupSet, groupSets));
}

Example 19 with ImmutableBitSet

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.ImmutableBitSet in project calcite by apache.

the class AggregateExpandDistinctAggregatesRule method onMatch.

// ~ Methods ----------------------------------------------------------------
public void onMatch(RelOptRuleCall call) {
    final Aggregate aggregate = call.rel(0);
    if (!aggregate.containsDistinctCall()) {
        return;
    }
    // Find all of the agg expressions. We use a LinkedHashSet to ensure determinism.
    // find all aggregate calls without distinct
    int nonDistinctAggCallCount = 0;
    int filterCount = 0;
    int unsupportedNonDistinctAggCallCount = 0;
    final Set<Pair<List<Integer>, Integer>> argLists = new LinkedHashSet<>();
    for (AggregateCall aggCall : aggregate.getAggCallList()) {
        if (aggCall.filterArg >= 0) {
            ++filterCount;
        }
        if (!aggCall.isDistinct()) {
            ++nonDistinctAggCallCount;
            final SqlKind aggCallKind = aggCall.getAggregation().getKind();
            // We only support COUNT/SUM/MIN/MAX for the "single" count distinct optimization
            switch(aggCallKind) {
                case COUNT:
                case SUM:
                case SUM0:
                case MIN:
                case MAX:
                    break;
                default:
                    ++unsupportedNonDistinctAggCallCount;
            }
        } else {
            argLists.add(Pair.of(aggCall.getArgList(), aggCall.filterArg));
        }
    }
    final int distinctAggCallCount = aggregate.getAggCallList().size() - nonDistinctAggCallCount;
    Preconditions.checkState(argLists.size() > 0, "containsDistinctCall lied");
    // arguments then we can use a more efficient form.
    if (nonDistinctAggCallCount == 0 && argLists.size() == 1 && aggregate.getGroupType() == Group.SIMPLE) {
        final Pair<List<Integer>, Integer> pair = Iterables.getOnlyElement(argLists);
        final RelBuilder relBuilder = call.builder();
        convertMonopole(relBuilder, aggregate, pair.left, pair.right);
        call.transformTo(relBuilder.build());
        return;
    }
    if (useGroupingSets) {
        rewriteUsingGroupingSets(call, aggregate);
        return;
    }
    // we can generate multi-phase aggregates
    if (// one distinct aggregate
    distinctAggCallCount == 1 && // no filter
    filterCount == 0 && // sum/min/max/count in non-distinct aggregate
    unsupportedNonDistinctAggCallCount == 0 && nonDistinctAggCallCount > 0) {
        // one or more non-distinct aggregates
        final RelBuilder relBuilder = call.builder();
        convertSingletonDistinct(relBuilder, aggregate, argLists);
        call.transformTo(relBuilder.build());
        return;
    }
    // Create a list of the expressions which will yield the final result.
    // Initially, the expressions point to the input field.
    final List<RelDataTypeField> aggFields = aggregate.getRowType().getFieldList();
    final List<RexInputRef> refs = new ArrayList<>();
    final List<String> fieldNames = aggregate.getRowType().getFieldNames();
    final ImmutableBitSet groupSet = aggregate.getGroupSet();
    final int groupAndIndicatorCount = aggregate.getGroupCount() + aggregate.getIndicatorCount();
    for (int i : Util.range(groupAndIndicatorCount)) {
        refs.add(RexInputRef.of(i, aggFields));
    }
    // Aggregate the original relation, including any non-distinct aggregates.
    final List<AggregateCall> newAggCallList = new ArrayList<>();
    int i = -1;
    for (AggregateCall aggCall : aggregate.getAggCallList()) {
        ++i;
        if (aggCall.isDistinct()) {
            refs.add(null);
            continue;
        }
        refs.add(new RexInputRef(groupAndIndicatorCount + newAggCallList.size(), aggFields.get(groupAndIndicatorCount + i).getType()));
        newAggCallList.add(aggCall);
    }
    // In the case where there are no non-distinct aggregates (regardless of
    // whether there are group bys), there's no need to generate the
    // extra aggregate and join.
    final RelBuilder relBuilder = call.builder();
    relBuilder.push(aggregate.getInput());
    int n = 0;
    if (!newAggCallList.isEmpty()) {
        final RelBuilder.GroupKey groupKey = relBuilder.groupKey(groupSet, aggregate.getGroupSets());
        relBuilder.aggregate(groupKey, newAggCallList);
        ++n;
    }
    // set of operands.
    for (Pair<List<Integer>, Integer> argList : argLists) {
        doRewrite(relBuilder, aggregate, n++, argList.left, argList.right, refs);
    }
    relBuilder.project(refs, fieldNames);
    call.transformTo(relBuilder.build());
}

Example 20 with ImmutableBitSet

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.ImmutableBitSet in project calcite by apache.

the class AggregateExpandDistinctAggregatesRule method doRewrite.

/**
 * Converts all distinct aggregate calls to a given set of arguments.
 *
 * <p>This method is called several times, one for each set of arguments.
 * Each time it is called, it generates a JOIN to a new SELECT DISTINCT
 * relational expression, and modifies the set of top-level calls.
 *
 * @param aggregate Original aggregate
 * @param n         Ordinal of this in a join. {@code relBuilder} contains the
 *                  input relational expression (either the original
 *                  aggregate, the output from the previous call to this
 *                  method. {@code n} is 0 if we're converting the
 *                  first distinct aggregate in a query with no non-distinct
 *                  aggregates)
 * @param argList   Arguments to the distinct aggregate function
 * @param filterArg Argument that filters input to aggregate function, or -1
 * @param refs      Array of expressions which will be the projected by the
 *                  result of this rule. Those relating to this arg list will
 *                  be modified  @return Relational expression
 */
private void doRewrite(RelBuilder relBuilder, Aggregate aggregate, int n, List<Integer> argList, int filterArg, List<RexInputRef> refs) {
    final RexBuilder rexBuilder = aggregate.getCluster().getRexBuilder();
    final List<RelDataTypeField> leftFields;
    if (n == 0) {
        leftFields = null;
    } else {
        leftFields = relBuilder.peek().getRowType().getFieldList();
    }
    // Aggregate(
    // child,
    // {COUNT(DISTINCT 1), SUM(DISTINCT 1), SUM(2)})
    // 
    // becomes
    // 
    // Aggregate(
    // Join(
    // child,
    // Aggregate(child, < all columns > {}),
    // INNER,
    // <f2 = f5>))
    // 
    // E.g.
    // SELECT deptno, SUM(DISTINCT sal), COUNT(DISTINCT gender), MAX(age)
    // FROM Emps
    // GROUP BY deptno
    // 
    // becomes
    // 
    // SELECT e.deptno, adsal.sum_sal, adgender.count_gender, e.max_age
    // FROM (
    // SELECT deptno, MAX(age) as max_age
    // FROM Emps GROUP BY deptno) AS e
    // JOIN (
    // SELECT deptno, COUNT(gender) AS count_gender FROM (
    // SELECT DISTINCT deptno, gender FROM Emps) AS dgender
    // GROUP BY deptno) AS adgender
    // ON e.deptno = adgender.deptno
    // JOIN (
    // SELECT deptno, SUM(sal) AS sum_sal FROM (
    // SELECT DISTINCT deptno, sal FROM Emps) AS dsal
    // GROUP BY deptno) AS adsal
    // ON e.deptno = adsal.deptno
    // GROUP BY e.deptno
    // 
    // Note that if a query contains no non-distinct aggregates, then the
    // very first join/group by is omitted.  In the example above, if
    // MAX(age) is removed, then the sub-select of "e" is not needed, and
    // instead the two other group by's are joined to one another.
    // Project the columns of the GROUP BY plus the arguments
    // to the agg function.
    final Map<Integer, Integer> sourceOf = new HashMap<>();
    createSelectDistinct(relBuilder, aggregate, argList, filterArg, sourceOf);
    // Now compute the aggregate functions on top of the distinct dataset.
    // Each distinct agg becomes a non-distinct call to the corresponding
    // field from the right; for example,
    // "COUNT(DISTINCT e.sal)"
    // becomes
    // "COUNT(distinct_e.sal)".
    final List<AggregateCall> aggCallList = new ArrayList<>();
    final List<AggregateCall> aggCalls = aggregate.getAggCallList();
    final int groupAndIndicatorCount = aggregate.getGroupCount() + aggregate.getIndicatorCount();
    int i = groupAndIndicatorCount - 1;
    for (AggregateCall aggCall : aggCalls) {
        ++i;
        // COUNT(DISTINCT gender) or SUM(sal).
        if (!aggCall.isDistinct()) {
            continue;
        }
        if (!aggCall.getArgList().equals(argList)) {
            continue;
        }
        // Re-map arguments.
        final int argCount = aggCall.getArgList().size();
        final List<Integer> newArgs = new ArrayList<>(argCount);
        for (int j = 0; j < argCount; j++) {
            final Integer arg = aggCall.getArgList().get(j);
            newArgs.add(sourceOf.get(arg));
        }
        final int newFilterArg = aggCall.filterArg >= 0 ? sourceOf.get(aggCall.filterArg) : -1;
        final AggregateCall newAggCall = AggregateCall.create(aggCall.getAggregation(), false, aggCall.isApproximate(), newArgs, newFilterArg, aggCall.getType(), aggCall.getName());
        assert refs.get(i) == null;
        if (n == 0) {
            refs.set(i, new RexInputRef(groupAndIndicatorCount + aggCallList.size(), newAggCall.getType()));
        } else {
            refs.set(i, new RexInputRef(leftFields.size() + groupAndIndicatorCount + aggCallList.size(), newAggCall.getType()));
        }
        aggCallList.add(newAggCall);
    }
    final Map<Integer, Integer> map = new HashMap<>();
    for (Integer key : aggregate.getGroupSet()) {
        map.put(key, map.size());
    }
    final ImmutableBitSet newGroupSet = aggregate.getGroupSet().permute(map);
    assert newGroupSet.equals(ImmutableBitSet.range(aggregate.getGroupSet().cardinality()));
    ImmutableList<ImmutableBitSet> newGroupingSets = null;
    if (aggregate.indicator) {
        newGroupingSets = ImmutableBitSet.ORDERING.immutableSortedCopy(ImmutableBitSet.permute(aggregate.getGroupSets(), map));
    }
    relBuilder.push(aggregate.copy(aggregate.getTraitSet(), relBuilder.build(), aggregate.indicator, newGroupSet, newGroupingSets, aggCallList));
    // If there's no left child yet, no need to create the join
    if (n == 0) {
        return;
    }
    // Create the join condition. It is of the form
    // 'left.f0 = right.f0 and left.f1 = right.f1 and ...'
    // where {f0, f1, ...} are the GROUP BY fields.
    final List<RelDataTypeField> distinctFields = relBuilder.peek().getRowType().getFieldList();
    final List<RexNode> conditions = Lists.newArrayList();
    for (i = 0; i < groupAndIndicatorCount; ++i) {
        // null values form its own group
        // use "is not distinct from" so that the join condition
        // allows null values to match.
        conditions.add(rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_DISTINCT_FROM, RexInputRef.of(i, leftFields), new RexInputRef(leftFields.size() + i, distinctFields.get(i).getType())));
    }
    // Join in the new 'select distinct' relation.
    relBuilder.join(JoinRelType.INNER, conditions);
}