Examples with ImmutableBitSet org.apache.calcite.util.ImmutableBitSet used on opensource projects

Example 46 with ImmutableBitSet

use of org.apache.calcite.util.ImmutableBitSet in project calcite by apache.

the class RelMdColumnUniqueness method areColumnsUnique.

public Boolean areColumnsUnique(Project rel, RelMetadataQuery mq, ImmutableBitSet columns, boolean ignoreNulls) {
    // LogicalProject maps a set of rows to a different set;
    // Without knowledge of the mapping function(whether it
    // preserves uniqueness), it is only safe to derive uniqueness
    // info from the child of a project when the mapping is f(a) => a.
    // 
    // Also need to map the input column set to the corresponding child
    // references
    List<RexNode> projExprs = rel.getProjects();
    ImmutableBitSet.Builder childColumns = ImmutableBitSet.builder();
    for (int bit : columns) {
        RexNode projExpr = projExprs.get(bit);
        if (projExpr instanceof RexInputRef) {
            childColumns.set(((RexInputRef) projExpr).getIndex());
        } else if (projExpr instanceof RexCall && ignoreNulls) {
            // If the expression is a cast such that the types are the same
            // except for the nullability, then if we're ignoring nulls,
            // it doesn't matter whether the underlying column reference
            // is nullable.  Check that the types are the same by making a
            // nullable copy of both types and then comparing them.
            RexCall call = (RexCall) projExpr;
            if (call.getOperator() != SqlStdOperatorTable.CAST) {
                continue;
            }
            RexNode castOperand = call.getOperands().get(0);
            if (!(castOperand instanceof RexInputRef)) {
                continue;
            }
            RelDataTypeFactory typeFactory = rel.getCluster().getTypeFactory();
            RelDataType castType = typeFactory.createTypeWithNullability(projExpr.getType(), true);
            RelDataType origType = typeFactory.createTypeWithNullability(castOperand.getType(), true);
            if (castType.equals(origType)) {
                childColumns.set(((RexInputRef) castOperand).getIndex());
            }
        } else {
            // projection, then skip it.
            continue;
        }
    }
    // If no columns can affect uniqueness, then return unknown
    if (childColumns.cardinality() == 0) {
        return null;
    }
    return mq.areColumnsUnique(rel.getInput(), childColumns.build(), ignoreNulls);
}

Example 47 with ImmutableBitSet

use of org.apache.calcite.util.ImmutableBitSet in project calcite by apache.

the class RelMdAllPredicates method getAllPredicates.

/**
 * Add the Filter condition to the list obtained from the input.
 */
public RelOptPredicateList getAllPredicates(Filter filter, RelMetadataQuery mq) {
    final RelNode input = filter.getInput();
    final RexBuilder rexBuilder = filter.getCluster().getRexBuilder();
    final RexNode pred = filter.getCondition();
    final RelOptPredicateList predsBelow = mq.getAllPredicates(input);
    if (predsBelow == null) {
        // Safety check
        return null;
    }
    // Extract input fields referenced by Filter condition
    final Set<RelDataTypeField> inputExtraFields = new LinkedHashSet<>();
    final RelOptUtil.InputFinder inputFinder = new RelOptUtil.InputFinder(inputExtraFields);
    pred.accept(inputFinder);
    final ImmutableBitSet inputFieldsUsed = inputFinder.inputBitSet.build();
    // Infer column origin expressions for given references
    final Map<RexInputRef, Set<RexNode>> mapping = new LinkedHashMap<>();
    for (int idx : inputFieldsUsed) {
        final RexInputRef ref = RexInputRef.of(idx, filter.getRowType().getFieldList());
        final Set<RexNode> originalExprs = mq.getExpressionLineage(filter, ref);
        if (originalExprs == null) {
            // Bail out
            return null;
        }
        mapping.put(ref, originalExprs);
    }
    // Replace with new expressions and return union of predicates
    return predsBelow.union(rexBuilder, RelOptPredicateList.of(rexBuilder, RelMdExpressionLineage.createAllPossibleExpressions(rexBuilder, pred, mapping)));
}

Example 48 with ImmutableBitSet

use of org.apache.calcite.util.ImmutableBitSet in project calcite by apache.

the class AggregateExpandDistinctAggregatesRule method doRewrite.

/**
 * Converts all distinct aggregate calls to a given set of arguments.
 *
 * <p>This method is called several times, one for each set of arguments.
 * Each time it is called, it generates a JOIN to a new SELECT DISTINCT
 * relational expression, and modifies the set of top-level calls.
 *
 * @param aggregate Original aggregate
 * @param n         Ordinal of this in a join. {@code relBuilder} contains the
 *                  input relational expression (either the original
 *                  aggregate, the output from the previous call to this
 *                  method. {@code n} is 0 if we're converting the
 *                  first distinct aggregate in a query with no non-distinct
 *                  aggregates)
 * @param argList   Arguments to the distinct aggregate function
 * @param filterArg Argument that filters input to aggregate function, or -1
 * @param refs      Array of expressions which will be the projected by the
 *                  result of this rule. Those relating to this arg list will
 *                  be modified  @return Relational expression
 */
private void doRewrite(RelBuilder relBuilder, Aggregate aggregate, int n, List<Integer> argList, int filterArg, List<RexInputRef> refs) {
    final RexBuilder rexBuilder = aggregate.getCluster().getRexBuilder();
    final List<RelDataTypeField> leftFields;
    if (n == 0) {
        leftFields = null;
    } else {
        leftFields = relBuilder.peek().getRowType().getFieldList();
    }
    // Aggregate(
    // child,
    // {COUNT(DISTINCT 1), SUM(DISTINCT 1), SUM(2)})
    // 
    // becomes
    // 
    // Aggregate(
    // Join(
    // child,
    // Aggregate(child, < all columns > {}),
    // INNER,
    // <f2 = f5>))
    // 
    // E.g.
    // SELECT deptno, SUM(DISTINCT sal), COUNT(DISTINCT gender), MAX(age)
    // FROM Emps
    // GROUP BY deptno
    // 
    // becomes
    // 
    // SELECT e.deptno, adsal.sum_sal, adgender.count_gender, e.max_age
    // FROM (
    // SELECT deptno, MAX(age) as max_age
    // FROM Emps GROUP BY deptno) AS e
    // JOIN (
    // SELECT deptno, COUNT(gender) AS count_gender FROM (
    // SELECT DISTINCT deptno, gender FROM Emps) AS dgender
    // GROUP BY deptno) AS adgender
    // ON e.deptno = adgender.deptno
    // JOIN (
    // SELECT deptno, SUM(sal) AS sum_sal FROM (
    // SELECT DISTINCT deptno, sal FROM Emps) AS dsal
    // GROUP BY deptno) AS adsal
    // ON e.deptno = adsal.deptno
    // GROUP BY e.deptno
    // 
    // Note that if a query contains no non-distinct aggregates, then the
    // very first join/group by is omitted.  In the example above, if
    // MAX(age) is removed, then the sub-select of "e" is not needed, and
    // instead the two other group by's are joined to one another.
    // Project the columns of the GROUP BY plus the arguments
    // to the agg function.
    final Map<Integer, Integer> sourceOf = new HashMap<>();
    createSelectDistinct(relBuilder, aggregate, argList, filterArg, sourceOf);
    // Now compute the aggregate functions on top of the distinct dataset.
    // Each distinct agg becomes a non-distinct call to the corresponding
    // field from the right; for example,
    // "COUNT(DISTINCT e.sal)"
    // becomes
    // "COUNT(distinct_e.sal)".
    final List<AggregateCall> aggCallList = new ArrayList<>();
    final List<AggregateCall> aggCalls = aggregate.getAggCallList();
    final int groupAndIndicatorCount = aggregate.getGroupCount() + aggregate.getIndicatorCount();
    int i = groupAndIndicatorCount - 1;
    for (AggregateCall aggCall : aggCalls) {
        ++i;
        // COUNT(DISTINCT gender) or SUM(sal).
        if (!aggCall.isDistinct()) {
            continue;
        }
        if (!aggCall.getArgList().equals(argList)) {
            continue;
        }
        // Re-map arguments.
        final int argCount = aggCall.getArgList().size();
        final List<Integer> newArgs = new ArrayList<>(argCount);
        for (int j = 0; j < argCount; j++) {
            final Integer arg = aggCall.getArgList().get(j);
            newArgs.add(sourceOf.get(arg));
        }
        final int newFilterArg = aggCall.filterArg >= 0 ? sourceOf.get(aggCall.filterArg) : -1;
        final AggregateCall newAggCall = AggregateCall.create(aggCall.getAggregation(), false, aggCall.isApproximate(), newArgs, newFilterArg, aggCall.getType(), aggCall.getName());
        assert refs.get(i) == null;
        if (n == 0) {
            refs.set(i, new RexInputRef(groupAndIndicatorCount + aggCallList.size(), newAggCall.getType()));
        } else {
            refs.set(i, new RexInputRef(leftFields.size() + groupAndIndicatorCount + aggCallList.size(), newAggCall.getType()));
        }
        aggCallList.add(newAggCall);
    }
    final Map<Integer, Integer> map = new HashMap<>();
    for (Integer key : aggregate.getGroupSet()) {
        map.put(key, map.size());
    }
    final ImmutableBitSet newGroupSet = aggregate.getGroupSet().permute(map);
    assert newGroupSet.equals(ImmutableBitSet.range(aggregate.getGroupSet().cardinality()));
    ImmutableList<ImmutableBitSet> newGroupingSets = null;
    if (aggregate.indicator) {
        newGroupingSets = ImmutableBitSet.ORDERING.immutableSortedCopy(ImmutableBitSet.permute(aggregate.getGroupSets(), map));
    }
    relBuilder.push(aggregate.copy(aggregate.getTraitSet(), relBuilder.build(), aggregate.indicator, newGroupSet, newGroupingSets, aggCallList));
    // If there's no left child yet, no need to create the join
    if (n == 0) {
        return;
    }
    // Create the join condition. It is of the form
    // 'left.f0 = right.f0 and left.f1 = right.f1 and ...'
    // where {f0, f1, ...} are the GROUP BY fields.
    final List<RelDataTypeField> distinctFields = relBuilder.peek().getRowType().getFieldList();
    final List<RexNode> conditions = Lists.newArrayList();
    for (i = 0; i < groupAndIndicatorCount; ++i) {
        // null values form its own group
        // use "is not distinct from" so that the join condition
        // allows null values to match.
        conditions.add(rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_DISTINCT_FROM, RexInputRef.of(i, leftFields), new RexInputRef(leftFields.size() + i, distinctFields.get(i).getType())));
    }
    // Join in the new 'select distinct' relation.
    relBuilder.join(JoinRelType.INNER, conditions);
}

Example 49 with ImmutableBitSet

use of org.apache.calcite.util.ImmutableBitSet in project calcite by apache.

the class LoptSemiJoinOptimizer method removeJoin.

/**
 * Determines whether a join of the dimension table in a semijoin can be
 * removed. It can be if the dimension keys are unique and the only fields
 * referenced from the dimension table are its semijoin keys. The semijoin
 * keys can be mapped to the corresponding keys from the fact table (because
 * of the equality condition associated with the semijoin keys). Therefore,
 * that's why the dimension table can be removed even though those fields
 * are referenced elsewhere in the query tree.
 *
 * @param multiJoin join factors being optimized
 * @param semiJoin semijoin under consideration
 * @param factIdx id of the fact table in the semijoin
 * @param dimIdx id of the dimension table in the semijoin
 */
private void removeJoin(LoptMultiJoin multiJoin, SemiJoin semiJoin, int factIdx, int dimIdx) {
    // no need to proceed any further
    if (multiJoin.getJoinRemovalFactor(dimIdx) != null) {
        return;
    }
    // Check if the semijoin keys corresponding to the dimension table
    // are unique.  The semijoin will filter out the nulls.
    final ImmutableBitSet dimKeys = ImmutableBitSet.of(semiJoin.getRightKeys());
    final RelNode dimRel = multiJoin.getJoinFactor(dimIdx);
    if (!RelMdUtil.areColumnsDefinitelyUniqueWhenNullsFiltered(mq, dimRel, dimKeys)) {
        return;
    }
    // check that the only fields referenced from the dimension table
    // in either its projection or join conditions are the dimension
    // keys
    ImmutableBitSet dimProjRefs = multiJoin.getProjFields(dimIdx);
    if (dimProjRefs == null) {
        int nDimFields = multiJoin.getNumFieldsInJoinFactor(dimIdx);
        dimProjRefs = ImmutableBitSet.range(0, nDimFields);
    }
    if (!dimKeys.contains(dimProjRefs)) {
        return;
    }
    int[] dimJoinRefCounts = multiJoin.getJoinFieldRefCounts(dimIdx);
    for (int i = 0; i < dimJoinRefCounts.length; i++) {
        if (dimJoinRefCounts[i] > 0) {
            if (!dimKeys.get(i)) {
                return;
            }
        }
    }
    // criteria met; keep track of the fact table and the semijoin that
    // allow the join of this dimension table to be removed
    multiJoin.setJoinRemovalFactor(dimIdx, factIdx);
    multiJoin.setJoinRemovalSemiJoin(dimIdx, semiJoin);
    // dimension table doesn't need to use those keys
    if (dimProjRefs.cardinality() != 0) {
        return;
    }
    for (int i = 0; i < dimJoinRefCounts.length; i++) {
        if (dimJoinRefCounts[i] > 1) {
            return;
        } else if (dimJoinRefCounts[i] == 1) {
            if (!dimKeys.get(i)) {
                return;
            }
        }
    }
    int[] factJoinRefCounts = multiJoin.getJoinFieldRefCounts(factIdx);
    for (Integer key : semiJoin.getLeftKeys()) {
        factJoinRefCounts[key]--;
    }
}

Example 50 with ImmutableBitSet

use of org.apache.calcite.util.ImmutableBitSet in project calcite by apache.

the class RelDecorrelator method decorrelateRel.

/**
 * Rewrites a {@link LogicalAggregate}.
 *
 * @param rel Aggregate to rewrite
 */
public Frame decorrelateRel(LogicalAggregate rel) {
    if (rel.getGroupType() != Aggregate.Group.SIMPLE) {
        throw new AssertionError(Bug.CALCITE_461_FIXED);
    }
    // Aggregate itself should not reference corVars.
    assert !cm.mapRefRelToCorRef.containsKey(rel);
    final RelNode oldInput = rel.getInput();
    final Frame frame = getInvoke(oldInput, rel);
    if (frame == null) {
        // If input has not been rewritten, do not rewrite this rel.
        return null;
    }
    final RelNode newInput = frame.r;
    // map from newInput
    Map<Integer, Integer> mapNewInputToProjOutputs = new HashMap<>();
    final int oldGroupKeyCount = rel.getGroupSet().cardinality();
    // Project projects the original expressions,
    // plus any correlated variables the input wants to pass along.
    final List<Pair<RexNode, String>> projects = Lists.newArrayList();
    List<RelDataTypeField> newInputOutput = newInput.getRowType().getFieldList();
    int newPos = 0;
    // oldInput has the original group by keys in the front.
    final NavigableMap<Integer, RexLiteral> omittedConstants = new TreeMap<>();
    for (int i = 0; i < oldGroupKeyCount; i++) {
        final RexLiteral constant = projectedLiteral(newInput, i);
        if (constant != null) {
            // Exclude constants. Aggregate({true}) occurs because Aggregate({})
            // would generate 1 row even when applied to an empty table.
            omittedConstants.put(i, constant);
            continue;
        }
        int newInputPos = frame.oldToNewOutputs.get(i);
        projects.add(RexInputRef.of2(newInputPos, newInputOutput));
        mapNewInputToProjOutputs.put(newInputPos, newPos);
        newPos++;
    }
    final SortedMap<CorDef, Integer> corDefOutputs = new TreeMap<>();
    if (!frame.corDefOutputs.isEmpty()) {
        // position oldGroupKeyCount.
        for (Map.Entry<CorDef, Integer> entry : frame.corDefOutputs.entrySet()) {
            projects.add(RexInputRef.of2(entry.getValue(), newInputOutput));
            corDefOutputs.put(entry.getKey(), newPos);
            mapNewInputToProjOutputs.put(entry.getValue(), newPos);
            newPos++;
        }
    }
    // add the remaining fields
    final int newGroupKeyCount = newPos;
    for (int i = 0; i < newInputOutput.size(); i++) {
        if (!mapNewInputToProjOutputs.containsKey(i)) {
            projects.add(RexInputRef.of2(i, newInputOutput));
            mapNewInputToProjOutputs.put(i, newPos);
            newPos++;
        }
    }
    assert newPos == newInputOutput.size();
    // This Project will be what the old input maps to,
    // replacing any previous mapping from old input).
    RelNode newProject = relBuilder.push(newInput).projectNamed(Pair.left(projects), Pair.right(projects), true).build();
    // update mappings:
    // oldInput ----> newInput
    // 
    // newProject
    // |
    // oldInput ----> newInput
    // 
    // is transformed to
    // 
    // oldInput ----> newProject
    // |
    // newInput
    Map<Integer, Integer> combinedMap = Maps.newHashMap();
    for (Integer oldInputPos : frame.oldToNewOutputs.keySet()) {
        combinedMap.put(oldInputPos, mapNewInputToProjOutputs.get(frame.oldToNewOutputs.get(oldInputPos)));
    }
    register(oldInput, newProject, combinedMap, corDefOutputs);
    // now it's time to rewrite the Aggregate
    final ImmutableBitSet newGroupSet = ImmutableBitSet.range(newGroupKeyCount);
    List<AggregateCall> newAggCalls = Lists.newArrayList();
    List<AggregateCall> oldAggCalls = rel.getAggCallList();
    int oldInputOutputFieldCount = rel.getGroupSet().cardinality();
    int newInputOutputFieldCount = newGroupSet.cardinality();
    int i = -1;
    for (AggregateCall oldAggCall : oldAggCalls) {
        ++i;
        List<Integer> oldAggArgs = oldAggCall.getArgList();
        List<Integer> aggArgs = Lists.newArrayList();
        // for the argument.
        for (int oldPos : oldAggArgs) {
            aggArgs.add(combinedMap.get(oldPos));
        }
        final int filterArg = oldAggCall.filterArg < 0 ? oldAggCall.filterArg : combinedMap.get(oldAggCall.filterArg);
        newAggCalls.add(oldAggCall.adaptTo(newProject, aggArgs, filterArg, oldGroupKeyCount, newGroupKeyCount));
        // The old to new output position mapping will be the same as that
        // of newProject, plus any aggregates that the oldAgg produces.
        combinedMap.put(oldInputOutputFieldCount + i, newInputOutputFieldCount + i);
    }
    relBuilder.push(LogicalAggregate.create(newProject, newGroupSet, null, newAggCalls));
    if (!omittedConstants.isEmpty()) {
        final List<RexNode> postProjects = new ArrayList<>(relBuilder.fields());
        for (Map.Entry<Integer, RexLiteral> entry : omittedConstants.descendingMap().entrySet()) {
            postProjects.add(entry.getKey() + frame.corDefOutputs.size(), entry.getValue());
        }
        relBuilder.project(postProjects);
    }
    // located at the same position as the input newProject.
    return register(rel, relBuilder.build(), combinedMap, corDefOutputs);
}