Examples with ImmutableBitSet org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.ImmutableBitSet used on opensource projects

Example 26 with ImmutableBitSet

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.ImmutableBitSet in project calcite by apache.

the class LoptOptimizeJoinRule method addFilters.

/**
 * Determines which join filters can be added to the current join tree. Note
 * that the join filter still reflects the original join ordering. It will
 * only be adjusted to reflect the new join ordering if the "adjust"
 * parameter is set to true.
 *
 * @param multiJoin join factors being optimized
 * @param leftTree left subtree of the join tree
 * @param leftIdx if ≥ 0, only consider filters that reference leftIdx in
 * leftTree; otherwise, consider all filters that reference any factor in
 * leftTree
 * @param rightTree right subtree of the join tree
 * @param filtersToAdd remaining join filters that need to be added; those
 * that are added are removed from the list
 * @param adjust if true, adjust filter to reflect new join ordering
 *
 * @return AND'd expression of the join filters that can be added to the
 * current join tree
 */
private RexNode addFilters(LoptMultiJoin multiJoin, LoptJoinTree leftTree, int leftIdx, LoptJoinTree rightTree, List<RexNode> filtersToAdd, boolean adjust) {
    // loop through the remaining filters to be added and pick out the
    // ones that reference only the factors in the new join tree
    final RexBuilder rexBuilder = multiJoin.getMultiJoinRel().getCluster().getRexBuilder();
    final ImmutableBitSet.Builder childFactorBuilder = ImmutableBitSet.builder();
    childFactorBuilder.addAll(rightTree.getTreeOrder());
    if (leftIdx >= 0) {
        childFactorBuilder.set(leftIdx);
    } else {
        childFactorBuilder.addAll(leftTree.getTreeOrder());
    }
    for (int child : rightTree.getTreeOrder()) {
        childFactorBuilder.set(child);
    }
    final ImmutableBitSet childFactor = childFactorBuilder.build();
    RexNode condition = null;
    final ListIterator<RexNode> filterIter = filtersToAdd.listIterator();
    while (filterIter.hasNext()) {
        RexNode joinFilter = filterIter.next();
        ImmutableBitSet filterBitmap = multiJoin.getFactorsRefByJoinFilter(joinFilter);
        // AND the filter to the current join condition
        if (childFactor.contains(filterBitmap)) {
            if (condition == null) {
                condition = joinFilter;
            } else {
                condition = rexBuilder.makeCall(SqlStdOperatorTable.AND, condition, joinFilter);
            }
            filterIter.remove();
        }
    }
    if (adjust && (condition != null)) {
        int[] adjustments = new int[multiJoin.getNumTotalFields()];
        if (needsAdjustment(multiJoin, adjustments, leftTree, rightTree, false)) {
            condition = condition.accept(new RelOptUtil.RexInputConverter(rexBuilder, multiJoin.getMultiJoinFields(), leftTree.getJoinTree().getRowType().getFieldList(), rightTree.getJoinTree().getRowType().getFieldList(), adjustments));
        }
    }
    if (condition == null) {
        condition = rexBuilder.makeLiteral(true);
    }
    return condition;
}

Example 27 with ImmutableBitSet

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.ImmutableBitSet in project calcite by apache.

the class LoptOptimizeJoinRule method findRemovableSelfJoins.

/**
 * Locates pairs of joins that are self-joins where the join can be removed
 * because the join condition between the two factors is an equality join on
 * unique keys.
 *
 * @param multiJoin join factors being optimized
 */
private void findRemovableSelfJoins(RelMetadataQuery mq, LoptMultiJoin multiJoin) {
    // Candidates for self-joins must be simple factors
    Map<Integer, RelOptTable> simpleFactors = getSimpleFactors(mq, multiJoin);
    // See if a simple factor is repeated and therefore potentially is
    // part of a self-join.  Restrict each factor to at most one
    // self-join.
    final List<RelOptTable> repeatedTables = new ArrayList<>();
    final TreeSet<Integer> sortedFactors = new TreeSet<>();
    sortedFactors.addAll(simpleFactors.keySet());
    final Map<Integer, Integer> selfJoinPairs = new HashMap<>();
    Integer[] factors = sortedFactors.toArray(new Integer[sortedFactors.size()]);
    for (int i = 0; i < factors.length; i++) {
        if (repeatedTables.contains(simpleFactors.get(factors[i]))) {
            continue;
        }
        for (int j = i + 1; j < factors.length; j++) {
            int leftFactor = factors[i];
            int rightFactor = factors[j];
            if (simpleFactors.get(leftFactor).getQualifiedName().equals(simpleFactors.get(rightFactor).getQualifiedName())) {
                selfJoinPairs.put(leftFactor, rightFactor);
                repeatedTables.add(simpleFactors.get(leftFactor));
                break;
            }
        }
    }
    // allow the join to be removed.
    for (Integer factor1 : selfJoinPairs.keySet()) {
        final int factor2 = selfJoinPairs.get(factor1);
        final List<RexNode> selfJoinFilters = new ArrayList<>();
        for (RexNode filter : multiJoin.getJoinFilters()) {
            ImmutableBitSet joinFactors = multiJoin.getFactorsRefByJoinFilter(filter);
            if ((joinFactors.cardinality() == 2) && joinFactors.get(factor1) && joinFactors.get(factor2)) {
                selfJoinFilters.add(filter);
            }
        }
        if ((selfJoinFilters.size() > 0) && isSelfJoinFilterUnique(mq, multiJoin, factor1, factor2, selfJoinFilters)) {
            multiJoin.addRemovableSelfJoinPair(factor1, factor2);
        }
    }
}

Example 28 with ImmutableBitSet

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.ImmutableBitSet in project calcite by apache.

the class IntersectToDistinctRule method onMatch.

// ~ Methods ----------------------------------------------------------------
public void onMatch(RelOptRuleCall call) {
    final Intersect intersect = call.rel(0);
    if (intersect.all) {
        // nothing we can do
        return;
    }
    final RelOptCluster cluster = intersect.getCluster();
    final RexBuilder rexBuilder = cluster.getRexBuilder();
    final RelBuilder relBuilder = call.builder();
    // 1st level GB: create a GB (col0, col1, count() as c) for each branch
    for (RelNode input : intersect.getInputs()) {
        relBuilder.push(input);
        relBuilder.aggregate(relBuilder.groupKey(relBuilder.fields()), relBuilder.countStar(null));
    }
    // create a union above all the branches
    final int branchCount = intersect.getInputs().size();
    relBuilder.union(true, branchCount);
    final RelNode union = relBuilder.peek();
    // 2nd level GB: create a GB (col0, col1, count(c)) for each branch
    // the index of c is union.getRowType().getFieldList().size() - 1
    final int fieldCount = union.getRowType().getFieldCount();
    final ImmutableBitSet groupSet = ImmutableBitSet.range(fieldCount - 1);
    relBuilder.aggregate(relBuilder.groupKey(groupSet, null), relBuilder.countStar(null));
    // add a filter count(c) = #branches
    relBuilder.filter(relBuilder.equals(relBuilder.field(fieldCount - 1), rexBuilder.makeBigintLiteral(new BigDecimal(branchCount))));
    // Project all but the last field
    relBuilder.project(Util.skipLast(relBuilder.fields()));
    // the schema for intersect distinct is like this
    // R3 on all attributes + count(c) as cnt
    // finally add a project to project out the last column
    call.transformTo(relBuilder.build());
}

Example 29 with ImmutableBitSet

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.ImmutableBitSet in project calcite by apache.

the class JoinAssociateRule method onMatch.

// ~ Methods ----------------------------------------------------------------
public void onMatch(final RelOptRuleCall call) {
    final Join topJoin = call.rel(0);
    final Join bottomJoin = call.rel(1);
    final RelNode relA = bottomJoin.getLeft();
    final RelNode relB = bottomJoin.getRight();
    final RelSubset relC = call.rel(2);
    final RelOptCluster cluster = topJoin.getCluster();
    final RexBuilder rexBuilder = cluster.getRexBuilder();
    if (relC.getConvention() != relA.getConvention()) {
        // EnumerableConvention, we're only interested in enumerable subsets.
        return;
    }
    // topJoin
    // /     \
    // bottomJoin  C
    // /    \
    // A      B
    final int aCount = relA.getRowType().getFieldCount();
    final int bCount = relB.getRowType().getFieldCount();
    final int cCount = relC.getRowType().getFieldCount();
    final ImmutableBitSet aBitSet = ImmutableBitSet.range(0, aCount);
    final ImmutableBitSet bBitSet = ImmutableBitSet.range(aCount, aCount + bCount);
    if (!topJoin.getSystemFieldList().isEmpty()) {
        // FIXME Enable this rule for joins with system fields
        return;
    }
    // (Is this too strict?)
    if (topJoin.getJoinType() != JoinRelType.INNER || bottomJoin.getJoinType() != JoinRelType.INNER) {
        return;
    }
    // Goal is to transform to
    // 
    // newTopJoin
    // /     \
    // A   newBottomJoin
    // /    \
    // B      C
    // Split the condition of topJoin and bottomJoin into a conjunctions. A
    // condition can be pushed down if it does not use columns from A.
    final List<RexNode> top = Lists.newArrayList();
    final List<RexNode> bottom = Lists.newArrayList();
    JoinPushThroughJoinRule.split(topJoin.getCondition(), aBitSet, top, bottom);
    JoinPushThroughJoinRule.split(bottomJoin.getCondition(), aBitSet, top, bottom);
    // Mapping for moving conditions from topJoin or bottomJoin to
    // newBottomJoin.
    // target: | B | C      |
    // source: | A       | B | C      |
    final Mappings.TargetMapping bottomMapping = Mappings.createShiftMapping(aCount + bCount + cCount, 0, aCount, bCount, bCount, aCount + bCount, cCount);
    final List<RexNode> newBottomList = Lists.newArrayList();
    new RexPermuteInputsShuttle(bottomMapping, relB, relC).visitList(bottom, newBottomList);
    RexNode newBottomCondition = RexUtil.composeConjunction(rexBuilder, newBottomList, false);
    final Join newBottomJoin = bottomJoin.copy(bottomJoin.getTraitSet(), newBottomCondition, relB, relC, JoinRelType.INNER, false);
    // Condition for newTopJoin consists of pieces from bottomJoin and topJoin.
    // Field ordinals do not need to be changed.
    RexNode newTopCondition = RexUtil.composeConjunction(rexBuilder, top, false);
    final Join newTopJoin = topJoin.copy(topJoin.getTraitSet(), newTopCondition, relA, newBottomJoin, JoinRelType.INNER, false);
    call.transformTo(newTopJoin);
}

Example 30 with ImmutableBitSet

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.ImmutableBitSet in project calcite by apache.

the class LoptSemiJoinOptimizer method removeJoin.

/**
 * Determines whether a join of the dimension table in a semijoin can be
 * removed. It can be if the dimension keys are unique and the only fields
 * referenced from the dimension table are its semijoin keys. The semijoin
 * keys can be mapped to the corresponding keys from the fact table (because
 * of the equality condition associated with the semijoin keys). Therefore,
 * that's why the dimension table can be removed even though those fields
 * are referenced elsewhere in the query tree.
 *
 * @param multiJoin join factors being optimized
 * @param semiJoin semijoin under consideration
 * @param factIdx id of the fact table in the semijoin
 * @param dimIdx id of the dimension table in the semijoin
 */
private void removeJoin(LoptMultiJoin multiJoin, SemiJoin semiJoin, int factIdx, int dimIdx) {
    // no need to proceed any further
    if (multiJoin.getJoinRemovalFactor(dimIdx) != null) {
        return;
    }
    // Check if the semijoin keys corresponding to the dimension table
    // are unique.  The semijoin will filter out the nulls.
    final ImmutableBitSet dimKeys = ImmutableBitSet.of(semiJoin.getRightKeys());
    final RelNode dimRel = multiJoin.getJoinFactor(dimIdx);
    if (!RelMdUtil.areColumnsDefinitelyUniqueWhenNullsFiltered(mq, dimRel, dimKeys)) {
        return;
    }
    // check that the only fields referenced from the dimension table
    // in either its projection or join conditions are the dimension
    // keys
    ImmutableBitSet dimProjRefs = multiJoin.getProjFields(dimIdx);
    if (dimProjRefs == null) {
        int nDimFields = multiJoin.getNumFieldsInJoinFactor(dimIdx);
        dimProjRefs = ImmutableBitSet.range(0, nDimFields);
    }
    if (!dimKeys.contains(dimProjRefs)) {
        return;
    }
    int[] dimJoinRefCounts = multiJoin.getJoinFieldRefCounts(dimIdx);
    for (int i = 0; i < dimJoinRefCounts.length; i++) {
        if (dimJoinRefCounts[i] > 0) {
            if (!dimKeys.get(i)) {
                return;
            }
        }
    }
    // criteria met; keep track of the fact table and the semijoin that
    // allow the join of this dimension table to be removed
    multiJoin.setJoinRemovalFactor(dimIdx, factIdx);
    multiJoin.setJoinRemovalSemiJoin(dimIdx, semiJoin);
    // dimension table doesn't need to use those keys
    if (dimProjRefs.cardinality() != 0) {
        return;
    }
    for (int i = 0; i < dimJoinRefCounts.length; i++) {
        if (dimJoinRefCounts[i] > 1) {
            return;
        } else if (dimJoinRefCounts[i] == 1) {
            if (!dimKeys.get(i)) {
                return;
            }
        }
    }
    int[] factJoinRefCounts = multiJoin.getJoinFieldRefCounts(factIdx);
    for (Integer key : semiJoin.getLeftKeys()) {
        factJoinRefCounts[key]--;
    }
}