Examples with RelDataTypeField org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.type.RelDataTypeField used on opensource projects

Example 46 with RelDataTypeField

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.type.RelDataTypeField in project calcite by apache.

the class AggregateExpandDistinctAggregatesRule method doRewrite.

/**
 * Converts all distinct aggregate calls to a given set of arguments.
 *
 * <p>This method is called several times, one for each set of arguments.
 * Each time it is called, it generates a JOIN to a new SELECT DISTINCT
 * relational expression, and modifies the set of top-level calls.
 *
 * @param aggregate Original aggregate
 * @param n         Ordinal of this in a join. {@code relBuilder} contains the
 *                  input relational expression (either the original
 *                  aggregate, the output from the previous call to this
 *                  method. {@code n} is 0 if we're converting the
 *                  first distinct aggregate in a query with no non-distinct
 *                  aggregates)
 * @param argList   Arguments to the distinct aggregate function
 * @param filterArg Argument that filters input to aggregate function, or -1
 * @param refs      Array of expressions which will be the projected by the
 *                  result of this rule. Those relating to this arg list will
 *                  be modified  @return Relational expression
 */
private void doRewrite(RelBuilder relBuilder, Aggregate aggregate, int n, List<Integer> argList, int filterArg, List<RexInputRef> refs) {
    final RexBuilder rexBuilder = aggregate.getCluster().getRexBuilder();
    final List<RelDataTypeField> leftFields;
    if (n == 0) {
        leftFields = null;
    } else {
        leftFields = relBuilder.peek().getRowType().getFieldList();
    }
    // Aggregate(
    // child,
    // {COUNT(DISTINCT 1), SUM(DISTINCT 1), SUM(2)})
    // 
    // becomes
    // 
    // Aggregate(
    // Join(
    // child,
    // Aggregate(child, < all columns > {}),
    // INNER,
    // <f2 = f5>))
    // 
    // E.g.
    // SELECT deptno, SUM(DISTINCT sal), COUNT(DISTINCT gender), MAX(age)
    // FROM Emps
    // GROUP BY deptno
    // 
    // becomes
    // 
    // SELECT e.deptno, adsal.sum_sal, adgender.count_gender, e.max_age
    // FROM (
    // SELECT deptno, MAX(age) as max_age
    // FROM Emps GROUP BY deptno) AS e
    // JOIN (
    // SELECT deptno, COUNT(gender) AS count_gender FROM (
    // SELECT DISTINCT deptno, gender FROM Emps) AS dgender
    // GROUP BY deptno) AS adgender
    // ON e.deptno = adgender.deptno
    // JOIN (
    // SELECT deptno, SUM(sal) AS sum_sal FROM (
    // SELECT DISTINCT deptno, sal FROM Emps) AS dsal
    // GROUP BY deptno) AS adsal
    // ON e.deptno = adsal.deptno
    // GROUP BY e.deptno
    // 
    // Note that if a query contains no non-distinct aggregates, then the
    // very first join/group by is omitted.  In the example above, if
    // MAX(age) is removed, then the sub-select of "e" is not needed, and
    // instead the two other group by's are joined to one another.
    // Project the columns of the GROUP BY plus the arguments
    // to the agg function.
    final Map<Integer, Integer> sourceOf = new HashMap<>();
    createSelectDistinct(relBuilder, aggregate, argList, filterArg, sourceOf);
    // Now compute the aggregate functions on top of the distinct dataset.
    // Each distinct agg becomes a non-distinct call to the corresponding
    // field from the right; for example,
    // "COUNT(DISTINCT e.sal)"
    // becomes
    // "COUNT(distinct_e.sal)".
    final List<AggregateCall> aggCallList = new ArrayList<>();
    final List<AggregateCall> aggCalls = aggregate.getAggCallList();
    final int groupAndIndicatorCount = aggregate.getGroupCount() + aggregate.getIndicatorCount();
    int i = groupAndIndicatorCount - 1;
    for (AggregateCall aggCall : aggCalls) {
        ++i;
        // COUNT(DISTINCT gender) or SUM(sal).
        if (!aggCall.isDistinct()) {
            continue;
        }
        if (!aggCall.getArgList().equals(argList)) {
            continue;
        }
        // Re-map arguments.
        final int argCount = aggCall.getArgList().size();
        final List<Integer> newArgs = new ArrayList<>(argCount);
        for (int j = 0; j < argCount; j++) {
            final Integer arg = aggCall.getArgList().get(j);
            newArgs.add(sourceOf.get(arg));
        }
        final int newFilterArg = aggCall.filterArg >= 0 ? sourceOf.get(aggCall.filterArg) : -1;
        final AggregateCall newAggCall = AggregateCall.create(aggCall.getAggregation(), false, aggCall.isApproximate(), newArgs, newFilterArg, aggCall.getType(), aggCall.getName());
        assert refs.get(i) == null;
        if (n == 0) {
            refs.set(i, new RexInputRef(groupAndIndicatorCount + aggCallList.size(), newAggCall.getType()));
        } else {
            refs.set(i, new RexInputRef(leftFields.size() + groupAndIndicatorCount + aggCallList.size(), newAggCall.getType()));
        }
        aggCallList.add(newAggCall);
    }
    final Map<Integer, Integer> map = new HashMap<>();
    for (Integer key : aggregate.getGroupSet()) {
        map.put(key, map.size());
    }
    final ImmutableBitSet newGroupSet = aggregate.getGroupSet().permute(map);
    assert newGroupSet.equals(ImmutableBitSet.range(aggregate.getGroupSet().cardinality()));
    ImmutableList<ImmutableBitSet> newGroupingSets = null;
    if (aggregate.indicator) {
        newGroupingSets = ImmutableBitSet.ORDERING.immutableSortedCopy(ImmutableBitSet.permute(aggregate.getGroupSets(), map));
    }
    relBuilder.push(aggregate.copy(aggregate.getTraitSet(), relBuilder.build(), aggregate.indicator, newGroupSet, newGroupingSets, aggCallList));
    // If there's no left child yet, no need to create the join
    if (n == 0) {
        return;
    }
    // Create the join condition. It is of the form
    // 'left.f0 = right.f0 and left.f1 = right.f1 and ...'
    // where {f0, f1, ...} are the GROUP BY fields.
    final List<RelDataTypeField> distinctFields = relBuilder.peek().getRowType().getFieldList();
    final List<RexNode> conditions = Lists.newArrayList();
    for (i = 0; i < groupAndIndicatorCount; ++i) {
        // null values form its own group
        // use "is not distinct from" so that the join condition
        // allows null values to match.
        conditions.add(rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_DISTINCT_FROM, RexInputRef.of(i, leftFields), new RexInputRef(leftFields.size() + i, distinctFields.get(i).getType())));
    }
    // Join in the new 'select distinct' relation.
    relBuilder.join(JoinRelType.INNER, conditions);
}

Example 47 with RelDataTypeField

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.type.RelDataTypeField in project calcite by apache.

the class LoptOptimizeJoinRule method createReplacementSemiJoin.

/**
 * In the event that a dimension table does not need to be joined because of
 * a semijoin, this method creates a join tree that consists of a projection
 * on top of an existing join tree. The existing join tree must contain the
 * fact table in the semijoin that allows the dimension table to be removed.
 *
 * <p>The projection created on top of the join tree mimics a join of the
 * fact and dimension tables. In order for the dimension table to have been
 * removed, the only fields referenced from the dimension table are its
 * dimension keys. Therefore, we can replace these dimension fields with the
 * fields corresponding to the semijoin keys from the fact table in the
 * projection.
 *
 * @param multiJoin join factors being optimized
 * @param semiJoinOpt optimal semijoins for each factor
 * @param factTree existing join tree containing the fact table
 * @param dimIdx dimension table factor id
 * @param filtersToAdd filters remaining to be added; filters added to the
 * new join tree are removed from the list
 *
 * @return created join tree or null if the corresponding fact table has not
 * been joined in yet
 */
private LoptJoinTree createReplacementSemiJoin(RelBuilder relBuilder, LoptMultiJoin multiJoin, LoptSemiJoinOptimizer semiJoinOpt, LoptJoinTree factTree, int dimIdx, List<RexNode> filtersToAdd) {
    // don't bother trying to create the replacement join just yet
    if (factTree == null) {
        return null;
    }
    int factIdx = multiJoin.getJoinRemovalFactor(dimIdx);
    final List<Integer> joinOrder = factTree.getTreeOrder();
    assert joinOrder.contains(factIdx);
    // figure out the position of the fact table in the current jointree
    int adjustment = 0;
    for (Integer factor : joinOrder) {
        if (factor == factIdx) {
            break;
        }
        adjustment += multiJoin.getNumFieldsInJoinFactor(factor);
    }
    // map the dimension keys to the corresponding keys from the fact
    // table, based on the fact table's position in the current jointree
    List<RelDataTypeField> dimFields = multiJoin.getJoinFactor(dimIdx).getRowType().getFieldList();
    int nDimFields = dimFields.size();
    Integer[] replacementKeys = new Integer[nDimFields];
    SemiJoin semiJoin = multiJoin.getJoinRemovalSemiJoin(dimIdx);
    ImmutableIntList dimKeys = semiJoin.getRightKeys();
    ImmutableIntList factKeys = semiJoin.getLeftKeys();
    for (int i = 0; i < dimKeys.size(); i++) {
        replacementKeys[dimKeys.get(i)] = factKeys.get(i) + adjustment;
    }
    return createReplacementJoin(relBuilder, multiJoin, semiJoinOpt, factTree, factIdx, dimIdx, dimKeys, replacementKeys, filtersToAdd);
}

Example 48 with RelDataTypeField

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.type.RelDataTypeField in project calcite by apache.

the class LoptOptimizeJoinRule method createReplacementJoin.

/**
 * Creates a replacement join, projecting either dummy columns or
 * replacement keys from the factor that doesn't actually need to be joined.
 *
 * @param multiJoin join factors being optimized
 * @param semiJoinOpt optimal semijoins for each factor
 * @param currJoinTree current join tree being added to
 * @param leftIdx if ≥ 0, when creating the replacement join, only consider
 * filters that reference leftIdx in currJoinTree; otherwise, consider all
 * filters that reference any factor in currJoinTree
 * @param factorToAdd new factor whose join can be removed
 * @param newKeys join keys that need to be replaced
 * @param replacementKeys the keys that replace the join keys; null if we're
 * removing the null generating factor in an outer join
 * @param filtersToAdd filters remaining to be added; filters added to the
 * new join tree are removed from the list
 *
 * @return created join tree with an appropriate projection for the factor
 * that can be removed
 */
private LoptJoinTree createReplacementJoin(RelBuilder relBuilder, LoptMultiJoin multiJoin, LoptSemiJoinOptimizer semiJoinOpt, LoptJoinTree currJoinTree, int leftIdx, int factorToAdd, ImmutableIntList newKeys, Integer[] replacementKeys, List<RexNode> filtersToAdd) {
    // create a projection, projecting the fields from the join tree
    // containing the current joinRel and the new factor; for fields
    // corresponding to join keys, replace them with the corresponding key
    // from the replacementKeys passed in; for other fields, just create a
    // null expression as a placeholder for the column; this is done so we
    // don't have to adjust the offsets of other expressions that reference
    // the new factor; the placeholder expression values should never be
    // referenced, so that's why it's ok to create these possibly invalid
    // expressions
    RelNode currJoinRel = currJoinTree.getJoinTree();
    List<RelDataTypeField> currFields = currJoinRel.getRowType().getFieldList();
    final int nCurrFields = currFields.size();
    List<RelDataTypeField> newFields = multiJoin.getJoinFactor(factorToAdd).getRowType().getFieldList();
    final int nNewFields = newFields.size();
    List<Pair<RexNode, String>> projects = Lists.newArrayList();
    RexBuilder rexBuilder = currJoinRel.getCluster().getRexBuilder();
    RelDataTypeFactory typeFactory = rexBuilder.getTypeFactory();
    for (int i = 0; i < nCurrFields; i++) {
        projects.add(Pair.of((RexNode) rexBuilder.makeInputRef(currFields.get(i).getType(), i), currFields.get(i).getName()));
    }
    for (int i = 0; i < nNewFields; i++) {
        RexNode projExpr;
        RelDataType newType = newFields.get(i).getType();
        if (!newKeys.contains(i)) {
            if (replacementKeys == null) {
                // null generating factor in an outer join; so make the
                // type nullable
                newType = typeFactory.createTypeWithNullability(newType, true);
            }
            projExpr = rexBuilder.makeCast(newType, rexBuilder.constantNull());
        } else {
            RelDataTypeField mappedField = currFields.get(replacementKeys[i]);
            RexNode mappedInput = rexBuilder.makeInputRef(mappedField.getType(), replacementKeys[i]);
            // if the types aren't the same, create a cast
            if (mappedField.getType() == newType) {
                projExpr = mappedInput;
            } else {
                projExpr = rexBuilder.makeCast(newFields.get(i).getType(), mappedInput);
            }
        }
        projects.add(Pair.of(projExpr, newFields.get(i).getName()));
    }
    relBuilder.push(currJoinRel);
    relBuilder.project(Pair.left(projects), Pair.right(projects));
    // remove the join conditions corresponding to the join we're removing;
    // we don't actually need to use them, but we need to remove them
    // from the list since they're no longer needed
    LoptJoinTree newTree = new LoptJoinTree(semiJoinOpt.getChosenSemiJoin(factorToAdd), factorToAdd);
    addFilters(multiJoin, currJoinTree, leftIdx, newTree, filtersToAdd, false);
    // LogicalFilter placed on top off the projection created above.
    if (leftIdx >= 0) {
        addAdditionalFilters(relBuilder, multiJoin, currJoinTree, newTree, filtersToAdd);
    }
    // from the new factor as we go up in the join tree
    return new LoptJoinTree(relBuilder.build(), currJoinTree.getFactorTree(), newTree.getFactorTree());
}

Example 49 with RelDataTypeField

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.type.RelDataTypeField in project calcite by apache.

the class JoinProjectTransposeRule method onMatch.

// ~ Methods ----------------------------------------------------------------
// implement RelOptRule
public void onMatch(RelOptRuleCall call) {
    Join joinRel = call.rel(0);
    JoinRelType joinType = joinRel.getJoinType();
    Project leftProj;
    Project rightProj;
    RelNode leftJoinChild;
    RelNode rightJoinChild;
    // 2) input's projection doesn't generate nulls
    if (hasLeftChild(call) && (includeOuter || !joinType.generatesNullsOnLeft())) {
        leftProj = call.rel(1);
        leftJoinChild = getProjectChild(call, leftProj, true);
    } else {
        leftProj = null;
        leftJoinChild = call.rel(1);
    }
    if (hasRightChild(call) && (includeOuter || !joinType.generatesNullsOnRight())) {
        rightProj = getRightChild(call);
        rightJoinChild = getProjectChild(call, rightProj, false);
    } else {
        rightProj = null;
        rightJoinChild = joinRel.getRight();
    }
    if ((leftProj == null) && (rightProj == null)) {
        return;
    }
    // Construct two RexPrograms and combine them.  The bottom program
    // is a join of the projection expressions from the left and/or
    // right projects that feed into the join.  The top program contains
    // the join condition.
    // Create a row type representing a concatenation of the inputs
    // underneath the projects that feed into the join.  This is the input
    // into the bottom RexProgram.  Note that the join type is an inner
    // join because the inputs haven't actually been joined yet.
    RelDataType joinChildrenRowType = SqlValidatorUtil.deriveJoinRowType(leftJoinChild.getRowType(), rightJoinChild.getRowType(), JoinRelType.INNER, joinRel.getCluster().getTypeFactory(), null, Collections.<RelDataTypeField>emptyList());
    // Create projection expressions, combining the projection expressions
    // from the projects that feed into the join.  For the RHS projection
    // expressions, shift them to the right by the number of fields on
    // the LHS.  If the join input was not a projection, simply create
    // references to the inputs.
    int nProjExprs = joinRel.getRowType().getFieldCount();
    final List<Pair<RexNode, String>> projects = new ArrayList<>();
    final RexBuilder rexBuilder = joinRel.getCluster().getRexBuilder();
    createProjectExprs(leftProj, leftJoinChild, 0, rexBuilder, joinChildrenRowType.getFieldList(), projects);
    List<RelDataTypeField> leftFields = leftJoinChild.getRowType().getFieldList();
    int nFieldsLeft = leftFields.size();
    createProjectExprs(rightProj, rightJoinChild, nFieldsLeft, rexBuilder, joinChildrenRowType.getFieldList(), projects);
    final List<RelDataType> projTypes = new ArrayList<>();
    for (int i = 0; i < nProjExprs; i++) {
        projTypes.add(projects.get(i).left.getType());
    }
    RelDataType projRowType = rexBuilder.getTypeFactory().createStructType(projTypes, Pair.right(projects));
    // create the RexPrograms and merge them
    RexProgram bottomProgram = RexProgram.create(joinChildrenRowType, Pair.left(projects), null, projRowType, rexBuilder);
    RexProgramBuilder topProgramBuilder = new RexProgramBuilder(projRowType, rexBuilder);
    topProgramBuilder.addIdentity();
    topProgramBuilder.addCondition(joinRel.getCondition());
    RexProgram topProgram = topProgramBuilder.getProgram();
    RexProgram mergedProgram = RexProgramBuilder.mergePrograms(topProgram, bottomProgram, rexBuilder);
    // expand out the join condition and construct a new LogicalJoin that
    // directly references the join children without the intervening
    // ProjectRels
    RexNode newCondition = mergedProgram.expandLocalRef(mergedProgram.getCondition());
    Join newJoinRel = joinRel.copy(joinRel.getTraitSet(), newCondition, leftJoinChild, rightJoinChild, joinRel.getJoinType(), joinRel.isSemiJoinDone());
    // expand out the new projection expressions; if the join is an
    // outer join, modify the expressions to reference the join output
    final List<RexNode> newProjExprs = new ArrayList<>();
    List<RexLocalRef> projList = mergedProgram.getProjectList();
    List<RelDataTypeField> newJoinFields = newJoinRel.getRowType().getFieldList();
    int nJoinFields = newJoinFields.size();
    int[] adjustments = new int[nJoinFields];
    for (int i = 0; i < nProjExprs; i++) {
        RexNode newExpr = mergedProgram.expandLocalRef(projList.get(i));
        if (joinType != JoinRelType.INNER) {
            newExpr = newExpr.accept(new RelOptUtil.RexInputConverter(rexBuilder, joinChildrenRowType.getFieldList(), newJoinFields, adjustments));
        }
        newProjExprs.add(newExpr);
    }
    // finally, create the projection on top of the join
    final RelBuilder relBuilder = call.builder();
    relBuilder.push(newJoinRel);
    relBuilder.project(newProjExprs, joinRel.getRowType().getFieldNames());
    // projection to fix differences wrt nullability of fields
    if (joinType != JoinRelType.INNER) {
        relBuilder.convert(joinRel.getRowType(), false);
    }
    call.transformTo(relBuilder.build());
}

Example 50 with RelDataTypeField

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.type.RelDataTypeField in project calcite by apache.

the class FilterAggregateTransposeRule method onMatch.

// ~ Methods ----------------------------------------------------------------
public void onMatch(RelOptRuleCall call) {
    final Filter filterRel = call.rel(0);
    final Aggregate aggRel = call.rel(1);
    final List<RexNode> conditions = RelOptUtil.conjunctions(filterRel.getCondition());
    final RexBuilder rexBuilder = filterRel.getCluster().getRexBuilder();
    final List<RelDataTypeField> origFields = aggRel.getRowType().getFieldList();
    final int[] adjustments = new int[origFields.size()];
    int j = 0;
    for (int i : aggRel.getGroupSet()) {
        adjustments[j] = i - j;
        j++;
    }
    final List<RexNode> pushedConditions = Lists.newArrayList();
    final List<RexNode> remainingConditions = Lists.newArrayList();
    for (RexNode condition : conditions) {
        ImmutableBitSet rCols = RelOptUtil.InputFinder.bits(condition);
        if (canPush(aggRel, rCols)) {
            pushedConditions.add(condition.accept(new RelOptUtil.RexInputConverter(rexBuilder, origFields, aggRel.getInput(0).getRowType().getFieldList(), adjustments)));
        } else {
            remainingConditions.add(condition);
        }
    }
    final RelBuilder builder = call.builder();
    RelNode rel = builder.push(aggRel.getInput()).filter(pushedConditions).build();
    if (rel == aggRel.getInput(0)) {
        return;
    }
    rel = aggRel.copy(aggRel.getTraitSet(), ImmutableList.of(rel));
    rel = builder.push(rel).filter(remainingConditions).build();
    call.transformTo(rel);
}