Examples with RexLocalRef org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexLocalRef used on opensource projects

Example 11 with RexLocalRef

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexLocalRef in project beam by apache.

the class BeamIOPushDownRule method isProjectRenameOnlyProgram.

/**
 * Determine whether a program only performs renames and/or projects. RexProgram#isTrivial is not
 * sufficient in this case, because number of projects does not need to be the same as inputs.
 * Calc should NOT be dropped in the following cases:<br>
 * 1. Projected fields are manipulated (ex: 'select field1+10').<br>
 * 2. When the same field projected more than once.<br>
 * 3. When an IO does not supports field reordering and projects fields in a different (from
 * schema) order.
 *
 * @param program A program to check.
 * @param projectReorderingSupported Whether project push-down supports field reordering.
 * @return True when program performs only projects (w/o any modifications), false otherwise.
 */
@VisibleForTesting
boolean isProjectRenameOnlyProgram(RexProgram program, boolean projectReorderingSupported) {
    int fieldCount = program.getInputRowType().getFieldCount();
    Set<Integer> projectIndex = new HashSet<>();
    int previousIndex = -1;
    for (RexLocalRef ref : program.getProjectList()) {
        int index = ref.getIndex();
        if (// Projected values are InputRefs.
        index >= fieldCount || // Each field projected once.
        !projectIndex.add(ref.getIndex()) || (!projectReorderingSupported && index <= previousIndex)) {
            // In the same order.
            return false;
        }
        previousIndex = index;
    }
    return true;
}

Example 12 with RexLocalRef

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexLocalRef in project beam by apache.

the class CalcRelSplitter method execute.

// ~ Methods ----------------------------------------------------------------
public RelNode execute() {
    // expressions to the left.
    assert program.isValid(Litmus.THROW, null);
    final List<RexNode> exprList = program.getExprList();
    final RexNode[] exprs = exprList.toArray(new RexNode[0]);
    assert !RexUtil.containComplexExprs(exprList);
    // Figure out what level each expression belongs to.
    int[] exprLevels = new int[exprs.length];
    // The type of a level is given by
    // relTypes[levelTypeOrdinals[level]].
    int[] levelTypeOrdinals = new int[exprs.length];
    int levelCount = chooseLevels(exprs, -1, exprLevels, levelTypeOrdinals);
    // For each expression, figure out which is the highest level where it
    // is used.
    int[] exprMaxUsingLevelOrdinals = new HighestUsageFinder(exprs, exprLevels).getMaxUsingLevelOrdinals();
    // If expressions are used as outputs, mark them as higher than that.
    final List<RexLocalRef> projectRefList = program.getProjectList();
    final RexLocalRef conditionRef = program.getCondition();
    for (RexLocalRef projectRef : projectRefList) {
        exprMaxUsingLevelOrdinals[projectRef.getIndex()] = levelCount;
    }
    if (conditionRef != null) {
        exprMaxUsingLevelOrdinals[conditionRef.getIndex()] = levelCount;
    }
    // Print out what we've got.
    if (RULE_LOGGER.isTraceEnabled()) {
        traceLevelExpressions(exprs, exprLevels, levelTypeOrdinals, levelCount);
    }
    // Now build the calcs.
    RelNode rel = child;
    final int inputFieldCount = program.getInputRowType().getFieldCount();
    int[] inputExprOrdinals = identityArray(inputFieldCount);
    boolean doneCondition = false;
    for (int level = 0; level < levelCount; level++) {
        final int[] projectExprOrdinals;
        final RelDataType outputRowType;
        if (level == (levelCount - 1)) {
            outputRowType = program.getOutputRowType();
            projectExprOrdinals = new int[projectRefList.size()];
            for (int i = 0; i < projectExprOrdinals.length; i++) {
                projectExprOrdinals[i] = projectRefList.get(i).getIndex();
            }
        } else {
            outputRowType = null;
            // Project the expressions which are computed at this level or
            // before, and will be used at later levels.
            List<Integer> projectExprOrdinalList = new ArrayList<>();
            for (int i = 0; i < exprs.length; i++) {
                RexNode expr = exprs[i];
                if (expr instanceof RexLiteral) {
                    // Don't project literals. They are always created in
                    // the level where they are used.
                    exprLevels[i] = -1;
                    continue;
                }
                if ((exprLevels[i] <= level) && (exprMaxUsingLevelOrdinals[i] > level)) {
                    projectExprOrdinalList.add(i);
                }
            }
            projectExprOrdinals = Ints.toArray(projectExprOrdinalList);
        }
        final RelType relType = relTypes[levelTypeOrdinals[level]];
        // Can we do the condition this level?
        int conditionExprOrdinal = -1;
        if ((conditionRef != null) && !doneCondition) {
            conditionExprOrdinal = conditionRef.getIndex();
            if ((exprLevels[conditionExprOrdinal] > level) || !relType.supportsCondition()) {
                // stand down -- we're not ready to do the condition yet
                conditionExprOrdinal = -1;
            } else {
                doneCondition = true;
            }
        }
        RexProgram program1 = createProgramForLevel(level, levelCount, rel.getRowType(), exprs, exprLevels, inputExprOrdinals, projectExprOrdinals, conditionExprOrdinal, outputRowType);
        rel = relType.makeRel(cluster, traits, relBuilder, rel, program1);
        rel = handle(rel);
        // The outputs of this level will be the inputs to the next level.
        inputExprOrdinals = projectExprOrdinals;
    }
    Preconditions.checkArgument(doneCondition || (conditionRef == null), "unhandled condition");
    return rel;
}

Example 13 with RexLocalRef

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexLocalRef in project beam by apache.

the class CalcRelSplitter method computeTopologicalOrdering.

/**
 * Computes the order in which to visit expressions, so that we decide the level of an expression
 * only after the levels of lower expressions have been decided.
 *
 * <p>First, we need to ensure that an expression is visited after all of its inputs.
 *
 * <p>Further, if the expression is a member of a cohort, we need to visit it after the inputs of
 * all other expressions in that cohort. With this condition, expressions in the same cohort will
 * very likely end up in the same level.
 *
 * <p>Note that if there are no cohorts, the expressions from the {@link RexProgram} are already
 * in a suitable order. We perform the topological sort just to ensure that the code path is
 * well-trodden.
 *
 * @param exprs Expressions
 * @param cohorts List of cohorts, each of which is a set of expr ordinals
 * @return Expression ordinals in topological order
 */
private static List<Integer> computeTopologicalOrdering(RexNode[] exprs, List<Set<Integer>> cohorts) {
    final DirectedGraph<Integer, DefaultEdge> graph = DefaultDirectedGraph.create();
    for (int i = 0; i < exprs.length; i++) {
        graph.addVertex(i);
    }
    for (int i = 0; i < exprs.length; i++) {
        final RexNode expr = exprs[i];
        final Set<Integer> cohort = findCohort(cohorts, i);
        final Set<Integer> targets;
        if (cohort == null) {
            targets = Collections.singleton(i);
        } else {
            targets = cohort;
        }
        expr.accept(new RexVisitorImpl<Void>(true) {

            @Override
            public Void visitLocalRef(RexLocalRef localRef) {
                for (Integer target : targets) {
                    graph.addEdge(localRef.getIndex(), target);
                }
                return null;
            }
        });
    }
    TopologicalOrderIterator<Integer, DefaultEdge> iter = new TopologicalOrderIterator<>(graph);
    final List<Integer> permutation = new ArrayList<>();
    while (iter.hasNext()) {
        permutation.add(iter.next());
    }
    return permutation;
}

Example 14 with RexLocalRef

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexLocalRef in project beam by apache.

the class AbstractBeamCalcRel method estimateFilterSelectivity.

private static double estimateFilterSelectivity(RelNode child, RexProgram program, RelMetadataQuery mq) {
    // Similar to calcite, if the calc node is representing filter operation we estimate the filter
    // selectivity based on the number of equality conditions, number of inequality conditions, ....
    RexLocalRef programCondition = program.getCondition();
    RexNode condition;
    if (programCondition == null) {
        condition = null;
    } else {
        condition = program.expandLocalRef(programCondition);
    }
    // Currently this gets the selectivity based on Calcite's Selectivity Handler (RelMdSelectivity)
    return mq.getSelectivity(child, condition);
}

Example 15 with RexLocalRef

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexLocalRef in project beam by apache.

the class CalcRelSplitter method createProgramForLevel.

/**
 * Creates a program containing the expressions for a given level.
 *
 * <p>The expression list of the program will consist of all entries in the expression list <code>
 * allExprs[i]</code> for which the corresponding level ordinal <code>exprLevels[i]</code> is
 * equal to <code>level</code>. Expressions are mapped according to <code>inputExprOrdinals</code>
 * .
 *
 * @param level Level ordinal
 * @param levelCount Number of levels
 * @param inputRowType Input row type
 * @param allExprs Array of all expressions
 * @param exprLevels Array of the level ordinal of each expression
 * @param inputExprOrdinals Ordinals in the expression list of input expressions. Input expression
 *     <code>i</code> will be found at position <code>inputExprOrdinals[i]</code>.
 * @param projectExprOrdinals Ordinals of the expressions to be output this level.
 * @param conditionExprOrdinal Ordinal of the expression to form the condition for this level, or
 *     -1 if there is no condition.
 * @param outputRowType Output row type
 * @return Relational expression
 */
private RexProgram createProgramForLevel(int level, int levelCount, RelDataType inputRowType, RexNode[] allExprs, int[] exprLevels, int[] inputExprOrdinals, final int[] projectExprOrdinals, int conditionExprOrdinal, @Nullable RelDataType outputRowType) {
    // Build a list of expressions to form the calc.
    List<RexNode> exprs = new ArrayList<>();
    // exprInverseOrdinals describes where an expression in allExprs comes
    // from -- from an input, from a calculated expression, or -1 if not
    // available at this level.
    int[] exprInverseOrdinals = new int[allExprs.length];
    Arrays.fill(exprInverseOrdinals, -1);
    int j = 0;
    // and are used here.
    for (int i = 0; i < inputExprOrdinals.length; i++) {
        final int inputExprOrdinal = inputExprOrdinals[i];
        exprs.add(new RexInputRef(i, allExprs[inputExprOrdinal].getType()));
        exprInverseOrdinals[inputExprOrdinal] = j;
        ++j;
    }
    // Next populate the computed expressions.
    final RexShuttle shuttle = new InputToCommonExprConverter(exprInverseOrdinals, exprLevels, level, inputExprOrdinals, allExprs);
    for (int i = 0; i < allExprs.length; i++) {
        if (exprLevels[i] == level || exprLevels[i] == -1 && level == (levelCount - 1) && allExprs[i] instanceof RexLiteral) {
            RexNode expr = allExprs[i];
            final RexNode translatedExpr = expr.accept(shuttle);
            exprs.add(translatedExpr);
            assert exprInverseOrdinals[i] == -1;
            exprInverseOrdinals[i] = j;
            ++j;
        }
    }
    // Form the projection and condition list. Project and condition
    // ordinals are offsets into allExprs, so we need to map them into
    // exprs.
    final List<RexLocalRef> projectRefs = new ArrayList<>(projectExprOrdinals.length);
    final List<String> fieldNames = new ArrayList<>(projectExprOrdinals.length);
    for (int i = 0; i < projectExprOrdinals.length; i++) {
        final int projectExprOrdinal = projectExprOrdinals[i];
        final int index = exprInverseOrdinals[projectExprOrdinal];
        assert index >= 0;
        RexNode expr = allExprs[projectExprOrdinal];
        projectRefs.add(new RexLocalRef(index, expr.getType()));
        // Inherit meaningful field name if possible.
        fieldNames.add(deriveFieldName(expr, i));
    }
    RexLocalRef conditionRef;
    if (conditionExprOrdinal >= 0) {
        final int index = exprInverseOrdinals[conditionExprOrdinal];
        conditionRef = new RexLocalRef(index, allExprs[conditionExprOrdinal].getType());
    } else {
        conditionRef = null;
    }
    if (outputRowType == null) {
        outputRowType = RexUtil.createStructType(typeFactory, projectRefs, fieldNames, null);
    }
    final RexProgram program = new RexProgram(inputRowType, exprs, projectRefs, conditionRef, outputRowType);
    // call operands), since literals should be inlined.
    return program;
}