Examples with Project org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Project used on opensource projects

Example 51 with Project

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Project in project beam by apache.

the class BeamBasicAggregationRule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    Aggregate aggregate = call.rel(0);
    RelNode relNode = call.rel(1);
    if (aggregate.getGroupType() != Aggregate.Group.SIMPLE) {
        return;
    }
    if (relNode instanceof Project || relNode instanceof Calc || relNode instanceof Filter) {
        if (isWindowed(relNode) || hasWindowedParents(relNode)) {
            // This case is expected to get handled by the 'BeamAggregationRule'
            return;
        }
    }
    RelNode newTableScan = relNode.copy(relNode.getTraitSet(), relNode.getInputs());
    call.transformTo(new BeamAggregationRel(aggregate.getCluster(), aggregate.getTraitSet().replace(BeamLogicalConvention.INSTANCE), convert(newTableScan, newTableScan.getTraitSet().replace(BeamLogicalConvention.INSTANCE)), aggregate.getGroupSet(), aggregate.getGroupSets(), aggregate.getAggCallList(), null, -1));
}

Example 52 with Project

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Project in project beam by apache.

the class BeamIOPushDownRule method canDropCalc.

/**
 * Perform a series of checks to determine whether a Calc can be dropped. Following conditions
 * need to be met in order for that to happen (logical AND):<br>
 * 1. Program should do simple projects, project each field once, and project fields in the same
 * order when field reordering is not supported.<br>
 * 2. Predicate can be completely pushed-down.<br>
 * 3. Project push-down is supported by the IO or all fields are projected by a Calc.
 *
 * @param program A {@code RexProgram} of a {@code Calc}.
 * @param projectSupport An enum containing information about IO project push-down capabilities.
 * @param tableFilter A class containing information about IO predicate push-down capabilities.
 * @return True when Calc can be dropped, false otherwise.
 */
private boolean canDropCalc(RexProgram program, ProjectSupport projectSupport, BeamSqlTableFilter tableFilter) {
    RelDataType calcInputRowType = program.getInputRowType();
    // Program should do simple projects, project each field once, and project fields in the same
    // order when field reordering is not supported.
    boolean fieldReorderingSupported = projectSupport.withFieldReordering();
    if (!isProjectRenameOnlyProgram(program, fieldReorderingSupported)) {
        return false;
    }
    // Predicate can be completely pushed-down
    if (!tableFilter.getNotSupported().isEmpty()) {
        return false;
    }
    // Project push-down is supported by the IO or all fields are projected by a Calc.
    boolean isProjectSupported = projectSupport.isSupported();
    boolean allFieldsProjected = program.getProjectList().stream().map(ref -> program.getInputRowType().getFieldList().get(ref.getIndex()).getName()).collect(Collectors.toList()).equals(calcInputRowType.getFieldNames());
    return isProjectSupported || allFieldsProjected;
}

Example 53 with Project

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Project in project beam by apache.

the class BeamIOPushDownRule method constructNodesWithPushDown.

/**
 * Construct a new {@link BeamIOSourceRel} with predicate and/or project pushed-down and a new
 * {@code Calc} to do field reordering/field duplication/complex projects.
 *
 * @param resolved A descriptor of fields used by a {@code Calc}.
 * @param relBuilder A {@code RelBuilder} for constructing {@code Project} and {@code Filter} Rel
 *     nodes with operations unsupported by the IO.
 * @param ioSourceRel Original {@code BeamIOSourceRel} we are attempting to perform push-down for.
 * @param tableFilter A class containing information about IO predicate push-down capabilities.
 * @param calcDataType A Calcite output schema of an original {@code Calc}.
 * @param calcProjects A list of projected {@code RexNode}s by a {@code Calc}.
 * @return An alternative {@code RelNode} with supported filters/projects pushed-down to IO Rel.
 */
private RelNode constructNodesWithPushDown(FieldAccessDescriptor resolved, RelBuilder relBuilder, BeamIOSourceRel ioSourceRel, BeamSqlTableFilter tableFilter, RelDataType calcDataType, List<RexNode> calcProjects) {
    Schema newSchema = SelectHelpers.getOutputSchema(ioSourceRel.getBeamSqlTable().getSchema(), resolved);
    RelDataType calcInputType = CalciteUtils.toCalciteRowType(newSchema, ioSourceRel.getCluster().getTypeFactory());
    BeamIOSourceRel newIoSourceRel = ioSourceRel.createPushDownRel(calcInputType, newSchema.getFieldNames(), tableFilter);
    relBuilder.push(newIoSourceRel);
    List<RexNode> newProjects = new ArrayList<>();
    List<RexNode> newFilter = new ArrayList<>();
    // Ex: let's say the original fields are (number before each element is the index):
    // {0:unused1, 1:id, 2:name, 3:unused2},
    // where only 'id' and 'name' are being used. Then the new calcInputType should be as follows:
    // {0:id, 1:name}.
    // A mapping list will contain 2 entries: {0:1, 1:2},
    // showing how used field names map to the original fields.
    List<Integer> mapping = resolved.getFieldsAccessed().stream().map(FieldDescriptor::getFieldId).collect(Collectors.toList());
    // Map filters to new RexInputRef.
    for (RexNode filter : tableFilter.getNotSupported()) {
        newFilter.add(reMapRexNodeToNewInputs(filter, mapping));
    }
    // Map projects to new RexInputRef.
    for (RexNode project : calcProjects) {
        newProjects.add(reMapRexNodeToNewInputs(project, mapping));
    }
    if (RexUtil.isIdentity(newProjects, newIoSourceRel.getRowType())) {
        // Force a rename prior to filter for identity function.
        relBuilder.project(newProjects, calcDataType.getFieldNames(), true);
    }
    relBuilder.filter(newFilter);
    relBuilder.project(newProjects, calcDataType.getFieldNames());
    return relBuilder.build();
}

Example 54 with Project

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Project in project beam by apache.

the class BeamIOPushDownRule method isProjectRenameOnlyProgram.

/**
 * Determine whether a program only performs renames and/or projects. RexProgram#isTrivial is not
 * sufficient in this case, because number of projects does not need to be the same as inputs.
 * Calc should NOT be dropped in the following cases:<br>
 * 1. Projected fields are manipulated (ex: 'select field1+10').<br>
 * 2. When the same field projected more than once.<br>
 * 3. When an IO does not supports field reordering and projects fields in a different (from
 * schema) order.
 *
 * @param program A program to check.
 * @param projectReorderingSupported Whether project push-down supports field reordering.
 * @return True when program performs only projects (w/o any modifications), false otherwise.
 */
@VisibleForTesting
boolean isProjectRenameOnlyProgram(RexProgram program, boolean projectReorderingSupported) {
    int fieldCount = program.getInputRowType().getFieldCount();
    Set<Integer> projectIndex = new HashSet<>();
    int previousIndex = -1;
    for (RexLocalRef ref : program.getProjectList()) {
        int index = ref.getIndex();
        if (// Projected values are InputRefs.
        index >= fieldCount || // Each field projected once.
        !projectIndex.add(ref.getIndex()) || (!projectReorderingSupported && index <= previousIndex)) {
            // In the same order.
            return false;
        }
        previousIndex = index;
    }
    return true;
}

Example 55 with Project

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Project in project beam by apache.

the class CalcRelSplitter method execute.

// ~ Methods ----------------------------------------------------------------
public RelNode execute() {
    // expressions to the left.
    assert program.isValid(Litmus.THROW, null);
    final List<RexNode> exprList = program.getExprList();
    final RexNode[] exprs = exprList.toArray(new RexNode[0]);
    assert !RexUtil.containComplexExprs(exprList);
    // Figure out what level each expression belongs to.
    int[] exprLevels = new int[exprs.length];
    // The type of a level is given by
    // relTypes[levelTypeOrdinals[level]].
    int[] levelTypeOrdinals = new int[exprs.length];
    int levelCount = chooseLevels(exprs, -1, exprLevels, levelTypeOrdinals);
    // For each expression, figure out which is the highest level where it
    // is used.
    int[] exprMaxUsingLevelOrdinals = new HighestUsageFinder(exprs, exprLevels).getMaxUsingLevelOrdinals();
    // If expressions are used as outputs, mark them as higher than that.
    final List<RexLocalRef> projectRefList = program.getProjectList();
    final RexLocalRef conditionRef = program.getCondition();
    for (RexLocalRef projectRef : projectRefList) {
        exprMaxUsingLevelOrdinals[projectRef.getIndex()] = levelCount;
    }
    if (conditionRef != null) {
        exprMaxUsingLevelOrdinals[conditionRef.getIndex()] = levelCount;
    }
    // Print out what we've got.
    if (RULE_LOGGER.isTraceEnabled()) {
        traceLevelExpressions(exprs, exprLevels, levelTypeOrdinals, levelCount);
    }
    // Now build the calcs.
    RelNode rel = child;
    final int inputFieldCount = program.getInputRowType().getFieldCount();
    int[] inputExprOrdinals = identityArray(inputFieldCount);
    boolean doneCondition = false;
    for (int level = 0; level < levelCount; level++) {
        final int[] projectExprOrdinals;
        final RelDataType outputRowType;
        if (level == (levelCount - 1)) {
            outputRowType = program.getOutputRowType();
            projectExprOrdinals = new int[projectRefList.size()];
            for (int i = 0; i < projectExprOrdinals.length; i++) {
                projectExprOrdinals[i] = projectRefList.get(i).getIndex();
            }
        } else {
            outputRowType = null;
            // Project the expressions which are computed at this level or
            // before, and will be used at later levels.
            List<Integer> projectExprOrdinalList = new ArrayList<>();
            for (int i = 0; i < exprs.length; i++) {
                RexNode expr = exprs[i];
                if (expr instanceof RexLiteral) {
                    // Don't project literals. They are always created in
                    // the level where they are used.
                    exprLevels[i] = -1;
                    continue;
                }
                if ((exprLevels[i] <= level) && (exprMaxUsingLevelOrdinals[i] > level)) {
                    projectExprOrdinalList.add(i);
                }
            }
            projectExprOrdinals = Ints.toArray(projectExprOrdinalList);
        }
        final RelType relType = relTypes[levelTypeOrdinals[level]];
        // Can we do the condition this level?
        int conditionExprOrdinal = -1;
        if ((conditionRef != null) && !doneCondition) {
            conditionExprOrdinal = conditionRef.getIndex();
            if ((exprLevels[conditionExprOrdinal] > level) || !relType.supportsCondition()) {
                // stand down -- we're not ready to do the condition yet
                conditionExprOrdinal = -1;
            } else {
                doneCondition = true;
            }
        }
        RexProgram program1 = createProgramForLevel(level, levelCount, rel.getRowType(), exprs, exprLevels, inputExprOrdinals, projectExprOrdinals, conditionExprOrdinal, outputRowType);
        rel = relType.makeRel(cluster, traits, relBuilder, rel, program1);
        rel = handle(rel);
        // The outputs of this level will be the inputs to the next level.
        inputExprOrdinals = projectExprOrdinals;
    }
    Preconditions.checkArgument(doneCondition || (conditionRef == null), "unhandled condition");
    return rel;
}