Examples with Calc org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Calc used on opensource projects

Example 16 with Calc

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Calc in project beam by apache.

the class AbstractBeamCalcRel method estimateFilterSelectivity.

private static double estimateFilterSelectivity(RelNode child, RexProgram program, RelMetadataQuery mq) {
    // Similar to calcite, if the calc node is representing filter operation we estimate the filter
    // selectivity based on the number of equality conditions, number of inequality conditions, ....
    RexLocalRef programCondition = program.getCondition();
    RexNode condition;
    if (programCondition == null) {
        condition = null;
    } else {
        condition = program.expandLocalRef(programCondition);
    }
    // Currently this gets the selectivity based on Calcite's Selectivity Handler (RelMdSelectivity)
    return mq.getSelectivity(child, condition);
}

Example 17 with Calc

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Calc in project beam by apache.

the class BeamCalcRule method convert.

@Override
public RelNode convert(RelNode rel) {
    final Calc calc = (Calc) rel;
    final RelNode input = calc.getInput();
    return new BeamCalcRel(calc.getCluster(), calc.getTraitSet().replace(BeamLogicalConvention.INSTANCE), RelOptRule.convert(input, input.getTraitSet().replace(BeamLogicalConvention.INSTANCE)), calc.getProgram());
}

Example 18 with Calc

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Calc in project beam by apache.

the class BeamCalcRule method matches.

@Override
public boolean matches(RelOptRuleCall x) {
    /**
     * The Analytic Functions (a.k.a. window functions) match with both Calc and Window rules. So,
     * it is necessary to skip the Calc rule in order to execute the more suitable conversion
     * (BeamWindowRule).
     */
    boolean hasRexOver = false;
    List<RelNode> resList = x.getRelList();
    for (RelNode relNode : resList) {
        if (relNode instanceof LogicalCalc) {
            LogicalCalc logicalCalc = (LogicalCalc) relNode;
            for (RexNode rexNode : logicalCalc.getProgram().getExprList()) {
                if (rexNode instanceof RexOver) {
                    hasRexOver = true;
                    break;
                }
            }
        }
    }
    return !hasRexOver;
}

Example 19 with Calc

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Calc in project beam by apache.

the class BeamIOPushDownRule method findUtilizedInputRefs.

/**
 * Given a {@code RexNode}, find all {@code RexInputRef}s a node or it's children nodes use.
 *
 * @param inputRowType {@code RelDataType} used for looking up names of {@code RexInputRef}.
 * @param startNode A node to start at.
 * @param usedFields Names of {@code RexInputRef}s are added to this list.
 */
@VisibleForTesting
void findUtilizedInputRefs(RelDataType inputRowType, RexNode startNode, Set<String> usedFields) {
    Queue<RexNode> prerequisites = new ArrayDeque<>();
    prerequisites.add(startNode);
    // Assuming there are no cyclic nodes, traverse dependency tree until all RexInputRefs are found
    while (!prerequisites.isEmpty()) {
        RexNode node = prerequisites.poll();
        if (node instanceof RexCall) {
            // Composite expression, example: "=($t11, $t12)"
            RexCall compositeNode = (RexCall) node;
            // Expression from example above contains 2 operands: $t11, $t12
            prerequisites.addAll(compositeNode.getOperands());
        } else if (node instanceof RexInputRef) {
            // Input reference
            // Find a field in an inputRowType for the input reference
            int inputFieldIndex = ((RexInputRef) node).getIndex();
            RelDataTypeField field = inputRowType.getFieldList().get(inputFieldIndex);
            // If we have not seen it before - add it to the list (hash set)
            usedFields.add(field.getName());
        } else if (node instanceof RexLiteral) {
        // Does not contain information about columns utilized by a Calc
        } else {
            throw new UnsupportedOperationException("Unexpected RexNode encountered: " + node.getClass().getSimpleName());
        }
    }
}

Example 20 with Calc

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Calc in project beam by apache.

the class BeamIOPushDownRule method onMatch.

// ~ Methods ----------------------------------------------------------------
@Override
public void onMatch(RelOptRuleCall call) {
    final BeamIOSourceRel ioSourceRel = call.rel(1);
    final BeamSqlTable beamSqlTable = ioSourceRel.getBeamSqlTable();
    if (ioSourceRel instanceof BeamPushDownIOSourceRel) {
        return;
    }
    // Nested rows are not supported at the moment
    for (RelDataTypeField field : ioSourceRel.getRowType().getFieldList()) {
        if (field.getType() instanceof RelRecordType) {
            return;
        }
    }
    final Calc calc = call.rel(0);
    final RexProgram program = calc.getProgram();
    final Pair<ImmutableList<RexNode>, ImmutableList<RexNode>> projectFilter = program.split();
    final RelDataType calcInputRowType = program.getInputRowType();
    // When predicate push-down is not supported - all filters are unsupported.
    final BeamSqlTableFilter tableFilter = beamSqlTable.constructFilter(projectFilter.right);
    if (!beamSqlTable.supportsProjects().isSupported() && tableFilter instanceof DefaultTableFilter) {
        // Either project or filter push-down must be supported by the IO.
        return;
    }
    Set<String> usedFields = new LinkedHashSet<>();
    if (!(tableFilter instanceof DefaultTableFilter) && !beamSqlTable.supportsProjects().isSupported()) {
        // When applying standalone filter push-down all fields must be project by an IO.
        // With a single exception: Calc projects all fields (in the same order) and does nothing
        // else.
        usedFields.addAll(calcInputRowType.getFieldNames());
    } else {
        // Find all input refs used by projects
        for (RexNode project : projectFilter.left) {
            findUtilizedInputRefs(calcInputRowType, project, usedFields);
        }
        // Find all input refs used by filters
        for (RexNode filter : tableFilter.getNotSupported()) {
            findUtilizedInputRefs(calcInputRowType, filter, usedFields);
        }
    }
    if (usedFields.isEmpty()) {
        // No need to do push-down for queries like this: "select UPPER('hello')".
        return;
    }
    // IO only projects fields utilized by a calc.
    if (tableFilter.getNotSupported().containsAll(projectFilter.right) && usedFields.containsAll(ioSourceRel.getRowType().getFieldNames())) {
        return;
    }
    FieldAccessDescriptor resolved = FieldAccessDescriptor.withFieldNames(usedFields);
    resolved = resolved.resolve(beamSqlTable.getSchema());
    if (canDropCalc(program, beamSqlTable.supportsProjects(), tableFilter)) {
        call.transformTo(ioSourceRel.createPushDownRel(calc.getRowType(), resolved.getFieldsAccessed().stream().map(FieldDescriptor::getFieldName).collect(Collectors.toList()), tableFilter));
        return;
    }
    // IO only projects fields utilised by a calc.
    if (tableFilter.getNotSupported().equals(projectFilter.right) && usedFields.containsAll(ioSourceRel.getRowType().getFieldNames())) {
        return;
    }
    RelNode result = constructNodesWithPushDown(resolved, call.builder(), ioSourceRel, tableFilter, calc.getRowType(), projectFilter.left);
    if (tableFilter.getNotSupported().size() <= projectFilter.right.size() || usedFields.size() < calcInputRowType.getFieldCount()) {
        // Smaller Calc programs are indisputably better, as well as IOs with less projected fields.
        // We can consider something with the same number of filters.
        call.transformTo(result);
    }
}