Search in sources :

Example 16 with RexProgram

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexProgram in project flink by apache.

the class PushFilterInCalcIntoTableSourceScanRule method matches.

@Override
public boolean matches(RelOptRuleCall call) {
    if (!super.matches(call)) {
        return false;
    }
    FlinkLogicalCalc calc = call.rel(0);
    RexProgram originProgram = calc.getProgram();
    if (originProgram.getCondition() == null) {
        return false;
    }
    FlinkLogicalTableSourceScan scan = call.rel(1);
    TableSourceTable tableSourceTable = scan.getTable().unwrap(TableSourceTable.class);
    // we can not push filter twice
    return canPushdownFilter(tableSourceTable);
}
Also used : FlinkLogicalCalc(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalCalc) RexProgram(org.apache.calcite.rex.RexProgram) FlinkLogicalTableSourceScan(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalTableSourceScan) TableSourceTable(org.apache.flink.table.planner.plan.schema.TableSourceTable)

Example 17 with RexProgram

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexProgram in project flink by apache.

the class PushFilterInCalcIntoTableSourceScanRule method pushFilterIntoScan.

private void pushFilterIntoScan(RelOptRuleCall call, Calc calc, FlinkLogicalTableSourceScan scan, FlinkPreparingTableBase relOptTable) {
    RexProgram originProgram = calc.getProgram();
    RelBuilder relBuilder = call.builder();
    Tuple2<RexNode[], RexNode[]> extractedPredicates = extractPredicates(originProgram.getInputRowType().getFieldNames().toArray(new String[0]), originProgram.expandLocalRef(originProgram.getCondition()), scan, relBuilder.getRexBuilder());
    RexNode[] convertiblePredicates = extractedPredicates._1;
    RexNode[] unconvertedPredicates = extractedPredicates._2;
    if (convertiblePredicates.length == 0) {
        // no condition can be translated to expression
        return;
    }
    Tuple2<SupportsFilterPushDown.Result, TableSourceTable> pushdownResultWithScan = resolveFiltersAndCreateTableSourceTable(convertiblePredicates, relOptTable.unwrap(TableSourceTable.class), scan, relBuilder);
    SupportsFilterPushDown.Result result = pushdownResultWithScan._1;
    TableSourceTable tableSourceTable = pushdownResultWithScan._2;
    FlinkLogicalTableSourceScan newScan = FlinkLogicalTableSourceScan.create(scan.getCluster(), scan.getHints(), tableSourceTable);
    // build new calc program
    RexProgramBuilder programBuilder = RexProgramBuilder.forProgram(originProgram, call.builder().getRexBuilder(), true);
    programBuilder.clearCondition();
    if (!result.getRemainingFilters().isEmpty() || unconvertedPredicates.length != 0) {
        RexNode remainingCondition = createRemainingCondition(relBuilder, result.getRemainingFilters(), unconvertedPredicates);
        RexNode simplifiedRemainingCondition = FlinkRexUtil.simplify(relBuilder.getRexBuilder(), remainingCondition, calc.getCluster().getPlanner().getExecutor());
        programBuilder.addCondition(simplifiedRemainingCondition);
    }
    RexProgram program = programBuilder.getProgram();
    if (program.isTrivial()) {
        call.transformTo(newScan);
    } else {
        FlinkLogicalCalc newCalc = FlinkLogicalCalc.create(newScan, program);
        call.transformTo(newCalc);
    }
}
Also used : RelBuilder(org.apache.calcite.tools.RelBuilder) FlinkLogicalCalc(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalCalc) RexProgram(org.apache.calcite.rex.RexProgram) FlinkLogicalTableSourceScan(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalTableSourceScan) SupportsFilterPushDown(org.apache.flink.table.connector.source.abilities.SupportsFilterPushDown) TableSourceTable(org.apache.flink.table.planner.plan.schema.TableSourceTable) RexProgramBuilder(org.apache.calcite.rex.RexProgramBuilder) RexNode(org.apache.calcite.rex.RexNode)

Example 18 with RexProgram

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexProgram in project flink by apache.

the class RelTimeIndicatorConverter method visitCalc.

private RelNode visitCalc(FlinkLogicalCalc calc) {
    // visit children and update inputs
    RelNode newInput = calc.getInput().accept(this);
    RexProgram program = calc.getProgram();
    // check if input field contains time indicator type
    // materialize field if no time indicator is present anymore
    // if input field is already materialized, change to timestamp type
    RexTimeIndicatorMaterializer materializer = new RexTimeIndicatorMaterializer(newInput);
    List<RexNode> newProjects = program.getProjectList().stream().map(project -> program.expandLocalRef(project).accept(materializer)).collect(Collectors.toList());
    // materialize condition due to filter will validate condition type
    RexNode newCondition = null;
    if (program.getCondition() != null) {
        newCondition = program.expandLocalRef(program.getCondition()).accept(materializer);
    }
    RexProgram newProgram = RexProgram.create(newInput.getRowType(), newProjects, newCondition, program.getOutputRowType().getFieldNames(), rexBuilder);
    return calc.copy(calc.getTraitSet(), newInput, newProgram);
}
Also used : WindowUtil.groupingContainsWindowStartEnd(org.apache.flink.table.planner.plan.utils.WindowUtil.groupingContainsWindowStartEnd) RexProgram(org.apache.calcite.rex.RexProgram) Tuple2(org.apache.flink.api.java.tuple.Tuple2) FlinkLogicalWindowAggregate(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalWindowAggregate) LogicalTableModify(org.apache.calcite.rel.logical.LogicalTableModify) FlinkLogicalLegacySink(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalLegacySink) FlinkLogicalDistribution(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalDistribution) FlinkLogicalIntersect(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalIntersect) TemporalJoinUtil(org.apache.flink.table.planner.plan.utils.TemporalJoinUtil) RexPatternFieldRef(org.apache.calcite.rex.RexPatternFieldRef) RexNode(org.apache.calcite.rex.RexNode) Map(java.util.Map) FlinkLogicalSnapshot(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalSnapshot) LocalZonedTimestampType(org.apache.flink.table.types.logical.LocalZonedTimestampType) FlinkLogicalMinus(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalMinus) Predicate(java.util.function.Predicate) FlinkLogicalTableFunctionScan(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalTableFunctionScan) FlinkLogicalRank(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalRank) Set(java.util.Set) FlinkLogicalJoin(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalJoin) Preconditions(org.apache.flink.util.Preconditions) Collectors(java.util.stream.Collectors) FlinkLogicalValues(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalValues) RexInputRef(org.apache.calcite.rex.RexInputRef) List(java.util.List) FlinkTypeFactory.isTimeIndicatorType(org.apache.flink.table.planner.calcite.FlinkTypeFactory.isTimeIndicatorType) LogicalType(org.apache.flink.table.types.logical.LogicalType) SqlStdOperatorTable(org.apache.calcite.sql.fun.SqlStdOperatorTable) FlinkLogicalSink(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalSink) RexLocalRef(org.apache.calcite.rex.RexLocalRef) ValidationException(org.apache.flink.table.api.ValidationException) RelDataTypeField(org.apache.calcite.rel.type.RelDataTypeField) FlinkTypeFactory.isRowtimeIndicatorType(org.apache.flink.table.planner.calcite.FlinkTypeFactory.isRowtimeIndicatorType) FlinkLogicalSort(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalSort) FlinkLogicalOverAggregate(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalOverAggregate) RexCall(org.apache.calcite.rex.RexCall) IntStream(java.util.stream.IntStream) TableScan(org.apache.calcite.rel.core.TableScan) LogicalCalc(org.apache.calcite.rel.logical.LogicalCalc) FlinkLogicalMatch(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalMatch) FlinkLogicalUnion(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalUnion) SetOp(org.apache.calcite.rel.core.SetOp) RelWindowProperties(org.apache.flink.table.planner.plan.trait.RelWindowProperties) RelOptUtil(org.apache.calcite.plan.RelOptUtil) Function(java.util.function.Function) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) LinkedHashMap(java.util.LinkedHashMap) MatchUtil.isFinalOnMatchTimeIndicator(org.apache.flink.table.planner.plan.utils.MatchUtil.isFinalOnMatchTimeIndicator) TimestampType(org.apache.flink.table.types.logical.TimestampType) Pair(org.apache.calcite.util.Pair) FlinkLogicalAggregate(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalAggregate) TimeIndicatorRelDataType(org.apache.flink.table.planner.plan.schema.TimeIndicatorRelDataType) SqlOperator(org.apache.calcite.sql.SqlOperator) SingleRel(org.apache.calcite.rel.SingleRel) RelCollations(org.apache.calcite.rel.RelCollations) RelDataType(org.apache.calcite.rel.type.RelDataType) SqlTypeName(org.apache.calcite.sql.type.SqlTypeName) FlinkRelMetadataQuery(org.apache.flink.table.planner.plan.metadata.FlinkRelMetadataQuery) RexBuilder(org.apache.calcite.rex.RexBuilder) FlinkTypeFactory.isProctimeIndicatorType(org.apache.flink.table.planner.calcite.FlinkTypeFactory.isProctimeIndicatorType) TableException(org.apache.flink.table.api.TableException) FlinkLogicalTableAggregate(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalTableAggregate) FlinkLogicalCalc(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalCalc) FlinkLogicalWindowTableAggregate(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalWindowTableAggregate) FlinkLogicalExpand(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalExpand) RelNode(org.apache.calcite.rel.RelNode) Aggregate(org.apache.calcite.rel.core.Aggregate) JoinUtil(org.apache.flink.table.planner.plan.utils.JoinUtil) RelHomogeneousShuttle(org.apache.calcite.rel.RelHomogeneousShuttle) RexProgramBuilder(org.apache.calcite.rex.RexProgramBuilder) FlinkSqlOperatorTable(org.apache.flink.table.planner.functions.sql.FlinkSqlOperatorTable) FlinkLogicalWatermarkAssigner(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalWatermarkAssigner) AggregateCall(org.apache.calcite.rel.core.AggregateCall) RexShuttle(org.apache.calcite.rex.RexShuttle) FlinkLogicalCorrelate(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalCorrelate) Collections(java.util.Collections) RelNode(org.apache.calcite.rel.RelNode) RexProgram(org.apache.calcite.rex.RexProgram) RexNode(org.apache.calcite.rex.RexNode)

Example 19 with RexProgram

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexProgram in project flink by apache.

the class RelTimeIndicatorConverter method createCalcToMaterializeTimeIndicators.

private RelNode createCalcToMaterializeTimeIndicators(RelNode input, Set<Integer> refIndices) {
    // create new calc
    List<RexNode> projects = input.getRowType().getFieldList().stream().map(field -> {
        RexNode project = new RexInputRef(field.getIndex(), field.getType());
        if (refIndices.contains(field.getIndex())) {
            project = materializeTimeIndicators(project);
        }
        return project;
    }).collect(Collectors.toList());
    RexProgram newProgram = RexProgram.create(input.getRowType(), projects, null, input.getRowType().getFieldNames(), rexBuilder);
    return FlinkLogicalCalc.create(input, newProgram);
}
Also used : WindowUtil.groupingContainsWindowStartEnd(org.apache.flink.table.planner.plan.utils.WindowUtil.groupingContainsWindowStartEnd) RexProgram(org.apache.calcite.rex.RexProgram) Tuple2(org.apache.flink.api.java.tuple.Tuple2) FlinkLogicalWindowAggregate(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalWindowAggregate) LogicalTableModify(org.apache.calcite.rel.logical.LogicalTableModify) FlinkLogicalLegacySink(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalLegacySink) FlinkLogicalDistribution(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalDistribution) FlinkLogicalIntersect(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalIntersect) TemporalJoinUtil(org.apache.flink.table.planner.plan.utils.TemporalJoinUtil) RexPatternFieldRef(org.apache.calcite.rex.RexPatternFieldRef) RexNode(org.apache.calcite.rex.RexNode) Map(java.util.Map) FlinkLogicalSnapshot(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalSnapshot) LocalZonedTimestampType(org.apache.flink.table.types.logical.LocalZonedTimestampType) FlinkLogicalMinus(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalMinus) Predicate(java.util.function.Predicate) FlinkLogicalTableFunctionScan(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalTableFunctionScan) FlinkLogicalRank(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalRank) Set(java.util.Set) FlinkLogicalJoin(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalJoin) Preconditions(org.apache.flink.util.Preconditions) Collectors(java.util.stream.Collectors) FlinkLogicalValues(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalValues) RexInputRef(org.apache.calcite.rex.RexInputRef) List(java.util.List) FlinkTypeFactory.isTimeIndicatorType(org.apache.flink.table.planner.calcite.FlinkTypeFactory.isTimeIndicatorType) LogicalType(org.apache.flink.table.types.logical.LogicalType) SqlStdOperatorTable(org.apache.calcite.sql.fun.SqlStdOperatorTable) FlinkLogicalSink(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalSink) RexLocalRef(org.apache.calcite.rex.RexLocalRef) ValidationException(org.apache.flink.table.api.ValidationException) RelDataTypeField(org.apache.calcite.rel.type.RelDataTypeField) FlinkTypeFactory.isRowtimeIndicatorType(org.apache.flink.table.planner.calcite.FlinkTypeFactory.isRowtimeIndicatorType) FlinkLogicalSort(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalSort) FlinkLogicalOverAggregate(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalOverAggregate) RexCall(org.apache.calcite.rex.RexCall) IntStream(java.util.stream.IntStream) TableScan(org.apache.calcite.rel.core.TableScan) LogicalCalc(org.apache.calcite.rel.logical.LogicalCalc) FlinkLogicalMatch(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalMatch) FlinkLogicalUnion(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalUnion) SetOp(org.apache.calcite.rel.core.SetOp) RelWindowProperties(org.apache.flink.table.planner.plan.trait.RelWindowProperties) RelOptUtil(org.apache.calcite.plan.RelOptUtil) Function(java.util.function.Function) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) LinkedHashMap(java.util.LinkedHashMap) MatchUtil.isFinalOnMatchTimeIndicator(org.apache.flink.table.planner.plan.utils.MatchUtil.isFinalOnMatchTimeIndicator) TimestampType(org.apache.flink.table.types.logical.TimestampType) Pair(org.apache.calcite.util.Pair) FlinkLogicalAggregate(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalAggregate) TimeIndicatorRelDataType(org.apache.flink.table.planner.plan.schema.TimeIndicatorRelDataType) SqlOperator(org.apache.calcite.sql.SqlOperator) SingleRel(org.apache.calcite.rel.SingleRel) RelCollations(org.apache.calcite.rel.RelCollations) RelDataType(org.apache.calcite.rel.type.RelDataType) SqlTypeName(org.apache.calcite.sql.type.SqlTypeName) FlinkRelMetadataQuery(org.apache.flink.table.planner.plan.metadata.FlinkRelMetadataQuery) RexBuilder(org.apache.calcite.rex.RexBuilder) FlinkTypeFactory.isProctimeIndicatorType(org.apache.flink.table.planner.calcite.FlinkTypeFactory.isProctimeIndicatorType) TableException(org.apache.flink.table.api.TableException) FlinkLogicalTableAggregate(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalTableAggregate) FlinkLogicalCalc(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalCalc) FlinkLogicalWindowTableAggregate(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalWindowTableAggregate) FlinkLogicalExpand(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalExpand) RelNode(org.apache.calcite.rel.RelNode) Aggregate(org.apache.calcite.rel.core.Aggregate) JoinUtil(org.apache.flink.table.planner.plan.utils.JoinUtil) RelHomogeneousShuttle(org.apache.calcite.rel.RelHomogeneousShuttle) RexProgramBuilder(org.apache.calcite.rex.RexProgramBuilder) FlinkSqlOperatorTable(org.apache.flink.table.planner.functions.sql.FlinkSqlOperatorTable) FlinkLogicalWatermarkAssigner(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalWatermarkAssigner) AggregateCall(org.apache.calcite.rel.core.AggregateCall) RexShuttle(org.apache.calcite.rex.RexShuttle) FlinkLogicalCorrelate(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalCorrelate) Collections(java.util.Collections) RexProgram(org.apache.calcite.rex.RexProgram) RexInputRef(org.apache.calcite.rex.RexInputRef) RexNode(org.apache.calcite.rex.RexNode)

Example 20 with RexProgram

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexProgram in project beam by apache.

the class BeamIOPushDownRule method canDropCalc.

/**
 * Perform a series of checks to determine whether a Calc can be dropped. Following conditions
 * need to be met in order for that to happen (logical AND):<br>
 * 1. Program should do simple projects, project each field once, and project fields in the same
 * order when field reordering is not supported.<br>
 * 2. Predicate can be completely pushed-down.<br>
 * 3. Project push-down is supported by the IO or all fields are projected by a Calc.
 *
 * @param program A {@code RexProgram} of a {@code Calc}.
 * @param projectSupport An enum containing information about IO project push-down capabilities.
 * @param tableFilter A class containing information about IO predicate push-down capabilities.
 * @return True when Calc can be dropped, false otherwise.
 */
private boolean canDropCalc(RexProgram program, ProjectSupport projectSupport, BeamSqlTableFilter tableFilter) {
    RelDataType calcInputRowType = program.getInputRowType();
    // Program should do simple projects, project each field once, and project fields in the same
    // order when field reordering is not supported.
    boolean fieldReorderingSupported = projectSupport.withFieldReordering();
    if (!isProjectRenameOnlyProgram(program, fieldReorderingSupported)) {
        return false;
    }
    // Predicate can be completely pushed-down
    if (!tableFilter.getNotSupported().isEmpty()) {
        return false;
    }
    // Project push-down is supported by the IO or all fields are projected by a Calc.
    boolean isProjectSupported = projectSupport.isSupported();
    boolean allFieldsProjected = program.getProjectList().stream().map(ref -> program.getInputRowType().getFieldList().get(ref.getIndex()).getName()).collect(Collectors.toList()).equals(calcInputRowType.getFieldNames());
    return isProjectSupported || allFieldsProjected;
}
Also used : RelDataType(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.type.RelDataType)

Aggregations

RexProgram (org.apache.calcite.rex.RexProgram)46 RexNode (org.apache.calcite.rex.RexNode)29 RexProgramBuilder (org.apache.calcite.rex.RexProgramBuilder)26 RexBuilder (org.apache.calcite.rex.RexBuilder)21 RelNode (org.apache.calcite.rel.RelNode)17 ArrayList (java.util.ArrayList)13 RelDataType (org.apache.calcite.rel.type.RelDataType)13 RexLocalRef (org.apache.calcite.rex.RexLocalRef)12 LogicalCalc (org.apache.calcite.rel.logical.LogicalCalc)11 FlinkLogicalCalc (org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalCalc)10 RelDataTypeField (org.apache.calcite.rel.type.RelDataTypeField)9 List (java.util.List)7 RexInputRef (org.apache.calcite.rex.RexInputRef)7 RexLocalRef (org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexLocalRef)6 Collectors (java.util.stream.Collectors)5 RelDataTypeFactory (org.apache.calcite.rel.type.RelDataTypeFactory)5 RexCall (org.apache.calcite.rex.RexCall)5 RexShuttle (org.apache.calcite.rex.RexShuttle)5 Collections (java.util.Collections)4 RelOptUtil (org.apache.calcite.plan.RelOptUtil)4