Search in sources :

Example 81 with RexInputRef

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexInputRef in project flink by apache.

the class CommonExecPythonCorrelate method extractPythonTableFunctionInfo.

private Tuple2<int[], PythonFunctionInfo> extractPythonTableFunctionInfo() {
    LinkedHashMap<RexNode, Integer> inputNodes = new LinkedHashMap<>();
    PythonFunctionInfo pythonTableFunctionInfo = CommonPythonUtil.createPythonFunctionInfo(invocation, inputNodes);
    int[] udtfInputOffsets = inputNodes.keySet().stream().filter(x -> x instanceof RexInputRef).map(x -> ((RexInputRef) x).getIndex()).mapToInt(i -> i).toArray();
    return Tuple2.of(udtfInputOffsets, pythonTableFunctionInfo);
}
Also used : InputProperty(org.apache.flink.table.planner.plan.nodes.exec.InputProperty) Tuple2(org.apache.flink.api.java.tuple.Tuple2) RowType(org.apache.flink.table.types.logical.RowType) Constructor(java.lang.reflect.Constructor) ExecNode(org.apache.flink.table.planner.plan.nodes.exec.ExecNode) LinkedHashMap(java.util.LinkedHashMap) ExecNodeUtil(org.apache.flink.table.planner.plan.nodes.exec.utils.ExecNodeUtil) RexNode(org.apache.calcite.rex.RexNode) ManagedMemoryUseCase(org.apache.flink.core.memory.ManagedMemoryUseCase) CodeGeneratorContext(org.apache.flink.table.planner.codegen.CodeGeneratorContext) FlinkJoinType(org.apache.flink.table.runtime.operators.join.FlinkJoinType) Projection(org.apache.flink.table.connector.Projection) ProjectionCodeGenerator(org.apache.flink.table.planner.codegen.ProjectionCodeGenerator) ExecNodeContext(org.apache.flink.table.planner.plan.nodes.exec.ExecNodeContext) RowData(org.apache.flink.table.data.RowData) PlannerBase(org.apache.flink.table.planner.delegation.PlannerBase) CommonPythonUtil(org.apache.flink.table.planner.plan.nodes.exec.utils.CommonPythonUtil) SingleTransformationTranslator(org.apache.flink.table.planner.plan.nodes.exec.SingleTransformationTranslator) ExecNodeConfig(org.apache.flink.table.planner.plan.nodes.exec.ExecNodeConfig) Configuration(org.apache.flink.configuration.Configuration) TableException(org.apache.flink.table.api.TableException) PythonFunctionInfo(org.apache.flink.table.functions.python.PythonFunctionInfo) OneInputTransformation(org.apache.flink.streaming.api.transformations.OneInputTransformation) RexInputRef(org.apache.calcite.rex.RexInputRef) List(java.util.List) InternalTypeInfo(org.apache.flink.table.runtime.typeutils.InternalTypeInfo) ExecEdge(org.apache.flink.table.planner.plan.nodes.exec.ExecEdge) Preconditions.checkArgument(org.apache.flink.util.Preconditions.checkArgument) ExecNodeBase(org.apache.flink.table.planner.plan.nodes.exec.ExecNodeBase) GeneratedProjection(org.apache.flink.table.runtime.generated.GeneratedProjection) Transformation(org.apache.flink.api.dag.Transformation) OneInputStreamOperator(org.apache.flink.streaming.api.operators.OneInputStreamOperator) RexCall(org.apache.calcite.rex.RexCall) PythonFunctionInfo(org.apache.flink.table.functions.python.PythonFunctionInfo) RexInputRef(org.apache.calcite.rex.RexInputRef) RexNode(org.apache.calcite.rex.RexNode) LinkedHashMap(java.util.LinkedHashMap)

Example 82 with RexInputRef

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexInputRef in project flink by apache.

the class FlinkRelMdCollation method project.

/**
 * Helper method to determine a {@link Project}'s collation.
 */
public static List<RelCollation> project(RelMetadataQuery mq, RelNode input, List<? extends RexNode> projects) {
    final SortedSet<RelCollation> collations = new TreeSet<>();
    final List<RelCollation> inputCollations = mq.collations(input);
    if (inputCollations == null || inputCollations.isEmpty()) {
        return com.google.common.collect.ImmutableList.of();
    }
    final com.google.common.collect.Multimap<Integer, Integer> targets = com.google.common.collect.LinkedListMultimap.create();
    final Map<Integer, SqlMonotonicity> targetsWithMonotonicity = new HashMap<>();
    for (Ord<RexNode> project : Ord.<RexNode>zip(projects)) {
        if (project.e instanceof RexInputRef) {
            targets.put(((RexInputRef) project.e).getIndex(), project.i);
        } else if (project.e instanceof RexCall) {
            final RexCall call = (RexCall) project.e;
            final RexCallBinding binding = RexCallBinding.create(input.getCluster().getTypeFactory(), call, inputCollations);
            targetsWithMonotonicity.put(project.i, call.getOperator().getMonotonicity(binding));
        }
    }
    final List<RelFieldCollation> fieldCollations = new ArrayList<>();
    loop: for (RelCollation ic : inputCollations) {
        if (ic.getFieldCollations().isEmpty()) {
            continue;
        }
        fieldCollations.clear();
        for (RelFieldCollation ifc : ic.getFieldCollations()) {
            final Collection<Integer> integers = targets.get(ifc.getFieldIndex());
            if (integers.isEmpty()) {
                // cannot do this collation
                continue loop;
            }
            fieldCollations.add(ifc.withFieldIndex(integers.iterator().next()));
        }
        assert !fieldCollations.isEmpty();
        collations.add(RelCollations.of(fieldCollations));
    }
    final List<RelFieldCollation> fieldCollationsForRexCalls = new ArrayList<>();
    for (Map.Entry<Integer, SqlMonotonicity> entry : targetsWithMonotonicity.entrySet()) {
        final SqlMonotonicity value = entry.getValue();
        switch(value) {
            case NOT_MONOTONIC:
            case CONSTANT:
                break;
            default:
                fieldCollationsForRexCalls.add(new RelFieldCollation(entry.getKey(), RelFieldCollation.Direction.of(value)));
                break;
        }
    }
    if (!fieldCollationsForRexCalls.isEmpty()) {
        collations.add(RelCollations.of(fieldCollationsForRexCalls));
    }
    return com.google.common.collect.ImmutableList.copyOf(collations);
}
Also used : HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) RexCall(org.apache.calcite.rex.RexCall) RexCallBinding(org.apache.calcite.rex.RexCallBinding) RelCollation(org.apache.calcite.rel.RelCollation) TreeSet(java.util.TreeSet) SqlMonotonicity(org.apache.calcite.sql.validate.SqlMonotonicity) RelFieldCollation(org.apache.calcite.rel.RelFieldCollation) RexInputRef(org.apache.calcite.rex.RexInputRef) Collection(java.util.Collection) Map(java.util.Map) HashMap(java.util.HashMap) RexNode(org.apache.calcite.rex.RexNode)

Example 83 with RexInputRef

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexInputRef in project flink by apache.

the class PushWatermarkIntoTableSourceScanAcrossCalcRule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    FlinkLogicalWatermarkAssigner watermarkAssigner = call.rel(0);
    FlinkLogicalCalc calc = call.rel(1);
    RexProgram originProgram = calc.getProgram();
    List<RexNode> projectList = originProgram.getProjectList().stream().map(originProgram::expandLocalRef).collect(Collectors.toList());
    // get watermark expression
    RexNode rowTimeColumn = projectList.get(watermarkAssigner.rowtimeFieldIndex());
    RexNode newWatermarkExpr = watermarkAssigner.watermarkExpr().accept(new RexShuttle() {

        @Override
        public RexNode visitInputRef(RexInputRef inputRef) {
            return projectList.get(inputRef.getIndex());
        }
    });
    // push watermark assigner into the scan
    FlinkLogicalTableSourceScan newScan = getNewScan(watermarkAssigner, newWatermarkExpr, call.rel(2), ShortcutUtils.unwrapContext(calc).getTableConfig(), // useWatermarkAssignerRowType
    false);
    FlinkTypeFactory typeFactory = ShortcutUtils.unwrapTypeFactory(calc);
    RexBuilder builder = call.builder().getRexBuilder();
    // cast timestamp/timestamp_ltz type to rowtime type.
    RexNode newRowTimeColumn = builder.makeReinterpretCast(typeFactory.createRowtimeIndicatorType(rowTimeColumn.getType().isNullable(), rowTimeColumn.getType().getSqlTypeName() == SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE), rowTimeColumn, null);
    // build new calc program
    RexProgramBuilder programBuilder = new RexProgramBuilder(newScan.getRowType(), builder);
    List<String> outputFieldNames = originProgram.getOutputRowType().getFieldNames();
    for (int i = 0; i < projectList.size(); i++) {
        if (i == watermarkAssigner.rowtimeFieldIndex()) {
            // replace the origin computed column to keep type consistent
            programBuilder.addProject(newRowTimeColumn, outputFieldNames.get(i));
        } else {
            programBuilder.addProject(projectList.get(i), outputFieldNames.get(i));
        }
    }
    if (originProgram.getCondition() != null) {
        programBuilder.addCondition(originProgram.expandLocalRef(originProgram.getCondition()));
    }
    FlinkLogicalCalc newCalc = FlinkLogicalCalc.create(newScan, programBuilder.getProgram());
    call.transformTo(newCalc);
}
Also used : RexShuttle(org.apache.calcite.rex.RexShuttle) FlinkLogicalCalc(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalCalc) RexProgram(org.apache.calcite.rex.RexProgram) FlinkLogicalTableSourceScan(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalTableSourceScan) FlinkLogicalWatermarkAssigner(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalWatermarkAssigner) FlinkTypeFactory(org.apache.flink.table.planner.calcite.FlinkTypeFactory) RexInputRef(org.apache.calcite.rex.RexInputRef) RexBuilder(org.apache.calcite.rex.RexBuilder) RexProgramBuilder(org.apache.calcite.rex.RexProgramBuilder) RexNode(org.apache.calcite.rex.RexNode)

Example 84 with RexInputRef

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexInputRef in project flink by apache.

the class WatermarkAssignerChangelogNormalizeTransposeRule method createTransformedProgramWithAllShuffleKeys.

private RexProgram createTransformedProgramWithAllShuffleKeys(RexProgram program, List<Integer> projectsOutShuffleKeys, RexBuilder rexBuilder) {
    RelDataType oldInputRowType = program.getInputRowType();
    List<String> visitedProjectNames = new ArrayList<>();
    RexProgramBuilder newProgramBuilder = new RexProgramBuilder(oldInputRowType, rexBuilder);
    program.getNamedProjects().forEach(pair -> {
        newProgramBuilder.addProject(program.expandLocalRef(pair.left), pair.right);
        visitedProjectNames.add(pair.right);
    });
    List<RelDataTypeField> oldFieldList = oldInputRowType.getFieldList();
    for (Integer projectsOutShuffleKey : projectsOutShuffleKeys) {
        RelDataTypeField oldField = oldFieldList.get(projectsOutShuffleKey);
        String oldFieldName = oldField.getName();
        String newProjectName = RowTypeUtils.getUniqueName(oldFieldName, visitedProjectNames);
        newProgramBuilder.addProject(new RexInputRef(projectsOutShuffleKey, oldField.getType()), newProjectName);
        visitedProjectNames.add(newProjectName);
    }
    if (program.getCondition() != null) {
        newProgramBuilder.addCondition(program.expandLocalRef(program.getCondition()));
    }
    return newProgramBuilder.getProgram();
}
Also used : RelDataTypeField(org.apache.calcite.rel.type.RelDataTypeField) ArrayList(java.util.ArrayList) RexInputRef(org.apache.calcite.rex.RexInputRef) RelDataType(org.apache.calcite.rel.type.RelDataType) RexProgramBuilder(org.apache.calcite.rex.RexProgramBuilder)

Example 85 with RexInputRef

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexInputRef in project flink by apache.

the class WatermarkAssignerChangelogNormalizeTransposeRule method pushDownTransformedWatermark.

private RelNode pushDownTransformedWatermark(StreamPhysicalWatermarkAssigner watermark, StreamPhysicalCalc calc, StreamPhysicalChangelogNormalize changelogNormalize, StreamPhysicalExchange exchange, Mappings.TargetMapping calcMapping, RexBuilder rexBuilder) {
    Mappings.TargetMapping inversedMapping = calcMapping.inverse();
    final int newRowTimeFieldIndex = inversedMapping.getTargetOpt(watermark.rowtimeFieldIndex());
    // Updates watermark properties after push down before Calc
    // 1. rewrites watermark expression
    // 2. clears distribution
    // 3. updates row time field index
    RexNode newWatermarkExpr = watermark.watermarkExpr();
    if (watermark.watermarkExpr() != null) {
        newWatermarkExpr = RexUtil.apply(inversedMapping, watermark.watermarkExpr());
    }
    final RelNode newWatermark = watermark.copy(watermark.getTraitSet().plus(FlinkRelDistribution.DEFAULT()), exchange.getInput(), newRowTimeFieldIndex, newWatermarkExpr);
    final RelNode newChangelogNormalize = buildTreeInOrder(newWatermark, Tuple2.of(exchange, exchange.getTraitSet()), Tuple2.of(changelogNormalize, changelogNormalize.getTraitSet()));
    // Rewrites Calc program because the field type of row time
    // field is changed after watermark pushed down
    final RexProgram oldProgram = calc.getProgram();
    final RexProgramBuilder programBuilder = new RexProgramBuilder(newChangelogNormalize.getRowType(), rexBuilder);
    final Function<RexNode, RexNode> rexShuttle = e -> e.accept(new RexShuttle() {

        @Override
        public RexNode visitInputRef(RexInputRef inputRef) {
            if (inputRef.getIndex() == newRowTimeFieldIndex) {
                return RexInputRef.of(newRowTimeFieldIndex, newChangelogNormalize.getRowType());
            } else {
                return inputRef;
            }
        }
    });
    oldProgram.getNamedProjects().forEach(pair -> programBuilder.addProject(rexShuttle.apply(oldProgram.expandLocalRef(pair.left)), pair.right));
    if (oldProgram.getCondition() != null) {
        programBuilder.addCondition(rexShuttle.apply(oldProgram.expandLocalRef(oldProgram.getCondition())));
    }
    final RexProgram newProgram = programBuilder.getProgram();
    return calc.copy(calc.getTraitSet(), newChangelogNormalize, newProgram);
}
Also used : Mappings(org.apache.calcite.util.mapping.Mappings) RexProgram(org.apache.calcite.rex.RexProgram) StreamPhysicalExchange(org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalExchange) Tuple2(org.apache.flink.api.java.tuple.Tuple2) HashMap(java.util.HashMap) FlinkRelDistribution(org.apache.flink.table.planner.plan.trait.FlinkRelDistribution) RelOptUtil(org.apache.calcite.plan.RelOptUtil) Function(java.util.function.Function) ArrayList(java.util.ArrayList) RexUtil(org.apache.calcite.rex.RexUtil) StreamPhysicalChangelogNormalize(org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalChangelogNormalize) RexNode(org.apache.calcite.rex.RexNode) Map(java.util.Map) RelTraitSet(org.apache.calcite.plan.RelTraitSet) FlinkRelDistributionTraitDef(org.apache.flink.table.planner.plan.trait.FlinkRelDistributionTraitDef) RelDataType(org.apache.calcite.rel.type.RelDataType) RexBuilder(org.apache.calcite.rex.RexBuilder) StreamPhysicalCalc(org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalCalc) RelRule(org.apache.calcite.plan.RelRule) RelNode(org.apache.calcite.rel.RelNode) RelOptRuleCall(org.apache.calcite.plan.RelOptRuleCall) RowTypeUtils(org.apache.flink.table.planner.typeutils.RowTypeUtils) RexInputRef(org.apache.calcite.rex.RexInputRef) RelOptRule(org.apache.calcite.plan.RelOptRule) RexProgramBuilder(org.apache.calcite.rex.RexProgramBuilder) List(java.util.List) Preconditions.checkArgument(org.apache.flink.util.Preconditions.checkArgument) RexLocalRef(org.apache.calcite.rex.RexLocalRef) RelDistribution(org.apache.calcite.rel.RelDistribution) RelDataTypeField(org.apache.calcite.rel.type.RelDataTypeField) StreamPhysicalWatermarkAssigner(org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalWatermarkAssigner) RexShuttle(org.apache.calcite.rex.RexShuttle) Collections(java.util.Collections) RexShuttle(org.apache.calcite.rex.RexShuttle) Mappings(org.apache.calcite.util.mapping.Mappings) RelNode(org.apache.calcite.rel.RelNode) RexProgram(org.apache.calcite.rex.RexProgram) RexInputRef(org.apache.calcite.rex.RexInputRef) RexProgramBuilder(org.apache.calcite.rex.RexProgramBuilder) RexNode(org.apache.calcite.rex.RexNode)

Aggregations

RexInputRef (org.apache.calcite.rex.RexInputRef)241 RexNode (org.apache.calcite.rex.RexNode)200 ArrayList (java.util.ArrayList)105 RelNode (org.apache.calcite.rel.RelNode)85 RelDataTypeField (org.apache.calcite.rel.type.RelDataTypeField)80 RexCall (org.apache.calcite.rex.RexCall)67 RelDataType (org.apache.calcite.rel.type.RelDataType)63 RexBuilder (org.apache.calcite.rex.RexBuilder)54 ImmutableBitSet (org.apache.calcite.util.ImmutableBitSet)52 HashMap (java.util.HashMap)47 AggregateCall (org.apache.calcite.rel.core.AggregateCall)36 List (java.util.List)35 HashSet (java.util.HashSet)32 Pair (org.apache.calcite.util.Pair)32 RexLiteral (org.apache.calcite.rex.RexLiteral)29 Map (java.util.Map)24 RelOptUtil (org.apache.calcite.plan.RelOptUtil)24 Set (java.util.Set)20 ImmutableList (com.google.common.collect.ImmutableList)19 LinkedHashMap (java.util.LinkedHashMap)19