Search in sources :

Example 1 with StreamPhysicalChangelogNormalize

use of org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalChangelogNormalize in project flink by apache.

the class WatermarkAssignerChangelogNormalizeTransposeRule method pushDownTransformedWatermark.

private RelNode pushDownTransformedWatermark(StreamPhysicalWatermarkAssigner watermark, StreamPhysicalCalc calc, StreamPhysicalChangelogNormalize changelogNormalize, StreamPhysicalExchange exchange, Mappings.TargetMapping calcMapping, RexBuilder rexBuilder) {
    Mappings.TargetMapping inversedMapping = calcMapping.inverse();
    final int newRowTimeFieldIndex = inversedMapping.getTargetOpt(watermark.rowtimeFieldIndex());
    // Updates watermark properties after push down before Calc
    // 1. rewrites watermark expression
    // 2. clears distribution
    // 3. updates row time field index
    RexNode newWatermarkExpr = watermark.watermarkExpr();
    if (watermark.watermarkExpr() != null) {
        newWatermarkExpr = RexUtil.apply(inversedMapping, watermark.watermarkExpr());
    }
    final RelNode newWatermark = watermark.copy(watermark.getTraitSet().plus(FlinkRelDistribution.DEFAULT()), exchange.getInput(), newRowTimeFieldIndex, newWatermarkExpr);
    final RelNode newChangelogNormalize = buildTreeInOrder(newWatermark, Tuple2.of(exchange, exchange.getTraitSet()), Tuple2.of(changelogNormalize, changelogNormalize.getTraitSet()));
    // Rewrites Calc program because the field type of row time
    // field is changed after watermark pushed down
    final RexProgram oldProgram = calc.getProgram();
    final RexProgramBuilder programBuilder = new RexProgramBuilder(newChangelogNormalize.getRowType(), rexBuilder);
    final Function<RexNode, RexNode> rexShuttle = e -> e.accept(new RexShuttle() {

        @Override
        public RexNode visitInputRef(RexInputRef inputRef) {
            if (inputRef.getIndex() == newRowTimeFieldIndex) {
                return RexInputRef.of(newRowTimeFieldIndex, newChangelogNormalize.getRowType());
            } else {
                return inputRef;
            }
        }
    });
    oldProgram.getNamedProjects().forEach(pair -> programBuilder.addProject(rexShuttle.apply(oldProgram.expandLocalRef(pair.left)), pair.right));
    if (oldProgram.getCondition() != null) {
        programBuilder.addCondition(rexShuttle.apply(oldProgram.expandLocalRef(oldProgram.getCondition())));
    }
    final RexProgram newProgram = programBuilder.getProgram();
    return calc.copy(calc.getTraitSet(), newChangelogNormalize, newProgram);
}
Also used : Mappings(org.apache.calcite.util.mapping.Mappings) RexProgram(org.apache.calcite.rex.RexProgram) StreamPhysicalExchange(org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalExchange) Tuple2(org.apache.flink.api.java.tuple.Tuple2) HashMap(java.util.HashMap) FlinkRelDistribution(org.apache.flink.table.planner.plan.trait.FlinkRelDistribution) RelOptUtil(org.apache.calcite.plan.RelOptUtil) Function(java.util.function.Function) ArrayList(java.util.ArrayList) RexUtil(org.apache.calcite.rex.RexUtil) StreamPhysicalChangelogNormalize(org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalChangelogNormalize) RexNode(org.apache.calcite.rex.RexNode) Map(java.util.Map) RelTraitSet(org.apache.calcite.plan.RelTraitSet) FlinkRelDistributionTraitDef(org.apache.flink.table.planner.plan.trait.FlinkRelDistributionTraitDef) RelDataType(org.apache.calcite.rel.type.RelDataType) RexBuilder(org.apache.calcite.rex.RexBuilder) StreamPhysicalCalc(org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalCalc) RelRule(org.apache.calcite.plan.RelRule) RelNode(org.apache.calcite.rel.RelNode) RelOptRuleCall(org.apache.calcite.plan.RelOptRuleCall) RowTypeUtils(org.apache.flink.table.planner.typeutils.RowTypeUtils) RexInputRef(org.apache.calcite.rex.RexInputRef) RelOptRule(org.apache.calcite.plan.RelOptRule) RexProgramBuilder(org.apache.calcite.rex.RexProgramBuilder) List(java.util.List) Preconditions.checkArgument(org.apache.flink.util.Preconditions.checkArgument) RexLocalRef(org.apache.calcite.rex.RexLocalRef) RelDistribution(org.apache.calcite.rel.RelDistribution) RelDataTypeField(org.apache.calcite.rel.type.RelDataTypeField) StreamPhysicalWatermarkAssigner(org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalWatermarkAssigner) RexShuttle(org.apache.calcite.rex.RexShuttle) Collections(java.util.Collections) RexShuttle(org.apache.calcite.rex.RexShuttle) Mappings(org.apache.calcite.util.mapping.Mappings) RelNode(org.apache.calcite.rel.RelNode) RexProgram(org.apache.calcite.rex.RexProgram) RexInputRef(org.apache.calcite.rex.RexInputRef) RexProgramBuilder(org.apache.calcite.rex.RexProgramBuilder) RexNode(org.apache.calcite.rex.RexNode)

Example 2 with StreamPhysicalChangelogNormalize

use of org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalChangelogNormalize in project flink by apache.

the class WatermarkAssignerChangelogNormalizeTransposeRule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    final StreamPhysicalWatermarkAssigner watermark = call.rel(0);
    final RelNode node = call.rel(1);
    RelNode newTree;
    if (node instanceof StreamPhysicalCalc) {
        // with calc
        final StreamPhysicalCalc calc = call.rel(1);
        final StreamPhysicalChangelogNormalize changelogNormalize = call.rel(2);
        final StreamPhysicalExchange exchange = call.rel(3);
        final Mappings.TargetMapping calcMapping = buildMapping(calc.getProgram());
        final RelDistribution exchangeDistribution = exchange.getDistribution();
        final RelDistribution newExchangeDistribution = exchangeDistribution.apply(calcMapping);
        final boolean shuffleKeysAreKeptByCalc = newExchangeDistribution.getType() == exchangeDistribution.getType() && newExchangeDistribution.getKeys().size() == exchangeDistribution.getKeys().size();
        if (shuffleKeysAreKeptByCalc) {
            // Pushes down WatermarkAssigner/Calc as a whole if shuffle keys of
            // Exchange are all kept by Calc
            newTree = pushDownOriginalWatermarkAndCalc(watermark, calc, changelogNormalize, exchange, newExchangeDistribution);
        } else {
            // 1. Creates a new Calc which contains all shuffle keys
            // 2. Pushes down new WatermarkAssigner/new Calc
            // 3. Adds a top Calc to remove new added shuffle keys in step 1
            newTree = pushDownTransformedWatermarkAndCalc(watermark, calc, changelogNormalize, exchange, exchangeDistribution.getKeys(), calcMapping);
        }
    } else if (node instanceof StreamPhysicalChangelogNormalize) {
        // without calc
        final StreamPhysicalChangelogNormalize changelogNormalize = call.rel(1);
        final StreamPhysicalExchange exchange = call.rel(2);
        newTree = buildTreeInOrder(exchange.getInput(), // Clears distribution on new WatermarkAssigner
        Tuple2.of(watermark, watermark.getTraitSet().plus(FlinkRelDistribution.DEFAULT())), Tuple2.of(exchange, exchange.getTraitSet()), Tuple2.of(changelogNormalize, changelogNormalize.getTraitSet()));
    } else {
        throw new IllegalStateException(this.getClass().getName() + " matches a wrong relation tree: " + RelOptUtil.toString(watermark));
    }
    call.transformTo(newTree);
}
Also used : StreamPhysicalChangelogNormalize(org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalChangelogNormalize) StreamPhysicalWatermarkAssigner(org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalWatermarkAssigner) RelNode(org.apache.calcite.rel.RelNode) Mappings(org.apache.calcite.util.mapping.Mappings) StreamPhysicalCalc(org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalCalc) StreamPhysicalExchange(org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalExchange) FlinkRelDistribution(org.apache.flink.table.planner.plan.trait.FlinkRelDistribution) RelDistribution(org.apache.calcite.rel.RelDistribution)

Example 3 with StreamPhysicalChangelogNormalize

use of org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalChangelogNormalize in project flink by apache.

the class PushFilterPastChangelogNormalizeRule method pushFiltersThroughChangelogNormalize.

/**
 * Pushes {@param primaryKeyPredicates} into the {@link StreamPhysicalChangelogNormalize}.
 */
private StreamPhysicalChangelogNormalize pushFiltersThroughChangelogNormalize(RelOptRuleCall call, List<RexNode> primaryKeyPredicates) {
    final StreamPhysicalChangelogNormalize changelogNormalize = call.rel(1);
    final StreamPhysicalExchange exchange = call.rel(2);
    if (primaryKeyPredicates.isEmpty()) {
        // There are no filters which can be pushed, so just return the existing node.
        return changelogNormalize;
    }
    final StreamPhysicalCalc pushedFiltersCalc = projectIdentityWithConditions(call.builder(), exchange.getInput(), primaryKeyPredicates);
    final StreamPhysicalExchange newExchange = (StreamPhysicalExchange) exchange.copy(exchange.getTraitSet(), Collections.singletonList(pushedFiltersCalc));
    return (StreamPhysicalChangelogNormalize) changelogNormalize.copy(changelogNormalize.getTraitSet(), Collections.singletonList(newExchange));
}
Also used : StreamPhysicalChangelogNormalize(org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalChangelogNormalize) StreamPhysicalExchange(org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalExchange) StreamPhysicalCalc(org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalCalc)

Example 4 with StreamPhysicalChangelogNormalize

use of org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalChangelogNormalize in project flink by apache.

the class PushFilterPastChangelogNormalizeRule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    final StreamPhysicalCalc calc = call.rel(0);
    final StreamPhysicalChangelogNormalize changelogNormalize = call.rel(1);
    final RexProgram program = calc.getProgram();
    final RexNode condition = RexUtil.toCnf(call.builder().getRexBuilder(), program.expandLocalRef(program.getCondition()));
    final Set<Integer> primaryKeyIndices = IntStream.of(changelogNormalize.uniqueKeys()).boxed().collect(Collectors.toSet());
    // Determine which filters can be pushed (= involve only primary key columns)
    final List<RexNode> primaryKeyPredicates = new ArrayList<>();
    final List<RexNode> otherPredicates = new ArrayList<>();
    partitionPrimaryKeyPredicates(RelOptUtil.conjunctions(condition), primaryKeyIndices, primaryKeyPredicates, otherPredicates);
    // Construct a new ChangelogNormalize which has primary key filters pushed into it
    final StreamPhysicalChangelogNormalize newChangelogNormalize = pushFiltersThroughChangelogNormalize(call, primaryKeyPredicates);
    // Retain only filters which haven't been pushed
    transformWithRemainingPredicates(call, newChangelogNormalize, otherPredicates);
}
Also used : StreamPhysicalChangelogNormalize(org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalChangelogNormalize) RexProgram(org.apache.calcite.rex.RexProgram) StreamPhysicalCalc(org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalCalc) ArrayList(java.util.ArrayList) RexNode(org.apache.calcite.rex.RexNode)

Aggregations

StreamPhysicalCalc (org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalCalc)4 StreamPhysicalChangelogNormalize (org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalChangelogNormalize)4 StreamPhysicalExchange (org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalExchange)3 ArrayList (java.util.ArrayList)2 RelDistribution (org.apache.calcite.rel.RelDistribution)2 RelNode (org.apache.calcite.rel.RelNode)2 RexNode (org.apache.calcite.rex.RexNode)2 RexProgram (org.apache.calcite.rex.RexProgram)2 Mappings (org.apache.calcite.util.mapping.Mappings)2 StreamPhysicalWatermarkAssigner (org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalWatermarkAssigner)2 FlinkRelDistribution (org.apache.flink.table.planner.plan.trait.FlinkRelDistribution)2 Collections (java.util.Collections)1 HashMap (java.util.HashMap)1 List (java.util.List)1 Map (java.util.Map)1 Function (java.util.function.Function)1 RelOptRule (org.apache.calcite.plan.RelOptRule)1 RelOptRuleCall (org.apache.calcite.plan.RelOptRuleCall)1 RelOptUtil (org.apache.calcite.plan.RelOptUtil)1 RelRule (org.apache.calcite.plan.RelRule)1