Examples with StreamPhysicalWatermarkAssigner org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalWatermarkAssigner used on opensource projects

Example 1 with StreamPhysicalWatermarkAssigner

use of org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalWatermarkAssigner in project flink by apache.

the class WatermarkAssignerChangelogNormalizeTransposeRule method pushDownTransformedWatermark.

private RelNode pushDownTransformedWatermark(StreamPhysicalWatermarkAssigner watermark, StreamPhysicalCalc calc, StreamPhysicalChangelogNormalize changelogNormalize, StreamPhysicalExchange exchange, Mappings.TargetMapping calcMapping, RexBuilder rexBuilder) {
    Mappings.TargetMapping inversedMapping = calcMapping.inverse();
    final int newRowTimeFieldIndex = inversedMapping.getTargetOpt(watermark.rowtimeFieldIndex());
    // Updates watermark properties after push down before Calc
    // 1. rewrites watermark expression
    // 2. clears distribution
    // 3. updates row time field index
    RexNode newWatermarkExpr = watermark.watermarkExpr();
    if (watermark.watermarkExpr() != null) {
        newWatermarkExpr = RexUtil.apply(inversedMapping, watermark.watermarkExpr());
    }
    final RelNode newWatermark = watermark.copy(watermark.getTraitSet().plus(FlinkRelDistribution.DEFAULT()), exchange.getInput(), newRowTimeFieldIndex, newWatermarkExpr);
    final RelNode newChangelogNormalize = buildTreeInOrder(newWatermark, Tuple2.of(exchange, exchange.getTraitSet()), Tuple2.of(changelogNormalize, changelogNormalize.getTraitSet()));
    // Rewrites Calc program because the field type of row time
    // field is changed after watermark pushed down
    final RexProgram oldProgram = calc.getProgram();
    final RexProgramBuilder programBuilder = new RexProgramBuilder(newChangelogNormalize.getRowType(), rexBuilder);
    final Function<RexNode, RexNode> rexShuttle = e -> e.accept(new RexShuttle() {

        @Override
        public RexNode visitInputRef(RexInputRef inputRef) {
            if (inputRef.getIndex() == newRowTimeFieldIndex) {
                return RexInputRef.of(newRowTimeFieldIndex, newChangelogNormalize.getRowType());
            } else {
                return inputRef;
            }
        }
    });
    oldProgram.getNamedProjects().forEach(pair -> programBuilder.addProject(rexShuttle.apply(oldProgram.expandLocalRef(pair.left)), pair.right));
    if (oldProgram.getCondition() != null) {
        programBuilder.addCondition(rexShuttle.apply(oldProgram.expandLocalRef(oldProgram.getCondition())));
    }
    final RexProgram newProgram = programBuilder.getProgram();
    return calc.copy(calc.getTraitSet(), newChangelogNormalize, newProgram);
}

Example 2 with StreamPhysicalWatermarkAssigner

use of org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalWatermarkAssigner in project flink by apache.

the class WatermarkAssignerChangelogNormalizeTransposeRule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    final StreamPhysicalWatermarkAssigner watermark = call.rel(0);
    final RelNode node = call.rel(1);
    RelNode newTree;
    if (node instanceof StreamPhysicalCalc) {
        // with calc
        final StreamPhysicalCalc calc = call.rel(1);
        final StreamPhysicalChangelogNormalize changelogNormalize = call.rel(2);
        final StreamPhysicalExchange exchange = call.rel(3);
        final Mappings.TargetMapping calcMapping = buildMapping(calc.getProgram());
        final RelDistribution exchangeDistribution = exchange.getDistribution();
        final RelDistribution newExchangeDistribution = exchangeDistribution.apply(calcMapping);
        final boolean shuffleKeysAreKeptByCalc = newExchangeDistribution.getType() == exchangeDistribution.getType() && newExchangeDistribution.getKeys().size() == exchangeDistribution.getKeys().size();
        if (shuffleKeysAreKeptByCalc) {
            // Pushes down WatermarkAssigner/Calc as a whole if shuffle keys of
            // Exchange are all kept by Calc
            newTree = pushDownOriginalWatermarkAndCalc(watermark, calc, changelogNormalize, exchange, newExchangeDistribution);
        } else {
            // 1. Creates a new Calc which contains all shuffle keys
            // 2. Pushes down new WatermarkAssigner/new Calc
            // 3. Adds a top Calc to remove new added shuffle keys in step 1
            newTree = pushDownTransformedWatermarkAndCalc(watermark, calc, changelogNormalize, exchange, exchangeDistribution.getKeys(), calcMapping);
        }
    } else if (node instanceof StreamPhysicalChangelogNormalize) {
        // without calc
        final StreamPhysicalChangelogNormalize changelogNormalize = call.rel(1);
        final StreamPhysicalExchange exchange = call.rel(2);
        newTree = buildTreeInOrder(exchange.getInput(), // Clears distribution on new WatermarkAssigner
        Tuple2.of(watermark, watermark.getTraitSet().plus(FlinkRelDistribution.DEFAULT())), Tuple2.of(exchange, exchange.getTraitSet()), Tuple2.of(changelogNormalize, changelogNormalize.getTraitSet()));
    } else {
        throw new IllegalStateException(this.getClass().getName() + " matches a wrong relation tree: " + RelOptUtil.toString(watermark));
    }
    call.transformTo(newTree);
}