Examples with StreamPhysicalChangelogNormalize org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalChangelogNormalize used on opensource projects

Example 1 with StreamPhysicalChangelogNormalize

use of org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalChangelogNormalize in project flink by apache.

the class WatermarkAssignerChangelogNormalizeTransposeRule method pushDownTransformedWatermark.

private RelNode pushDownTransformedWatermark(StreamPhysicalWatermarkAssigner watermark, StreamPhysicalCalc calc, StreamPhysicalChangelogNormalize changelogNormalize, StreamPhysicalExchange exchange, Mappings.TargetMapping calcMapping, RexBuilder rexBuilder) {
    Mappings.TargetMapping inversedMapping = calcMapping.inverse();
    final int newRowTimeFieldIndex = inversedMapping.getTargetOpt(watermark.rowtimeFieldIndex());
    // Updates watermark properties after push down before Calc
    // 1. rewrites watermark expression
    // 2. clears distribution
    // 3. updates row time field index
    RexNode newWatermarkExpr = watermark.watermarkExpr();
    if (watermark.watermarkExpr() != null) {
        newWatermarkExpr = RexUtil.apply(inversedMapping, watermark.watermarkExpr());
    }
    final RelNode newWatermark = watermark.copy(watermark.getTraitSet().plus(FlinkRelDistribution.DEFAULT()), exchange.getInput(), newRowTimeFieldIndex, newWatermarkExpr);
    final RelNode newChangelogNormalize = buildTreeInOrder(newWatermark, Tuple2.of(exchange, exchange.getTraitSet()), Tuple2.of(changelogNormalize, changelogNormalize.getTraitSet()));
    // Rewrites Calc program because the field type of row time
    // field is changed after watermark pushed down
    final RexProgram oldProgram = calc.getProgram();
    final RexProgramBuilder programBuilder = new RexProgramBuilder(newChangelogNormalize.getRowType(), rexBuilder);
    final Function<RexNode, RexNode> rexShuttle = e -> e.accept(new RexShuttle() {

        @Override
        public RexNode visitInputRef(RexInputRef inputRef) {
            if (inputRef.getIndex() == newRowTimeFieldIndex) {
                return RexInputRef.of(newRowTimeFieldIndex, newChangelogNormalize.getRowType());
            } else {
                return inputRef;
            }
        }
    });
    oldProgram.getNamedProjects().forEach(pair -> programBuilder.addProject(rexShuttle.apply(oldProgram.expandLocalRef(pair.left)), pair.right));
    if (oldProgram.getCondition() != null) {
        programBuilder.addCondition(rexShuttle.apply(oldProgram.expandLocalRef(oldProgram.getCondition())));
    }
    final RexProgram newProgram = programBuilder.getProgram();
    return calc.copy(calc.getTraitSet(), newChangelogNormalize, newProgram);
}

Example 2 with StreamPhysicalChangelogNormalize

use of org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalChangelogNormalize in project flink by apache.

the class WatermarkAssignerChangelogNormalizeTransposeRule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    final StreamPhysicalWatermarkAssigner watermark = call.rel(0);
    final RelNode node = call.rel(1);
    RelNode newTree;
    if (node instanceof StreamPhysicalCalc) {
        // with calc
        final StreamPhysicalCalc calc = call.rel(1);
        final StreamPhysicalChangelogNormalize changelogNormalize = call.rel(2);
        final StreamPhysicalExchange exchange = call.rel(3);
        final Mappings.TargetMapping calcMapping = buildMapping(calc.getProgram());
        final RelDistribution exchangeDistribution = exchange.getDistribution();
        final RelDistribution newExchangeDistribution = exchangeDistribution.apply(calcMapping);
        final boolean shuffleKeysAreKeptByCalc = newExchangeDistribution.getType() == exchangeDistribution.getType() && newExchangeDistribution.getKeys().size() == exchangeDistribution.getKeys().size();
        if (shuffleKeysAreKeptByCalc) {
            // Pushes down WatermarkAssigner/Calc as a whole if shuffle keys of
            // Exchange are all kept by Calc
            newTree = pushDownOriginalWatermarkAndCalc(watermark, calc, changelogNormalize, exchange, newExchangeDistribution);
        } else {
            // 1. Creates a new Calc which contains all shuffle keys
            // 2. Pushes down new WatermarkAssigner/new Calc
            // 3. Adds a top Calc to remove new added shuffle keys in step 1
            newTree = pushDownTransformedWatermarkAndCalc(watermark, calc, changelogNormalize, exchange, exchangeDistribution.getKeys(), calcMapping);
        }
    } else if (node instanceof StreamPhysicalChangelogNormalize) {
        // without calc
        final StreamPhysicalChangelogNormalize changelogNormalize = call.rel(1);
        final StreamPhysicalExchange exchange = call.rel(2);
        newTree = buildTreeInOrder(exchange.getInput(), // Clears distribution on new WatermarkAssigner
        Tuple2.of(watermark, watermark.getTraitSet().plus(FlinkRelDistribution.DEFAULT())), Tuple2.of(exchange, exchange.getTraitSet()), Tuple2.of(changelogNormalize, changelogNormalize.getTraitSet()));
    } else {
        throw new IllegalStateException(this.getClass().getName() + " matches a wrong relation tree: " + RelOptUtil.toString(watermark));
    }
    call.transformTo(newTree);
}

Example 3 with StreamPhysicalChangelogNormalize

use of org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalChangelogNormalize in project flink by apache.

the class PushFilterPastChangelogNormalizeRule method pushFiltersThroughChangelogNormalize.

/**
 * Pushes {@param primaryKeyPredicates} into the {@link StreamPhysicalChangelogNormalize}.
 */
private StreamPhysicalChangelogNormalize pushFiltersThroughChangelogNormalize(RelOptRuleCall call, List<RexNode> primaryKeyPredicates) {
    final StreamPhysicalChangelogNormalize changelogNormalize = call.rel(1);
    final StreamPhysicalExchange exchange = call.rel(2);
    if (primaryKeyPredicates.isEmpty()) {
        // There are no filters which can be pushed, so just return the existing node.
        return changelogNormalize;
    }
    final StreamPhysicalCalc pushedFiltersCalc = projectIdentityWithConditions(call.builder(), exchange.getInput(), primaryKeyPredicates);
    final StreamPhysicalExchange newExchange = (StreamPhysicalExchange) exchange.copy(exchange.getTraitSet(), Collections.singletonList(pushedFiltersCalc));
    return (StreamPhysicalChangelogNormalize) changelogNormalize.copy(changelogNormalize.getTraitSet(), Collections.singletonList(newExchange));
}

Example 4 with StreamPhysicalChangelogNormalize

use of org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalChangelogNormalize in project flink by apache.

the class PushFilterPastChangelogNormalizeRule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    final StreamPhysicalCalc calc = call.rel(0);
    final StreamPhysicalChangelogNormalize changelogNormalize = call.rel(1);
    final RexProgram program = calc.getProgram();
    final RexNode condition = RexUtil.toCnf(call.builder().getRexBuilder(), program.expandLocalRef(program.getCondition()));
    final Set<Integer> primaryKeyIndices = IntStream.of(changelogNormalize.uniqueKeys()).boxed().collect(Collectors.toSet());
    // Determine which filters can be pushed (= involve only primary key columns)
    final List<RexNode> primaryKeyPredicates = new ArrayList<>();
    final List<RexNode> otherPredicates = new ArrayList<>();
    partitionPrimaryKeyPredicates(RelOptUtil.conjunctions(condition), primaryKeyIndices, primaryKeyPredicates, otherPredicates);
    // Construct a new ChangelogNormalize which has primary key filters pushed into it
    final StreamPhysicalChangelogNormalize newChangelogNormalize = pushFiltersThroughChangelogNormalize(call, primaryKeyPredicates);
    // Retain only filters which haven't been pushed
    transformWithRemainingPredicates(call, newChangelogNormalize, otherPredicates);
}