Examples with StreamPhysicalExchange org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalExchange used on opensource projects

Example 1 with StreamPhysicalExchange

use of org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalExchange in project flink by apache.

the class WatermarkAssignerChangelogNormalizeTransposeRule method pushDownTransformedWatermark.

private RelNode pushDownTransformedWatermark(StreamPhysicalWatermarkAssigner watermark, StreamPhysicalCalc calc, StreamPhysicalChangelogNormalize changelogNormalize, StreamPhysicalExchange exchange, Mappings.TargetMapping calcMapping, RexBuilder rexBuilder) {
    Mappings.TargetMapping inversedMapping = calcMapping.inverse();
    final int newRowTimeFieldIndex = inversedMapping.getTargetOpt(watermark.rowtimeFieldIndex());
    // Updates watermark properties after push down before Calc
    // 1. rewrites watermark expression
    // 2. clears distribution
    // 3. updates row time field index
    RexNode newWatermarkExpr = watermark.watermarkExpr();
    if (watermark.watermarkExpr() != null) {
        newWatermarkExpr = RexUtil.apply(inversedMapping, watermark.watermarkExpr());
    }
    final RelNode newWatermark = watermark.copy(watermark.getTraitSet().plus(FlinkRelDistribution.DEFAULT()), exchange.getInput(), newRowTimeFieldIndex, newWatermarkExpr);
    final RelNode newChangelogNormalize = buildTreeInOrder(newWatermark, Tuple2.of(exchange, exchange.getTraitSet()), Tuple2.of(changelogNormalize, changelogNormalize.getTraitSet()));
    // Rewrites Calc program because the field type of row time
    // field is changed after watermark pushed down
    final RexProgram oldProgram = calc.getProgram();
    final RexProgramBuilder programBuilder = new RexProgramBuilder(newChangelogNormalize.getRowType(), rexBuilder);
    final Function<RexNode, RexNode> rexShuttle = e -> e.accept(new RexShuttle() {

        @Override
        public RexNode visitInputRef(RexInputRef inputRef) {
            if (inputRef.getIndex() == newRowTimeFieldIndex) {
                return RexInputRef.of(newRowTimeFieldIndex, newChangelogNormalize.getRowType());
            } else {
                return inputRef;
            }
        }
    });
    oldProgram.getNamedProjects().forEach(pair -> programBuilder.addProject(rexShuttle.apply(oldProgram.expandLocalRef(pair.left)), pair.right));
    if (oldProgram.getCondition() != null) {
        programBuilder.addCondition(rexShuttle.apply(oldProgram.expandLocalRef(oldProgram.getCondition())));
    }
    final RexProgram newProgram = programBuilder.getProgram();
    return calc.copy(calc.getTraitSet(), newChangelogNormalize, newProgram);
}

Example 2 with StreamPhysicalExchange

use of org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalExchange in project flink by apache.

the class WatermarkAssignerChangelogNormalizeTransposeRule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    final StreamPhysicalWatermarkAssigner watermark = call.rel(0);
    final RelNode node = call.rel(1);
    RelNode newTree;
    if (node instanceof StreamPhysicalCalc) {
        // with calc
        final StreamPhysicalCalc calc = call.rel(1);
        final StreamPhysicalChangelogNormalize changelogNormalize = call.rel(2);
        final StreamPhysicalExchange exchange = call.rel(3);
        final Mappings.TargetMapping calcMapping = buildMapping(calc.getProgram());
        final RelDistribution exchangeDistribution = exchange.getDistribution();
        final RelDistribution newExchangeDistribution = exchangeDistribution.apply(calcMapping);
        final boolean shuffleKeysAreKeptByCalc = newExchangeDistribution.getType() == exchangeDistribution.getType() && newExchangeDistribution.getKeys().size() == exchangeDistribution.getKeys().size();
        if (shuffleKeysAreKeptByCalc) {
            // Pushes down WatermarkAssigner/Calc as a whole if shuffle keys of
            // Exchange are all kept by Calc
            newTree = pushDownOriginalWatermarkAndCalc(watermark, calc, changelogNormalize, exchange, newExchangeDistribution);
        } else {
            // 1. Creates a new Calc which contains all shuffle keys
            // 2. Pushes down new WatermarkAssigner/new Calc
            // 3. Adds a top Calc to remove new added shuffle keys in step 1
            newTree = pushDownTransformedWatermarkAndCalc(watermark, calc, changelogNormalize, exchange, exchangeDistribution.getKeys(), calcMapping);
        }
    } else if (node instanceof StreamPhysicalChangelogNormalize) {
        // without calc
        final StreamPhysicalChangelogNormalize changelogNormalize = call.rel(1);
        final StreamPhysicalExchange exchange = call.rel(2);
        newTree = buildTreeInOrder(exchange.getInput(), // Clears distribution on new WatermarkAssigner
        Tuple2.of(watermark, watermark.getTraitSet().plus(FlinkRelDistribution.DEFAULT())), Tuple2.of(exchange, exchange.getTraitSet()), Tuple2.of(changelogNormalize, changelogNormalize.getTraitSet()));
    } else {
        throw new IllegalStateException(this.getClass().getName() + " matches a wrong relation tree: " + RelOptUtil.toString(watermark));
    }
    call.transformTo(newTree);
}

Example 3 with StreamPhysicalExchange

use of org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalExchange in project flink by apache.

the class PushFilterPastChangelogNormalizeRule method pushFiltersThroughChangelogNormalize.

/**
 * Pushes {@param primaryKeyPredicates} into the {@link StreamPhysicalChangelogNormalize}.
 */
private StreamPhysicalChangelogNormalize pushFiltersThroughChangelogNormalize(RelOptRuleCall call, List<RexNode> primaryKeyPredicates) {
    final StreamPhysicalChangelogNormalize changelogNormalize = call.rel(1);
    final StreamPhysicalExchange exchange = call.rel(2);
    if (primaryKeyPredicates.isEmpty()) {
        // There are no filters which can be pushed, so just return the existing node.
        return changelogNormalize;
    }
    final StreamPhysicalCalc pushedFiltersCalc = projectIdentityWithConditions(call.builder(), exchange.getInput(), primaryKeyPredicates);
    final StreamPhysicalExchange newExchange = (StreamPhysicalExchange) exchange.copy(exchange.getTraitSet(), Collections.singletonList(pushedFiltersCalc));
    return (StreamPhysicalChangelogNormalize) changelogNormalize.copy(changelogNormalize.getTraitSet(), Collections.singletonList(newExchange));
}

Example 4 with StreamPhysicalExchange

use of org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalExchange in project flink by apache.

the class WatermarkAssignerChangelogNormalizeTransposeRule method buildTreeInOrder.

/**
 * Build a new {@link RelNode} tree in the given nodes order which is in bottom-up direction.
 */
@SafeVarargs
private final RelNode buildTreeInOrder(RelNode leafNode, Tuple2<RelNode, RelTraitSet>... nodeAndTraits) {
    checkArgument(nodeAndTraits.length >= 1);
    RelNode inputNode = leafNode;
    RelNode currentNode = null;
    for (Tuple2<RelNode, RelTraitSet> nodeAndTrait : nodeAndTraits) {
        currentNode = nodeAndTrait.f0;
        if (currentNode instanceof StreamPhysicalExchange) {
            currentNode = ((StreamPhysicalExchange) currentNode).copy(nodeAndTrait.f1, inputNode, nodeAndTrait.f1.getTrait(FlinkRelDistributionTraitDef.INSTANCE()));
        } else {
            currentNode = currentNode.copy(nodeAndTrait.f1, Collections.singletonList(inputNode));
        }
        inputNode = currentNode;
    }
    return currentNode;
}