Examples with ExecEdge org.apache.flink.table.planner.plan.nodes.exec.ExecEdge used on opensource projects

Example 41 with ExecEdge

use of org.apache.flink.table.planner.plan.nodes.exec.ExecEdge in project flink by apache.

the class MultipleInputNodeCreationProcessor method createBatchMultipleInputNode.

private BatchExecMultipleInput createBatchMultipleInputNode(MultipleInputGroup group, List<Tuple3<ExecNode<?>, InputProperty, ExecEdge>> inputs) {
    // first calculate the input orders using InputPriorityConflictResolver
    Set<ExecNode<?>> inputSet = new HashSet<>();
    for (Tuple3<ExecNode<?>, InputProperty, ExecEdge> tuple3 : inputs) {
        inputSet.add(tuple3.f0);
    }
    InputOrderCalculator calculator = new InputOrderCalculator(group.root.execNode, inputSet, InputProperty.DamBehavior.BLOCKING);
    Map<ExecNode<?>, Integer> inputOrderMap = calculator.calculate();
    // then create input rels and edges with the input orders
    ExecNode<?> rootNode = group.root.execNode;
    List<ExecNode<?>> inputNodes = new ArrayList<>();
    List<InputProperty> inputProperties = new ArrayList<>();
    List<ExecEdge> originalEdges = new ArrayList<>();
    for (Tuple3<ExecNode<?>, InputProperty, ExecEdge> tuple3 : inputs) {
        ExecNode<?> inputNode = tuple3.f0;
        InputProperty originalInputEdge = tuple3.f1;
        ExecEdge edge = tuple3.f2;
        inputNodes.add(inputNode);
        inputProperties.add(InputProperty.builder().requiredDistribution(originalInputEdge.getRequiredDistribution()).damBehavior(originalInputEdge.getDamBehavior()).priority(inputOrderMap.get(inputNode)).build());
        originalEdges.add(edge);
    }
    String description = ExecNodeUtil.getMultipleInputDescription(rootNode, inputNodes, inputProperties);
    BatchExecMultipleInput multipleInput = new BatchExecMultipleInput(inputProperties, rootNode, originalEdges, description);
    List<ExecEdge> inputEdges = new ArrayList<>(inputNodes.size());
    for (ExecNode<?> inputNode : inputNodes) {
        inputEdges.add(ExecEdge.builder().source(inputNode).target(multipleInput).build());
    }
    multipleInput.setInputEdges(inputEdges);
    return multipleInput;
}

Example 42 with ExecEdge

use of org.apache.flink.table.planner.plan.nodes.exec.ExecEdge in project flink by apache.

the class ForwardHashExchangeProcessor method updateOriginalEdgeInMultipleInput.

/**
 * Add new exchange node between the input node and the target node for the given edge, and
 * reconnect the edges. So that the transformations can be connected correctly.
 */
private void updateOriginalEdgeInMultipleInput(BatchExecMultipleInput multipleInput, int edgeIdx, BatchExecExchange newExchange) {
    ExecEdge originalEdge = multipleInput.getOriginalEdges().get(edgeIdx);
    ExecNode<?> inputNode = originalEdge.getSource();
    ExecNode<?> targetNode = originalEdge.getTarget();
    int edgeIdxInTargetNode = targetNode.getInputEdges().indexOf(originalEdge);
    checkArgument(edgeIdxInTargetNode >= 0);
    List<ExecEdge> newEdges = new ArrayList<>(targetNode.getInputEdges());
    // connect input node to new exchange node
    ExecEdge newEdge1 = new ExecEdge(inputNode, newExchange, originalEdge.getShuffle(), originalEdge.getExchangeMode());
    newExchange.setInputEdges(Collections.singletonList(newEdge1));
    // connect new exchange node to target node
    ExecEdge newEdge2 = new ExecEdge(newExchange, targetNode, originalEdge.getShuffle(), originalEdge.getExchangeMode());
    newEdges.set(edgeIdxInTargetNode, newEdge2);
    targetNode.setInputEdges(newEdges);
}

Example 43 with ExecEdge

use of org.apache.flink.table.planner.plan.nodes.exec.ExecEdge in project flink by apache.

the class ForwardHashExchangeProcessor method addExchangeAndReconnectEdge.

// TODO This implementation should be updated once FLINK-21224 is finished.
private ExecEdge addExchangeAndReconnectEdge(ExecEdge edge, InputProperty inputProperty, boolean strict) {
    ExecNode<?> target = edge.getTarget();
    ExecNode<?> source = edge.getSource();
    if (source instanceof CommonExecExchange) {
        return edge;
    }
    // only Calc, Correlate and Sort can propagate sort property and distribution property
    if (source instanceof BatchExecCalc || source instanceof BatchExecPythonCalc || source instanceof BatchExecSort || source instanceof BatchExecCorrelate || source instanceof BatchExecPythonCorrelate) {
        ExecEdge newEdge = addExchangeAndReconnectEdge(source.getInputEdges().get(0), inputProperty, strict);
        source.setInputEdges(Collections.singletonList(newEdge));
    }
    BatchExecExchange exchange = createExchangeWithKeepInputAsIsDistribution(inputProperty, strict, (RowType) edge.getOutputType());
    ExecEdge newEdge = new ExecEdge(source, exchange, edge.getShuffle(), edge.getExchangeMode());
    exchange.setInputEdges(Collections.singletonList(newEdge));
    return new ExecEdge(exchange, target, edge.getShuffle(), edge.getExchangeMode());
}

Example 44 with ExecEdge

use of org.apache.flink.table.planner.plan.nodes.exec.ExecEdge in project flink by apache.

the class ForwardHashExchangeProcessor method process.

@Override
public ExecNodeGraph process(ExecNodeGraph execGraph, ProcessorContext context) {
    if (execGraph.getRootNodes().get(0) instanceof StreamExecNode) {
        throw new TableException("StreamExecNode is not supported yet");
    }
    if (!context.getPlanner().getExecEnv().getConfig().isDynamicGraph()) {
        return execGraph;
    }
    ExecNodeVisitor visitor = new AbstractExecNodeExactlyOnceVisitor() {

        @Override
        protected void visitNode(ExecNode<?> node) {
            visitInputs(node);
            if (node instanceof CommonExecExchange) {
                return;
            }
            boolean changed = false;
            List<ExecEdge> newEdges = new ArrayList<>(node.getInputEdges());
            for (int i = 0; i < node.getInputProperties().size(); ++i) {
                InputProperty inputProperty = node.getInputProperties().get(i);
                RequiredDistribution requiredDistribution = inputProperty.getRequiredDistribution();
                ExecEdge edge = node.getInputEdges().get(i);
                if (requiredDistribution.getType() == DistributionType.SINGLETON) {
                    if (!hasExchangeInput(edge) && isInputSortedNode(node)) {
                        // if operation chaining is disabled, this could mark sure the
                        // sort node and its output can also be connected by
                        // ForwardPartitioner
                        ExecEdge newEdge = addExchangeAndReconnectEdge(edge, inputProperty, true);
                        newEdges.set(i, newEdge);
                        changed = true;
                    }
                    continue;
                }
                if (requiredDistribution.getType() != DistributionType.HASH) {
                    continue;
                }
                if (!hasExchangeInput(edge)) {
                    ExecEdge newEdge;
                    if (isInputSortedNode(node)) {
                        if (hasSortInputForInputSortedNode(node)) {
                            // add Exchange with keep_input_as_is distribution as the
                            // input of Sort
                            ExecNode<?> sort = edge.getSource();
                            ExecEdge newEdgeOfSort = addExchangeAndReconnectEdge(sort.getInputEdges().get(0), inputProperty, false);
                            sort.setInputEdges(Collections.singletonList(newEdgeOfSort));
                        }
                        // if operation chaining is disabled, this could mark sure the
                        // sort node and its output can also be connected by
                        // ForwardPartitioner
                        newEdge = addExchangeAndReconnectEdge(edge, inputProperty, true);
                    } else {
                        // add Exchange with keep_input_as_is distribution as the input
                        // of the node
                        newEdge = addExchangeAndReconnectEdge(edge, inputProperty, false);
                        updateOriginalEdgeInMultipleInput(node, i, (BatchExecExchange) newEdge.getSource());
                    }
                    // update the edge
                    newEdges.set(i, newEdge);
                    changed = true;
                } else if (hasSortInputForInputSortedNode(node)) {
                    // if operation chaining is disabled, this could mark sure the sort
                    // node and its output can also be connected by ForwardPartitioner
                    ExecEdge newEdge = addExchangeAndReconnectEdge(edge, inputProperty, true);
                    newEdges.set(i, newEdge);
                    changed = true;
                }
            }
            if (changed) {
                node.setInputEdges(newEdges);
            }
        }
    };
    execGraph.getRootNodes().forEach(s -> s.accept(visitor));
    return execGraph;
}

Example 45 with ExecEdge

use of org.apache.flink.table.planner.plan.nodes.exec.ExecEdge in project flink by apache.

the class InputPriorityConflictResolver method createExchange.

private BatchExecExchange createExchange(ExecNode<?> node, int idx) {
    ExecNode<?> inputNode = node.getInputEdges().get(idx).getSource();
    InputProperty inputProperty = node.getInputProperties().get(idx);
    InputProperty.RequiredDistribution requiredDistribution = inputProperty.getRequiredDistribution();
    if (requiredDistribution.getType() == InputProperty.DistributionType.BROADCAST) {
        // should not occur
        throw new IllegalStateException("Trying to resolve input priority conflict on broadcast side. This is not expected.");
    }
    InputProperty newInputProperty = InputProperty.builder().requiredDistribution(requiredDistribution).priority(inputProperty.getPriority()).damBehavior(getDamBehavior()).build();
    BatchExecExchange exchange = new BatchExecExchange(newInputProperty, (RowType) inputNode.getOutputType(), "Exchange");
    exchange.setRequiredExchangeMode(exchangeMode);
    ExecEdge execEdge = ExecEdge.builder().source(inputNode).target(exchange).build();
    exchange.setInputEdges(Collections.singletonList(execEdge));
    return exchange;
}