Search in sources :

Example 1 with StreamExecNode

use of org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecNode in project flink by apache.

the class ForwardHashExchangeProcessor method process.

@Override
public ExecNodeGraph process(ExecNodeGraph execGraph, ProcessorContext context) {
    if (execGraph.getRootNodes().get(0) instanceof StreamExecNode) {
        throw new TableException("StreamExecNode is not supported yet");
    }
    if (!context.getPlanner().getExecEnv().getConfig().isDynamicGraph()) {
        return execGraph;
    }
    ExecNodeVisitor visitor = new AbstractExecNodeExactlyOnceVisitor() {

        @Override
        protected void visitNode(ExecNode<?> node) {
            visitInputs(node);
            if (node instanceof CommonExecExchange) {
                return;
            }
            boolean changed = false;
            List<ExecEdge> newEdges = new ArrayList<>(node.getInputEdges());
            for (int i = 0; i < node.getInputProperties().size(); ++i) {
                InputProperty inputProperty = node.getInputProperties().get(i);
                RequiredDistribution requiredDistribution = inputProperty.getRequiredDistribution();
                ExecEdge edge = node.getInputEdges().get(i);
                if (requiredDistribution.getType() == DistributionType.SINGLETON) {
                    if (!hasExchangeInput(edge) && isInputSortedNode(node)) {
                        // if operation chaining is disabled, this could mark sure the
                        // sort node and its output can also be connected by
                        // ForwardPartitioner
                        ExecEdge newEdge = addExchangeAndReconnectEdge(edge, inputProperty, true);
                        newEdges.set(i, newEdge);
                        changed = true;
                    }
                    continue;
                }
                if (requiredDistribution.getType() != DistributionType.HASH) {
                    continue;
                }
                if (!hasExchangeInput(edge)) {
                    ExecEdge newEdge;
                    if (isInputSortedNode(node)) {
                        if (hasSortInputForInputSortedNode(node)) {
                            // add Exchange with keep_input_as_is distribution as the
                            // input of Sort
                            ExecNode<?> sort = edge.getSource();
                            ExecEdge newEdgeOfSort = addExchangeAndReconnectEdge(sort.getInputEdges().get(0), inputProperty, false);
                            sort.setInputEdges(Collections.singletonList(newEdgeOfSort));
                        }
                        // if operation chaining is disabled, this could mark sure the
                        // sort node and its output can also be connected by
                        // ForwardPartitioner
                        newEdge = addExchangeAndReconnectEdge(edge, inputProperty, true);
                    } else {
                        // add Exchange with keep_input_as_is distribution as the input
                        // of the node
                        newEdge = addExchangeAndReconnectEdge(edge, inputProperty, false);
                        updateOriginalEdgeInMultipleInput(node, i, (BatchExecExchange) newEdge.getSource());
                    }
                    // update the edge
                    newEdges.set(i, newEdge);
                    changed = true;
                } else if (hasSortInputForInputSortedNode(node)) {
                    // if operation chaining is disabled, this could mark sure the sort
                    // node and its output can also be connected by ForwardPartitioner
                    ExecEdge newEdge = addExchangeAndReconnectEdge(edge, inputProperty, true);
                    newEdges.set(i, newEdge);
                    changed = true;
                }
            }
            if (changed) {
                node.setInputEdges(newEdges);
            }
        }
    };
    execGraph.getRootNodes().forEach(s -> s.accept(visitor));
    return execGraph;
}
Also used : AbstractExecNodeExactlyOnceVisitor(org.apache.flink.table.planner.plan.nodes.exec.visitor.AbstractExecNodeExactlyOnceVisitor) RequiredDistribution(org.apache.flink.table.planner.plan.nodes.exec.InputProperty.RequiredDistribution) TableException(org.apache.flink.table.api.TableException) ExecEdge(org.apache.flink.table.planner.plan.nodes.exec.ExecEdge) InputProperty(org.apache.flink.table.planner.plan.nodes.exec.InputProperty) ArrayList(java.util.ArrayList) StreamExecNode(org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecNode) CommonExecExchange(org.apache.flink.table.planner.plan.nodes.exec.common.CommonExecExchange) ExecNodeVisitor(org.apache.flink.table.planner.plan.nodes.exec.visitor.ExecNodeVisitor) InputSortedExecNode(org.apache.flink.table.planner.plan.nodes.exec.batch.InputSortedExecNode) ExecNode(org.apache.flink.table.planner.plan.nodes.exec.ExecNode) StreamExecNode(org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecNode)

Aggregations

ArrayList (java.util.ArrayList)1 TableException (org.apache.flink.table.api.TableException)1 ExecEdge (org.apache.flink.table.planner.plan.nodes.exec.ExecEdge)1 ExecNode (org.apache.flink.table.planner.plan.nodes.exec.ExecNode)1 InputProperty (org.apache.flink.table.planner.plan.nodes.exec.InputProperty)1 RequiredDistribution (org.apache.flink.table.planner.plan.nodes.exec.InputProperty.RequiredDistribution)1 InputSortedExecNode (org.apache.flink.table.planner.plan.nodes.exec.batch.InputSortedExecNode)1 CommonExecExchange (org.apache.flink.table.planner.plan.nodes.exec.common.CommonExecExchange)1 StreamExecNode (org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecNode)1 AbstractExecNodeExactlyOnceVisitor (org.apache.flink.table.planner.plan.nodes.exec.visitor.AbstractExecNodeExactlyOnceVisitor)1 ExecNodeVisitor (org.apache.flink.table.planner.plan.nodes.exec.visitor.ExecNodeVisitor)1