Examples with ExecNodeVisitor org.apache.flink.table.planner.plan.nodes.exec.visitor.ExecNodeVisitor used on opensource projects

Example 1 with ExecNodeVisitor

use of org.apache.flink.table.planner.plan.nodes.exec.visitor.ExecNodeVisitor in project flink by apache.

the class JsonPlanGraph method fromExecNodeGraph.

static JsonPlanGraph fromExecNodeGraph(ExecNodeGraph execGraph) {
    final List<ExecNode<?>> allNodes = new ArrayList<>();
    final List<JsonPlanEdge> allEdges = new ArrayList<>();
    final Set<Integer> nodesIds = new HashSet<>();
    // for quick search
    final Set<ExecNode<?>> visitedNodes = Sets.newIdentityHashSet();
    // visit the nodes as topological ordering
    final ExecNodeVisitor visitor = new ExecNodeVisitorImpl() {

        @Override
        public void visit(ExecNode<?> node) {
            if (visitedNodes.contains(node)) {
                return;
            }
            super.visitInputs(node);
            final int id = node.getId();
            if (nodesIds.contains(id)) {
                throw new TableException(String.format("The id: %s is not unique for ExecNode: %s.\nplease check it.", id, node.getDescription()));
            }
            allNodes.add(node);
            nodesIds.add(id);
            visitedNodes.add(node);
            for (ExecEdge execEdge : node.getInputEdges()) {
                allEdges.add(JsonPlanEdge.fromExecEdge(execEdge));
            }
        }
    };
    execGraph.getRootNodes().forEach(visitor::visit);
    checkArgument(allNodes.size() == nodesIds.size());
    return new JsonPlanGraph(execGraph.getFlinkVersion(), allNodes, allEdges);
}

Example 2 with ExecNodeVisitor

use of org.apache.flink.table.planner.plan.nodes.exec.visitor.ExecNodeVisitor in project flink by apache.

the class ForwardHashExchangeProcessor method process.

@Override
public ExecNodeGraph process(ExecNodeGraph execGraph, ProcessorContext context) {
    if (execGraph.getRootNodes().get(0) instanceof StreamExecNode) {
        throw new TableException("StreamExecNode is not supported yet");
    }
    if (!context.getPlanner().getExecEnv().getConfig().isDynamicGraph()) {
        return execGraph;
    }
    ExecNodeVisitor visitor = new AbstractExecNodeExactlyOnceVisitor() {

        @Override
        protected void visitNode(ExecNode<?> node) {
            visitInputs(node);
            if (node instanceof CommonExecExchange) {
                return;
            }
            boolean changed = false;
            List<ExecEdge> newEdges = new ArrayList<>(node.getInputEdges());
            for (int i = 0; i < node.getInputProperties().size(); ++i) {
                InputProperty inputProperty = node.getInputProperties().get(i);
                RequiredDistribution requiredDistribution = inputProperty.getRequiredDistribution();
                ExecEdge edge = node.getInputEdges().get(i);
                if (requiredDistribution.getType() == DistributionType.SINGLETON) {
                    if (!hasExchangeInput(edge) && isInputSortedNode(node)) {
                        // if operation chaining is disabled, this could mark sure the
                        // sort node and its output can also be connected by
                        // ForwardPartitioner
                        ExecEdge newEdge = addExchangeAndReconnectEdge(edge, inputProperty, true);
                        newEdges.set(i, newEdge);
                        changed = true;
                    }
                    continue;
                }
                if (requiredDistribution.getType() != DistributionType.HASH) {
                    continue;
                }
                if (!hasExchangeInput(edge)) {
                    ExecEdge newEdge;
                    if (isInputSortedNode(node)) {
                        if (hasSortInputForInputSortedNode(node)) {
                            // add Exchange with keep_input_as_is distribution as the
                            // input of Sort
                            ExecNode<?> sort = edge.getSource();
                            ExecEdge newEdgeOfSort = addExchangeAndReconnectEdge(sort.getInputEdges().get(0), inputProperty, false);
                            sort.setInputEdges(Collections.singletonList(newEdgeOfSort));
                        }
                        // if operation chaining is disabled, this could mark sure the
                        // sort node and its output can also be connected by
                        // ForwardPartitioner
                        newEdge = addExchangeAndReconnectEdge(edge, inputProperty, true);
                    } else {
                        // add Exchange with keep_input_as_is distribution as the input
                        // of the node
                        newEdge = addExchangeAndReconnectEdge(edge, inputProperty, false);
                        updateOriginalEdgeInMultipleInput(node, i, (BatchExecExchange) newEdge.getSource());
                    }
                    // update the edge
                    newEdges.set(i, newEdge);
                    changed = true;
                } else if (hasSortInputForInputSortedNode(node)) {
                    // if operation chaining is disabled, this could mark sure the sort
                    // node and its output can also be connected by ForwardPartitioner
                    ExecEdge newEdge = addExchangeAndReconnectEdge(edge, inputProperty, true);
                    newEdges.set(i, newEdge);
                    changed = true;
                }
            }
            if (changed) {
                node.setInputEdges(newEdges);
            }
        }
    };
    execGraph.getRootNodes().forEach(s -> s.accept(visitor));
    return execGraph;
}

Example 3 with ExecNodeVisitor

use of org.apache.flink.table.planner.plan.nodes.exec.visitor.ExecNodeVisitor in project flink by apache.

the class ExecNodeGraphJsonSerializer method validate.

/**
 * Check whether the given {@link ExecNodeGraph} is completely legal.
 */
private static void validate(ExecNodeGraph execGraph) {
    ExecNodeVisitor visitor = new ExecNodeGraphValidator();
    execGraph.getRootNodes().forEach(visitor::visit);
}

Example 4 with ExecNodeVisitor

use of org.apache.flink.table.planner.plan.nodes.exec.visitor.ExecNodeVisitor in project flink by apache.

the class ExecNodePlanDumper method doConvertTreeToString.

private static String doConvertTreeToString(ExecNode<?> node, ReuseInfo reuseInfo, boolean updateVisitedTimes, List<ExecNode<?>> stopVisitNodes, boolean includingBorders) {
    StringBuilder sb = new StringBuilder();
    ExecNodeVisitor visitor = new ExecNodeStringTreeBuilder(sb, reuseInfo, updateVisitedTimes, stopVisitNodes, includingBorders);
    node.accept(visitor);
    return sb.toString();
}

Example 5 with ExecNodeVisitor

use of org.apache.flink.table.planner.plan.nodes.exec.visitor.ExecNodeVisitor in project flink by apache.

the class ExecNodePlanDumper method dagToString.

/**
 * Converts an {@link ExecNode} DAG to a string as a tree style.
 *
 * <p>The following DAG of {@link ExecNode}
 *
 * <pre>{@code
 *    Sink1    Sink2
 *     |        |
 *  Filter3  Filter4
 *      \     /
 *       Join
 *     /      \
 * Filter1  Filter2
 *    \     /
 *    Project
 *      |
 *    Scan
 * }</pre>
 *
 * <p>would be converted to the tree style as following:
 *
 * <pre>{@code
 * Join(reuse_id=[2])
 * :- Filter1
 * :  +- Project(reuse_id=[1])
 * :     +- Scan
 * +- Filter2
 *    +- Reused(reference_id=[1])
 *
 * Sink1
 * +- Filter3
 *    +- Reused(reference_id=[2])
 *
 * Sink2
 * +- Filter4
 *    +- Reused(reference_id=[2])
 * }</pre>
 *
 * @param nodes the ExecNodes to convert
 * @return the plan of ExecNode
 */
public static String dagToString(List<ExecNode<?>> nodes) {
    Preconditions.checkArgument(nodes != null && !nodes.isEmpty(), "nodes should not be null or empty.");
    if (nodes.size() == 1) {
        return treeToString(nodes.get(0));
    }
    // nodes that stop visit when meet them
    final List<ExecNode<?>> stopVisitNodes = new ArrayList<>();
    final StringBuilder sb = new StringBuilder();
    final DagReuseInfo reuseInfo = new DagReuseInfo(nodes, new ArrayList<>());
    final ExecNodeVisitor visitor = new ExecNodeVisitorImpl() {

        @Override
        public void visit(ExecNode<?> node) {
            int visitedTimes = reuseInfo.addVisitedTimes(node);
            boolean isFirstVisit = visitedTimes == 1;
            if (isFirstVisit) {
                super.visit(node);
            }
            int reuseId = reuseInfo.getReuseId(node);
            boolean isReuseNode = reuseId >= 0;
            if (node instanceof CommonExecLegacySink || node instanceof CommonExecSink || (isReuseNode && isFirstVisit)) {
                if (isReuseNode) {
                    reuseInfo.setFirstVisited(node, true);
                }
                String reusePlan = doConvertTreeToString(node, reuseInfo, false, stopVisitNodes, false);
                sb.append(reusePlan).append(System.lineSeparator());
                if (isReuseNode) {
                    // update visit info after the reuse node visited
                    stopVisitNodes.add(node);
                    reuseInfo.setFirstVisited(node, false);
                }
            }
        }
    };
    nodes.forEach(visitor::visit);
    if (sb.length() > 0) {
        // delete last line separator
        sb.deleteCharAt(sb.length() - 1);
    }
    return sb.toString();
}