Examples with LoopFragmentWhole org.graalvm.compiler.nodes.loop.LoopFragmentWhole used on opensource projects

Example 1 with LoopFragmentWhole

use of org.graalvm.compiler.nodes.loop.LoopFragmentWhole in project graal by oracle.

the class LoopTransformations method unswitch.

public static void unswitch(LoopEx loop, List<ControlSplitNode> controlSplitNodeSet, boolean isTrivialUnswitch) {
    ControlSplitNode firstNode = controlSplitNodeSet.iterator().next();
    LoopFragmentWhole originalLoop = loop.whole();
    StructuredGraph graph = firstNode.graph();
    if (!isTrivialUnswitch) {
        loop.loopBegin().incrementUnswitches();
    }
    // create new control split out of loop
    ControlSplitNode newControlSplit = (ControlSplitNode) firstNode.copyWithInputs();
    originalLoop.entryPoint().replaceAtPredecessor(newControlSplit);
    /*
         * The code below assumes that all of the control split nodes have the same successor
         * structure, which should have been enforced by findUnswitchable.
         */
    Iterator<Position> successors = firstNode.successorPositions().iterator();
    assert successors.hasNext();
    // original loop is used as first successor
    Position firstPosition = successors.next();
    AbstractBeginNode originalLoopBegin = BeginNode.begin(originalLoop.entryPoint());
    firstPosition.set(newControlSplit, originalLoopBegin);
    originalLoopBegin.setNodeSourcePosition(firstPosition.get(firstNode).getNodeSourcePosition());
    while (successors.hasNext()) {
        Position position = successors.next();
        // create a new loop duplicate and connect it.
        LoopFragmentWhole duplicateLoop = originalLoop.duplicate();
        AbstractBeginNode newBegin = BeginNode.begin(duplicateLoop.entryPoint());
        newBegin.setNodeSourcePosition(position.get(firstNode).getNodeSourcePosition());
        position.set(newControlSplit, newBegin);
        // For each cloned ControlSplitNode, simplify the proper path
        for (ControlSplitNode controlSplitNode : controlSplitNodeSet) {
            ControlSplitNode duplicatedControlSplit = duplicateLoop.getDuplicatedNode(controlSplitNode);
            if (duplicatedControlSplit.isAlive()) {
                AbstractBeginNode survivingSuccessor = (AbstractBeginNode) position.get(duplicatedControlSplit);
                survivingSuccessor.replaceAtUsages(newBegin, InputType.Guard);
                graph.removeSplitPropagate(duplicatedControlSplit, survivingSuccessor);
            }
        }
    }
    // original loop is simplified last to avoid deleting controlSplitNode too early
    for (ControlSplitNode controlSplitNode : controlSplitNodeSet) {
        if (controlSplitNode.isAlive()) {
            AbstractBeginNode survivingSuccessor = (AbstractBeginNode) firstPosition.get(controlSplitNode);
            survivingSuccessor.replaceAtUsages(originalLoopBegin, InputType.Guard);
            graph.removeSplitPropagate(controlSplitNode, survivingSuccessor);
        }
    }
// TODO (gd) probabilities need some amount of fixup.. (probably also in other transforms)
}

Example 2 with LoopFragmentWhole

use of org.graalvm.compiler.nodes.loop.LoopFragmentWhole in project graal by oracle.

the class LoopTransformations method insertPrePostLoops.

// This function splits candidate loops into pre, main and post loops,
// dividing the iteration space to facilitate the majority of iterations
// being executed in a main loop, which will have RCE implemented upon it.
// The initial loop form is constrained to single entry/exit, but can have
// flow. The translation looks like:
// 
// @formatter:off
// 
// (Simple Loop entry)                   (Pre Loop Entry)
// |                                  |
// (LoopBeginNode)                    (LoopBeginNode)
// |                                  |
// (Loop Control Test)<------   ==>  (Loop control Test)<------
// /               \       \         /               \       \
// (Loop Exit)      (Loop Body) |    (Loop Exit)      (Loop Body) |
// |                |       |        |                |       |
// (continue code)     (Loop End)  |  if (M < length)*   (Loop End)  |
// \       /       /      \           \      /
// ----->        /       |            ----->
// /  if ( ... )*
// /     /       \
// /     /         \
// /     /           \
// |     /     (Main Loop Entry)
// |    |             |
// |    |      (LoopBeginNode)
// |    |             |
// |    |     (Loop Control Test)<------
// |    |      /               \        \
// |    |  (Loop Exit)      (Loop Body) |
// \   \      |                |       |
// \   \     |            (Loop End)  |
// \   \    |                \       /
// \   \   |                 ------>
// \   \  |
// (Main Loop Merge)*
// |
// (Post Loop Entry)
// |
// (LoopBeginNode)
// |
// (Loop Control Test)<-----
// /               \       \
// (Loop Exit)     (Loop Body) |
// |               |       |
// (continue code)    (Loop End)  |
// \      /
// ----->
// 
// Key: "*" = optional.
// @formatter:on
// 
// The value "M" is the maximal value of the loop trip for the original
// loop. The value of "length" is applicable to the number of arrays found
// in the loop but is reduced if some or all of the arrays are known to be
// the same length as "M". The maximum number of tests can be equal to the
// number of arrays in the loop, where multiple instances of an array are
// subsumed into a single test for that arrays length.
// 
// If the optional main loop entry tests are absent, the Pre Loop exit
// connects to the Main loops entry and there is no merge hanging off the
// main loops exit to converge flow from said tests. All split use data
// flow is mitigated through phi(s) in the main merge if present and
// passed through the main and post loop phi(s) from the originating pre
// loop with final phi(s) and data flow patched to the "continue code".
// The pre loop is constrained to one iteration for now and will likely
// be updated to produce vector alignment if applicable.
public static PreMainPostResult insertPrePostLoops(LoopEx loop) {
    assert loop.loopBegin().loopExits().isEmpty() || loop.loopBegin().graph().isAfterStage(StageFlag.VALUE_PROXY_REMOVAL) || loop.counted().getCountedExit() instanceof LoopExitNode : "Can only unroll loops, if they have exits, if the counted exit is a regular loop exit " + loop;
    StructuredGraph graph = loop.loopBegin().graph();
    // prepare clean exit states
    ensureExitsHaveUniqueStates(loop);
    graph.getDebug().log("LoopTransformations.insertPrePostLoops %s", loop);
    LoopFragmentWhole preLoop = loop.whole();
    CountedLoopInfo preCounted = loop.counted();
    LoopBeginNode preLoopBegin = loop.loopBegin();
    /*
         * When transforming counted loops with multiple loop exits the counted exit is the one that
         * is interesting for the pre-main-post transformation since it is the regular, non-early,
         * exit.
         */
    final AbstractBeginNode preLoopExitNode = preCounted.getCountedExit();
    assert preLoop.nodes().contains(preLoopBegin);
    assert preLoop.nodes().contains(preLoopExitNode);
    /*
         * Duplicate the original loop two times, each duplication will create a merge for the loop
         * exits of the original loop and the duplication one.
         */
    LoopFragmentWhole mainLoop = preLoop.duplicate();
    LoopBeginNode mainLoopBegin = mainLoop.getDuplicatedNode(preLoopBegin);
    AbstractBeginNode mainLoopExitNode = mainLoop.getDuplicatedNode(preLoopExitNode);
    EndNode mainEndNode = getBlockEndAfterLoopExit(mainLoopExitNode);
    AbstractMergeNode mainMergeNode = mainEndNode.merge();
    graph.getDebug().dump(DebugContext.VERY_DETAILED_LEVEL, graph, "After  duplication of main loop %s", mainLoop);
    LoopFragmentWhole postLoop = preLoop.duplicate();
    LoopBeginNode postLoopBegin = postLoop.getDuplicatedNode(preLoopBegin);
    AbstractBeginNode postLoopExitNode = postLoop.getDuplicatedNode(preLoopExitNode);
    EndNode postEndNode = getBlockEndAfterLoopExit(postLoopExitNode);
    AbstractMergeNode postMergeNode = postEndNode.merge();
    graph.getDebug().dump(DebugContext.VERY_DETAILED_LEVEL, graph, "After post loop duplication");
    preLoopBegin.incrementSplits();
    preLoopBegin.incrementSplits();
    preLoopBegin.setPreLoop();
    mainLoopBegin.setMainLoop();
    postLoopBegin.setPostLoop();
    if (graph.isBeforeStage(StageFlag.VALUE_PROXY_REMOVAL)) {
        // clear state to avoid problems with usages on the merge
        cleanupAndDeleteState(mainMergeNode);
        cleanupPostDominatingValues(mainLoopBegin, mainMergeNode, postEndNode);
        removeStateAndPhis(postMergeNode);
        /*
             * Fix the framestates for the pre loop exit node and the main loop exit node.
             *
             * The only exit that actually really exits the original loop is the loop exit of the
             * post-loop. All other paths have to fully go through pre->main->post loops. We can
             * never go from pre/main loop directly to the code after the loop, we always have to go
             * through the original loop header, thus we need to fix the correct state on the
             * pre/main loop exit.
             *
             * However, depending on the shape of the loop this is either
             *
             * for head counted loops: the loop header state with the values fixed
             *
             * for tail counted loops: the last state inside the body of the loop dominating the
             * tail check (This is different since tail counted loops have protection control flow
             * meaning it is possible to go pre -> after post, pre->main->after post, pre -> post ->
             * after post. For the protected main and post loops it is enough to deopt to the last
             * body state and the interpreter can then re-execute any failing counter check).
             *
             * For both scenarios we proxy the necessary nodes.
             */
        createExitState(preLoopBegin, (LoopExitNode) preLoopExitNode, loop.counted().isInverted(), preLoop);
        createExitState(mainLoopBegin, (LoopExitNode) mainLoopExitNode, loop.counted().isInverted(), mainLoop);
    }
    assert graph.isAfterStage(StageFlag.VALUE_PROXY_REMOVAL) || preLoopExitNode instanceof LoopExitNode : "Unrolling with proxies requires actual loop exit nodes as counted exits";
    rewirePreToMainPhis(preLoopBegin, mainLoop, preLoop, graph.isBeforeStage(StageFlag.VALUE_PROXY_REMOVAL) ? (LoopExitNode) preLoopExitNode : null, loop.counted().isInverted());
    AbstractEndNode postEntryNode = postLoopBegin.forwardEnd();
    // Exits have been merged, find the continuation below the merge
    FixedNode continuationNode = mainMergeNode.next();
    // In the case of no Bounds tests, we just flow right into the main loop
    AbstractBeginNode mainLandingNode = BeginNode.begin(postEntryNode);
    mainLoopExitNode.setNext(mainLandingNode);
    preLoopExitNode.setNext(mainLoopBegin.forwardEnd());
    // Add and update any phi edges as per merge usage as needed and update usages
    assert graph.isAfterStage(StageFlag.VALUE_PROXY_REMOVAL) || mainLoopExitNode instanceof LoopExitNode : "Unrolling with proxies requires actual loop exit nodes as counted exits";
    processPreLoopPhis(loop, graph.isBeforeStage(StageFlag.VALUE_PROXY_REMOVAL) ? (LoopExitNode) mainLoopExitNode : null, mainLoop, postLoop);
    graph.getDebug().dump(DebugContext.VERY_DETAILED_LEVEL, graph, "After processing pre loop phis");
    continuationNode.predecessor().clearSuccessors();
    postLoopExitNode.setNext(continuationNode);
    cleanupMerge(postMergeNode, postLoopExitNode);
    cleanupMerge(mainMergeNode, mainLandingNode);
    // Change the preLoop to execute one iteration for now
    if (graph.isBeforeStage(StageFlag.VALUE_PROXY_REMOVAL)) {
        /*
             * The pre-loop exit's condition's induction variable start node might be already
             * re-written to be a phi of merged loop exits from a previous pre-main-post creation,
             * thus use an updated loop info.
             */
        loop.resetCounted();
        loop.detectCounted();
        updatePreLoopLimit(loop.counted());
    } else {
        updatePreLoopLimit(preCounted);
    }
    double originalFrequency = loop.localLoopFrequency();
    preLoopBegin.setLoopOrigFrequency(originalFrequency);
    mainLoopBegin.setLoopOrigFrequency(originalFrequency);
    postLoopBegin.setLoopOrigFrequency(originalFrequency);
    assert preLoopExitNode.predecessor() instanceof IfNode;
    assert mainLoopExitNode.predecessor() instanceof IfNode;
    assert postLoopExitNode.predecessor() instanceof IfNode;
    setSingleVisitedLoopFrequencySplitProbability(preLoopExitNode);
    setSingleVisitedLoopFrequencySplitProbability(postLoopExitNode);
    if (graph.isAfterStage(StageFlag.VALUE_PROXY_REMOVAL)) {
        // The pre and post loops don't require safepoints at all
        for (SafepointNode safepoint : preLoop.nodes().filter(SafepointNode.class)) {
            graph.removeFixed(safepoint);
        }
        for (SafepointNode safepoint : postLoop.nodes().filter(SafepointNode.class)) {
            graph.removeFixed(safepoint);
        }
    }
    graph.getDebug().dump(DebugContext.DETAILED_LEVEL, graph, "InsertPrePostLoops %s", loop);
    return new PreMainPostResult(preLoopBegin, mainLoopBegin, postLoopBegin, preLoop, mainLoop, postLoop);
}