Examples with CountedLoopInfo org.graalvm.compiler.nodes.loop.CountedLoopInfo used on opensource projects

Example 1 with CountedLoopInfo

use of org.graalvm.compiler.nodes.loop.CountedLoopInfo in project graal by oracle.

the class AMD64HotSpotAddressLowering method signExtend.

/**
 * Create a sign extend for {@code input}, or zero extend if {@code input} can be proven
 * positive.
 */
private static ValueNode signExtend(ValueNode input, LoopEx loop) {
    StructuredGraph graph = input.graph();
    if (input instanceof PhiNode) {
        EconomicMap<Node, InductionVariable> ivs = loop.getInductionVariables();
        InductionVariable inductionVariable = ivs.get(input);
        if (inductionVariable != null && inductionVariable instanceof BasicInductionVariable) {
            CountedLoopInfo countedLoopInfo = loop.counted();
            IntegerStamp initStamp = (IntegerStamp) inductionVariable.initNode().stamp(NodeView.DEFAULT);
            if (initStamp.isPositive()) {
                if (inductionVariable.isConstantExtremum() && countedLoopInfo.counterNeverOverflows()) {
                    long init = inductionVariable.constantInit();
                    long stride = inductionVariable.constantStride();
                    long extremum = inductionVariable.constantExtremum();
                    if (init >= 0 && extremum >= 0) {
                        long shortestTrip = (extremum - init) / stride + 1;
                        if (countedLoopInfo.constantMaxTripCount().equals(shortestTrip)) {
                            return graph.unique(new ZeroExtendNode(input, INT_BITS, ADDRESS_BITS, true));
                        }
                    }
                }
                if (countedLoopInfo.getLimitCheckedIV() == inductionVariable && inductionVariable.direction() == InductionVariable.Direction.Up && (countedLoopInfo.getOverFlowGuard() != null || countedLoopInfo.counterNeverOverflows())) {
                    return graph.unique(new ZeroExtendNode(input, INT_BITS, ADDRESS_BITS, true));
                }
            }
        }
    }
    return input.graph().maybeAddOrUnique(SignExtendNode.create(input, ADDRESS_BITS, NodeView.DEFAULT));
}

Example 2 with CountedLoopInfo

use of org.graalvm.compiler.nodes.loop.CountedLoopInfo in project graal by oracle.

the class LoopTransformations method insertPrePostLoops.

// This function splits candidate loops into pre, main and post loops,
// dividing the iteration space to facilitate the majority of iterations
// being executed in a main loop, which will have RCE implemented upon it.
// The initial loop form is constrained to single entry/exit, but can have
// flow. The translation looks like:
// 
// @formatter:off
// 
// (Simple Loop entry)                   (Pre Loop Entry)
// |                                  |
// (LoopBeginNode)                    (LoopBeginNode)
// |                                  |
// (Loop Control Test)<------   ==>  (Loop control Test)<------
// /               \       \         /               \       \
// (Loop Exit)      (Loop Body) |    (Loop Exit)      (Loop Body) |
// |                |       |        |                |       |
// (continue code)     (Loop End)  |  if (M < length)*   (Loop End)  |
// \       /       /      \           \      /
// ----->        /       |            ----->
// /  if ( ... )*
// /     /       \
// /     /         \
// /     /           \
// |     /     (Main Loop Entry)
// |    |             |
// |    |      (LoopBeginNode)
// |    |             |
// |    |     (Loop Control Test)<------
// |    |      /               \        \
// |    |  (Loop Exit)      (Loop Body) |
// \   \      |                |       |
// \   \     |            (Loop End)  |
// \   \    |                \       /
// \   \   |                 ------>
// \   \  |
// (Main Loop Merge)*
// |
// (Post Loop Entry)
// |
// (LoopBeginNode)
// |
// (Loop Control Test)<-----
// /               \       \
// (Loop Exit)     (Loop Body) |
// |               |       |
// (continue code)    (Loop End)  |
// \      /
// ----->
// 
// Key: "*" = optional.
// @formatter:on
// 
// The value "M" is the maximal value of the loop trip for the original
// loop. The value of "length" is applicable to the number of arrays found
// in the loop but is reduced if some or all of the arrays are known to be
// the same length as "M". The maximum number of tests can be equal to the
// number of arrays in the loop, where multiple instances of an array are
// subsumed into a single test for that arrays length.
// 
// If the optional main loop entry tests are absent, the Pre Loop exit
// connects to the Main loops entry and there is no merge hanging off the
// main loops exit to converge flow from said tests. All split use data
// flow is mitigated through phi(s) in the main merge if present and
// passed through the main and post loop phi(s) from the originating pre
// loop with final phi(s) and data flow patched to the "continue code".
// The pre loop is constrained to one iteration for now and will likely
// be updated to produce vector alignment if applicable.
public static PreMainPostResult insertPrePostLoops(LoopEx loop) {
    assert loop.loopBegin().loopExits().isEmpty() || loop.loopBegin().graph().isAfterStage(StageFlag.VALUE_PROXY_REMOVAL) || loop.counted().getCountedExit() instanceof LoopExitNode : "Can only unroll loops, if they have exits, if the counted exit is a regular loop exit " + loop;
    StructuredGraph graph = loop.loopBegin().graph();
    // prepare clean exit states
    ensureExitsHaveUniqueStates(loop);
    graph.getDebug().log("LoopTransformations.insertPrePostLoops %s", loop);
    LoopFragmentWhole preLoop = loop.whole();
    CountedLoopInfo preCounted = loop.counted();
    LoopBeginNode preLoopBegin = loop.loopBegin();
    /*
         * When transforming counted loops with multiple loop exits the counted exit is the one that
         * is interesting for the pre-main-post transformation since it is the regular, non-early,
         * exit.
         */
    final AbstractBeginNode preLoopExitNode = preCounted.getCountedExit();
    assert preLoop.nodes().contains(preLoopBegin);
    assert preLoop.nodes().contains(preLoopExitNode);
    /*
         * Duplicate the original loop two times, each duplication will create a merge for the loop
         * exits of the original loop and the duplication one.
         */
    LoopFragmentWhole mainLoop = preLoop.duplicate();
    LoopBeginNode mainLoopBegin = mainLoop.getDuplicatedNode(preLoopBegin);
    AbstractBeginNode mainLoopExitNode = mainLoop.getDuplicatedNode(preLoopExitNode);
    EndNode mainEndNode = getBlockEndAfterLoopExit(mainLoopExitNode);
    AbstractMergeNode mainMergeNode = mainEndNode.merge();
    graph.getDebug().dump(DebugContext.VERY_DETAILED_LEVEL, graph, "After  duplication of main loop %s", mainLoop);
    LoopFragmentWhole postLoop = preLoop.duplicate();
    LoopBeginNode postLoopBegin = postLoop.getDuplicatedNode(preLoopBegin);
    AbstractBeginNode postLoopExitNode = postLoop.getDuplicatedNode(preLoopExitNode);
    EndNode postEndNode = getBlockEndAfterLoopExit(postLoopExitNode);
    AbstractMergeNode postMergeNode = postEndNode.merge();
    graph.getDebug().dump(DebugContext.VERY_DETAILED_LEVEL, graph, "After post loop duplication");
    preLoopBegin.incrementSplits();
    preLoopBegin.incrementSplits();
    preLoopBegin.setPreLoop();
    mainLoopBegin.setMainLoop();
    postLoopBegin.setPostLoop();
    if (graph.isBeforeStage(StageFlag.VALUE_PROXY_REMOVAL)) {
        // clear state to avoid problems with usages on the merge
        cleanupAndDeleteState(mainMergeNode);
        cleanupPostDominatingValues(mainLoopBegin, mainMergeNode, postEndNode);
        removeStateAndPhis(postMergeNode);
        /*
             * Fix the framestates for the pre loop exit node and the main loop exit node.
             *
             * The only exit that actually really exits the original loop is the loop exit of the
             * post-loop. All other paths have to fully go through pre->main->post loops. We can
             * never go from pre/main loop directly to the code after the loop, we always have to go
             * through the original loop header, thus we need to fix the correct state on the
             * pre/main loop exit.
             *
             * However, depending on the shape of the loop this is either
             *
             * for head counted loops: the loop header state with the values fixed
             *
             * for tail counted loops: the last state inside the body of the loop dominating the
             * tail check (This is different since tail counted loops have protection control flow
             * meaning it is possible to go pre -> after post, pre->main->after post, pre -> post ->
             * after post. For the protected main and post loops it is enough to deopt to the last
             * body state and the interpreter can then re-execute any failing counter check).
             *
             * For both scenarios we proxy the necessary nodes.
             */
        createExitState(preLoopBegin, (LoopExitNode) preLoopExitNode, loop.counted().isInverted(), preLoop);
        createExitState(mainLoopBegin, (LoopExitNode) mainLoopExitNode, loop.counted().isInverted(), mainLoop);
    }
    assert graph.isAfterStage(StageFlag.VALUE_PROXY_REMOVAL) || preLoopExitNode instanceof LoopExitNode : "Unrolling with proxies requires actual loop exit nodes as counted exits";
    rewirePreToMainPhis(preLoopBegin, mainLoop, preLoop, graph.isBeforeStage(StageFlag.VALUE_PROXY_REMOVAL) ? (LoopExitNode) preLoopExitNode : null, loop.counted().isInverted());
    AbstractEndNode postEntryNode = postLoopBegin.forwardEnd();
    // Exits have been merged, find the continuation below the merge
    FixedNode continuationNode = mainMergeNode.next();
    // In the case of no Bounds tests, we just flow right into the main loop
    AbstractBeginNode mainLandingNode = BeginNode.begin(postEntryNode);
    mainLoopExitNode.setNext(mainLandingNode);
    preLoopExitNode.setNext(mainLoopBegin.forwardEnd());
    // Add and update any phi edges as per merge usage as needed and update usages
    assert graph.isAfterStage(StageFlag.VALUE_PROXY_REMOVAL) || mainLoopExitNode instanceof LoopExitNode : "Unrolling with proxies requires actual loop exit nodes as counted exits";
    processPreLoopPhis(loop, graph.isBeforeStage(StageFlag.VALUE_PROXY_REMOVAL) ? (LoopExitNode) mainLoopExitNode : null, mainLoop, postLoop);
    graph.getDebug().dump(DebugContext.VERY_DETAILED_LEVEL, graph, "After processing pre loop phis");
    continuationNode.predecessor().clearSuccessors();
    postLoopExitNode.setNext(continuationNode);
    cleanupMerge(postMergeNode, postLoopExitNode);
    cleanupMerge(mainMergeNode, mainLandingNode);
    // Change the preLoop to execute one iteration for now
    if (graph.isBeforeStage(StageFlag.VALUE_PROXY_REMOVAL)) {
        /*
             * The pre-loop exit's condition's induction variable start node might be already
             * re-written to be a phi of merged loop exits from a previous pre-main-post creation,
             * thus use an updated loop info.
             */
        loop.resetCounted();
        loop.detectCounted();
        updatePreLoopLimit(loop.counted());
    } else {
        updatePreLoopLimit(preCounted);
    }
    double originalFrequency = loop.localLoopFrequency();
    preLoopBegin.setLoopOrigFrequency(originalFrequency);
    mainLoopBegin.setLoopOrigFrequency(originalFrequency);
    postLoopBegin.setLoopOrigFrequency(originalFrequency);
    assert preLoopExitNode.predecessor() instanceof IfNode;
    assert mainLoopExitNode.predecessor() instanceof IfNode;
    assert postLoopExitNode.predecessor() instanceof IfNode;
    setSingleVisitedLoopFrequencySplitProbability(preLoopExitNode);
    setSingleVisitedLoopFrequencySplitProbability(postLoopExitNode);
    if (graph.isAfterStage(StageFlag.VALUE_PROXY_REMOVAL)) {
        // The pre and post loops don't require safepoints at all
        for (SafepointNode safepoint : preLoop.nodes().filter(SafepointNode.class)) {
            graph.removeFixed(safepoint);
        }
        for (SafepointNode safepoint : postLoop.nodes().filter(SafepointNode.class)) {
            graph.removeFixed(safepoint);
        }
    }
    graph.getDebug().dump(DebugContext.DETAILED_LEVEL, graph, "InsertPrePostLoops %s", loop);
    return new PreMainPostResult(preLoopBegin, mainLoopBegin, postLoopBegin, preLoop, mainLoop, postLoop);
}

Example 3 with CountedLoopInfo

use of org.graalvm.compiler.nodes.loop.CountedLoopInfo in project graal by oracle.

the class LoopPredicationPhase method run.

@Override
@SuppressWarnings("try")
protected void run(StructuredGraph graph, MidTierContext context) {
    DebugContext debug = graph.getDebug();
    final SpeculationLog speculationLog = graph.getSpeculationLog();
    if (graph.hasLoops() && graph.getGuardsStage().allowsFloatingGuards() && context.getOptimisticOptimizations().useLoopLimitChecks(graph.getOptions()) && speculationLog != null) {
        LoopsData data = context.getLoopsDataProvider().getLoopsData(graph);
        final ControlFlowGraph cfg = data.getCFG();
        try (DebugContext.Scope s = debug.scope("predication", cfg)) {
            for (LoopEx loop : data.loops()) {
                // Only inner most loops.
                if (!loop.loop().getChildren().isEmpty()) {
                    continue;
                }
                if (!loop.detectCounted()) {
                    continue;
                }
                final FrameState state = loop.loopBegin().stateAfter();
                final BytecodePosition pos = new BytecodePosition(null, state.getMethod(), state.bci);
                SpeculationLog.SpeculationReason reason = LOOP_PREDICATION.createSpeculationReason(pos);
                if (speculationLog.maySpeculate(reason)) {
                    final CountedLoopInfo counted = loop.counted();
                    final InductionVariable counter = counted.getLimitCheckedIV();
                    final Condition condition = ((CompareNode) counted.getLimitTest().condition()).condition().asCondition();
                    final boolean inverted = loop.counted().isInverted();
                    if ((((IntegerStamp) counter.valueNode().stamp(NodeView.DEFAULT)).getBits() == 32) && !counted.isUnsignedCheck() && ((condition != NE && condition != EQ) || (counter.isConstantStride() && Math.abs(counter.constantStride()) == 1)) && (loop.loopBegin().isMainLoop() || loop.loopBegin().isSimpleLoop())) {
                        NodeIterable<GuardNode> guards = loop.whole().nodes().filter(GuardNode.class);
                        if (LoopPredicationMainPath.getValue(graph.getOptions())) {
                            // C2 only applies loop predication to guards dominating the
                            // backedge.
                            // The following logic emulates that behavior.
                            final NodeIterable<LoopEndNode> loopEndNodes = loop.loopBegin().loopEnds();
                            final Block end = data.getCFG().commonDominatorFor(loopEndNodes);
                            guards = guards.filter(guard -> {
                                final ValueNode anchor = ((GuardNode) guard).getAnchor().asNode();
                                final Block anchorBlock = data.getCFG().getNodeToBlock().get(anchor);
                                return AbstractControlFlowGraph.dominates(anchorBlock, end);
                            });
                        }
                        final AbstractBeginNode body = loop.counted().getBody();
                        final Block bodyBlock = cfg.getNodeToBlock().get(body);
                        for (GuardNode guard : guards) {
                            final AnchoringNode anchor = guard.getAnchor();
                            final Block anchorBlock = cfg.getNodeToBlock().get(anchor.asNode());
                            // for inverted loop the anchor can dominate the body
                            if (!inverted) {
                                if (!AbstractControlFlowGraph.dominates(bodyBlock, anchorBlock)) {
                                    continue;
                                }
                            }
                            processGuard(loop, guard);
                        }
                    }
                }
            }
        } catch (Throwable t) {
            throw debug.handle(t);
        }
    }
}