Examples with PredictionContextCache org.antlr.v4.runtime.atn.PredictionContextCache used on opensource projects

Example 1 with PredictionContextCache

use of org.antlr.v4.runtime.atn.PredictionContextCache in project robozonky by RoboZonky.

the class SideEffectFreeParser method modifyInterpreter.

private static void modifyInterpreter(final NaturalLanguageStrategyParser p) {
    final int originalSize = p.getInterpreter().decisionToDFA.length;
    // give our own array so the static one isn't used
    final DFA[] emptyDFA = new DFA[originalSize];
    final ParserATNSimulator newInterpreter = new ParserATNSimulator(p, p.getATN(), emptyDFA, new PredictionContextCache());
    // initialize our array so that the parser functions properly
    newInterpreter.clearDFA();
    // replace the interpreter to bypass all static caches
    p.setInterpreter(newInterpreter);
}

Example 2 with PredictionContextCache

use of org.antlr.v4.runtime.atn.PredictionContextCache in project robozonky by RoboZonky.

the class SideEffectFreeParser method modifyInterpreter.

private static void modifyInterpreter(final NaturalLanguageStrategyLexer l) {
    final int originalSize = l.getInterpreter().decisionToDFA.length;
    // give our own array so the static one isn't used
    final DFA[] emptyDFA = new DFA[originalSize];
    final LexerATNSimulator newInterpreter = new LexerATNSimulator(l, l.getATN(), emptyDFA, new PredictionContextCache());
    // initialize our array so that the lexer functions properly
    newInterpreter.clearDFA();
    // replace the interpreter to bypass all static caches
    l.setInterpreter(newInterpreter);
}

Example 3 with PredictionContextCache

use of org.antlr.v4.runtime.atn.PredictionContextCache in project antlr4 by tunnelvisionlabs.

the class ParserATNSimulator method execATN.

/**
 * Performs ATN simulation to compute a predicted alternative based
 *  upon the remaining input, but also updates the DFA cache to avoid
 *  having to traverse the ATN again for the same input sequence.
 *
 *	 There are some key conditions we're looking for after computing a new
 *	 set of ATN configs (proposed DFA state):
 * if the set is empty, there is no viable alternative for current symbol
 * does the state uniquely predict an alternative?
 * does the state have a conflict that would prevent us from
 *	         putting it on the work list?
 * if in non-greedy decision is there a config at a rule stop state?
 *
 *	 We also have some key operations to do:
 * add an edge from previous DFA state to potentially new DFA state, D,
 *	         upon current symbol but only if adding to work list, which means in all
 *	         cases except no viable alternative (and possibly non-greedy decisions?)
 * collecting predicates and adding semantic context to DFA accept states
 * adding rule context to context-sensitive DFA accept states
 * consuming an input symbol
 * reporting a conflict
 * reporting an ambiguity
 * reporting a context sensitivity
 * reporting insufficient predicates
 *
 *	 We should isolate those operations, which are side-effecting, to the
 *	 main work loop. We can isolate lots of code into other functions, but
 *	 they should be side effect free. They can return package that
 *	 indicates whether we should report something, whether we need to add a
 *	 DFA edge, whether we need to augment accept state with semantic
 *	 context or rule invocation context. Actually, it seems like we always
 *	 add predicates if they exist, so that can simply be done in the main
 *	 loop for any accept state creation or modification request.
 *
 *	 cover these cases:
 *	    dead end
 *	    single alt
 *	    single alt + preds
 *	    conflict
 *	    conflict + preds
 *
 *	 TODO: greedy + those
 */
protected int execATN(@NotNull DFA dfa, @NotNull TokenStream input, int startIndex, @NotNull SimulatorState initialState) {
    if (debug)
        System.out.println("execATN decision " + dfa.decision + " exec LA(1)==" + getLookaheadName(input));
    final ParserRuleContext outerContext = initialState.outerContext;
    final boolean useContext = initialState.useContext;
    int t = input.LA(1);
    SimulatorState previous = initialState;
    PredictionContextCache contextCache = new PredictionContextCache();
    while (true) {
        // while more work
        SimulatorState nextState = computeReachSet(dfa, previous, t, contextCache);
        if (nextState == null) {
            addDFAEdge(previous.s0, input.LA(1), ERROR);
            return handleNoViableAlt(input, startIndex, previous);
        }
        DFAState D = nextState.s0;
        // predicted alt => accept state
        assert D.isAcceptState() || D.getPrediction() == ATN.INVALID_ALT_NUMBER;
        // conflicted => accept state
        assert D.isAcceptState() || D.configs.getConflictInfo() == null;
        if (isAcceptState(D, useContext)) {
            BitSet conflictingAlts = D.configs.getConflictingAlts();
            int predictedAlt = conflictingAlts == null ? D.getPrediction() : ATN.INVALID_ALT_NUMBER;
            if (predictedAlt != ATN.INVALID_ALT_NUMBER) {
                if (optimize_ll1 && input.index() == startIndex && !dfa.isPrecedenceDfa() && nextState.outerContext == nextState.remainingOuterContext && dfa.decision >= 0 && !D.configs.hasSemanticContext()) {
                    if (t >= 0 && t <= Short.MAX_VALUE) {
                        int key = (dfa.decision << 16) + t;
                        atn.LL1Table.put(key, predictedAlt);
                    }
                }
                if (useContext && always_try_local_context) {
                    reportContextSensitivity(dfa, predictedAlt, nextState, startIndex, input.index());
                }
            }
            predictedAlt = D.getPrediction();
            // int k = input.index() - startIndex + 1; // how much input we used
            // System.out.println("used k="+k);
            boolean attemptFullContext = conflictingAlts != null && userWantsCtxSensitive;
            if (attemptFullContext) {
                // Only exact conflicts are known to be ambiguous when local
                // prediction does not step out of the decision rule.
                attemptFullContext = !useContext && (D.configs.getDipsIntoOuterContext() || !D.configs.isExactConflict()) && (!treat_sllk1_conflict_as_ambiguity || input.index() != startIndex);
            }
            if (D.configs.hasSemanticContext()) {
                DFAState.PredPrediction[] predPredictions = D.predicates;
                if (predPredictions != null) {
                    int conflictIndex = input.index();
                    if (conflictIndex != startIndex) {
                        input.seek(startIndex);
                    }
                    // use complete evaluation here if we'll want to retry with full context if still ambiguous
                    conflictingAlts = evalSemanticContext(predPredictions, outerContext, attemptFullContext || reportAmbiguities);
                    switch(conflictingAlts.cardinality()) {
                        case 0:
                            throw noViableAlt(input, outerContext, D.configs, startIndex);
                        case 1:
                            return conflictingAlts.nextSetBit(0);
                        default:
                            break;
                    }
                    if (conflictIndex != startIndex) {
                        // restore the index so reporting the fallback to full
                        // context occurs with the index at the correct spot
                        input.seek(conflictIndex);
                    }
                }
            }
            if (!attemptFullContext) {
                if (conflictingAlts != null) {
                    if (reportAmbiguities && conflictingAlts.cardinality() > 1) {
                        reportAmbiguity(dfa, D, startIndex, input.index(), D.configs.isExactConflict(), conflictingAlts, D.configs);
                    }
                    predictedAlt = conflictingAlts.nextSetBit(0);
                }
                return predictedAlt;
            } else {
                assert !useContext;
                assert isAcceptState(D, false);
                if (debug)
                    System.out.println("RETRY with outerContext=" + outerContext);
                SimulatorState fullContextState = computeStartState(dfa, outerContext, true);
                if (reportAmbiguities) {
                    reportAttemptingFullContext(dfa, conflictingAlts, nextState, startIndex, input.index());
                }
                input.seek(startIndex);
                return execATN(dfa, input, startIndex, fullContextState);
            }
        }
        previous = nextState;
        if (t != IntStream.EOF) {
            input.consume();
            t = input.LA(1);
        }
    }
}

Example 4 with PredictionContextCache

use of org.antlr.v4.runtime.atn.PredictionContextCache in project antlr4 by tunnelvisionlabs.

the class ParserATNSimulator method computeStartState.

@NotNull
protected SimulatorState computeStartState(DFA dfa, ParserRuleContext globalContext, boolean useContext) {
    DFAState s0 = dfa.isPrecedenceDfa() ? dfa.getPrecedenceStartState(parser.getPrecedence(), useContext) : useContext ? dfa.s0full.get() : dfa.s0.get();
    if (s0 != null) {
        if (!useContext) {
            return new SimulatorState(globalContext, s0, useContext, globalContext);
        }
        s0.setContextSensitive(atn);
    }
    final int decision = dfa.decision;
    @NotNull final ATNState p = dfa.atnStartState;
    int previousContext = 0;
    ParserRuleContext remainingGlobalContext = globalContext;
    // always at least the implicit call to start rule
    PredictionContext initialContext = useContext ? PredictionContext.EMPTY_FULL : PredictionContext.EMPTY_LOCAL;
    PredictionContextCache contextCache = new PredictionContextCache();
    if (useContext) {
        if (!enable_global_context_dfa) {
            while (remainingGlobalContext != null) {
                if (remainingGlobalContext.isEmpty()) {
                    previousContext = PredictionContext.EMPTY_FULL_STATE_KEY;
                    remainingGlobalContext = null;
                } else {
                    previousContext = getReturnState(remainingGlobalContext);
                    initialContext = initialContext.appendContext(previousContext, contextCache);
                    remainingGlobalContext = remainingGlobalContext.getParent();
                }
            }
        }
        while (s0 != null && s0.isContextSensitive() && remainingGlobalContext != null) {
            DFAState next;
            remainingGlobalContext = skipTailCalls(remainingGlobalContext);
            if (remainingGlobalContext.isEmpty()) {
                next = s0.getContextTarget(PredictionContext.EMPTY_FULL_STATE_KEY);
                previousContext = PredictionContext.EMPTY_FULL_STATE_KEY;
                remainingGlobalContext = null;
            } else {
                previousContext = getReturnState(remainingGlobalContext);
                next = s0.getContextTarget(previousContext);
                initialContext = initialContext.appendContext(previousContext, contextCache);
                remainingGlobalContext = remainingGlobalContext.getParent();
            }
            if (next == null) {
                break;
            }
            s0 = next;
        }
    }
    if (s0 != null && !s0.isContextSensitive()) {
        return new SimulatorState(globalContext, s0, useContext, remainingGlobalContext);
    }
    ATNConfigSet configs = new ATNConfigSet();
    while (true) {
        ATNConfigSet reachIntermediate = new ATNConfigSet();
        int n = p.getNumberOfTransitions();
        for (int ti = 0; ti < n; ti++) {
            // for each transition
            ATNState target = p.transition(ti).target;
            reachIntermediate.add(ATNConfig.create(target, ti + 1, initialContext));
        }
        boolean hasMoreContext = remainingGlobalContext != null;
        if (!hasMoreContext) {
            configs.setOutermostConfigSet(true);
        }
        final boolean collectPredicates = true;
        closure(reachIntermediate, configs, collectPredicates, hasMoreContext, contextCache, false);
        boolean stepIntoGlobal = configs.getDipsIntoOuterContext();
        DFAState next;
        if (useContext && !enable_global_context_dfa) {
            s0 = addDFAState(dfa, configs, contextCache);
            break;
        } else if (s0 == null) {
            if (!dfa.isPrecedenceDfa()) {
                AtomicReference<DFAState> reference = useContext ? dfa.s0full : dfa.s0;
                next = addDFAState(dfa, configs, contextCache);
                if (!reference.compareAndSet(null, next)) {
                    next = reference.get();
                }
            } else {
                /* If this is a precedence DFA, we use applyPrecedenceFilter
					 * to convert the computed start state to a precedence start
					 * state. We then use DFA.setPrecedenceStartState to set the
					 * appropriate start state for the precedence level rather
					 * than simply setting DFA.s0.
					 */
                configs = applyPrecedenceFilter(configs, globalContext, contextCache);
                next = addDFAState(dfa, configs, contextCache);
                dfa.setPrecedenceStartState(parser.getPrecedence(), useContext, next);
            }
        } else {
            if (dfa.isPrecedenceDfa()) {
                configs = applyPrecedenceFilter(configs, globalContext, contextCache);
            }
            next = addDFAState(dfa, configs, contextCache);
            s0.setContextTarget(previousContext, next);
        }
        s0 = next;
        if (!useContext || !stepIntoGlobal) {
            break;
        }
        // TODO: make sure it distinguishes empty stack states
        next.setContextSensitive(atn);
        configs.clear();
        remainingGlobalContext = skipTailCalls(remainingGlobalContext);
        int nextContextElement = getReturnState(remainingGlobalContext);
        if (remainingGlobalContext.isEmpty()) {
            remainingGlobalContext = null;
        } else {
            remainingGlobalContext = remainingGlobalContext.getParent();
        }
        if (nextContextElement != PredictionContext.EMPTY_FULL_STATE_KEY) {
            initialContext = initialContext.appendContext(nextContextElement, contextCache);
        }
        previousContext = nextContextElement;
    }
    return new SimulatorState(globalContext, s0, useContext, remainingGlobalContext);
}

Example 5 with PredictionContextCache

use of org.antlr.v4.runtime.atn.PredictionContextCache in project antlr4 by tunnelvisionlabs.

the class ParserATNSimulator method addDFAEdge.

@NotNull
protected DFAState addDFAEdge(@NotNull DFA dfa, @NotNull DFAState fromState, int t, IntegerList contextTransitions, @NotNull ATNConfigSet toConfigs, PredictionContextCache contextCache) {
    assert contextTransitions == null || contextTransitions.isEmpty() || dfa.isContextSensitive();
    DFAState from = fromState;
    DFAState to = addDFAState(dfa, toConfigs, contextCache);
    if (contextTransitions != null) {
        for (int context : contextTransitions.toArray()) {
            if (context == PredictionContext.EMPTY_FULL_STATE_KEY) {
                if (from.configs.isOutermostConfigSet()) {
                    continue;
                }
            }
            from.setContextSensitive(atn);
            from.setContextSymbol(t);
            DFAState next = from.getContextTarget(context);
            if (next != null) {
                from = next;
                continue;
            }
            next = addDFAContextState(dfa, from.configs, context, contextCache);
            assert context != PredictionContext.EMPTY_FULL_STATE_KEY || next.configs.isOutermostConfigSet();
            from.setContextTarget(context, next);
            from = next;
        }
    }
    if (debug)
        System.out.println("EDGE " + from + " -> " + to + " upon " + getTokenName(t));
    addDFAEdge(from, t, to);
    if (debug)
        System.out.println("DFA=\n" + dfa.toString(parser != null ? parser.getVocabulary() : VocabularyImpl.EMPTY_VOCABULARY, parser != null ? parser.getRuleNames() : null));
    return to;
}