Examples with DFA org.antlr.v4.runtime.dfa.DFA used on opensource projects

Example 26 with DFA

use of org.antlr.v4.runtime.dfa.DFA in project antlr4 by tunnelvisionlabs.

the class ParserATNSimulator method execDFA.

protected int execDFA(@NotNull DFA dfa, @NotNull TokenStream input, int startIndex, @NotNull SimulatorState state) {
    ParserRuleContext outerContext = state.outerContext;
    if (dfa_debug)
        System.out.println("DFA decision " + dfa.decision + " exec LA(1)==" + getLookaheadName(input) + ", outerContext=" + outerContext.toString(parser));
    if (dfa_debug)
        System.out.print(dfa.toString(parser.getVocabulary(), parser.getRuleNames()));
    DFAState s = state.s0;
    int t = input.LA(1);
    ParserRuleContext remainingOuterContext = state.remainingOuterContext;
    while (true) {
        if (dfa_debug)
            System.out.println("DFA state " + s.stateNumber + " LA(1)==" + getLookaheadName(input));
        if (state.useContext) {
            while (s.isContextSymbol(t)) {
                DFAState next = null;
                if (remainingOuterContext != null) {
                    remainingOuterContext = skipTailCalls(remainingOuterContext);
                    next = s.getContextTarget(getReturnState(remainingOuterContext));
                }
                if (next == null) {
                    // fail over to ATN
                    SimulatorState initialState = new SimulatorState(state.outerContext, s, state.useContext, remainingOuterContext);
                    return execATN(dfa, input, startIndex, initialState);
                }
                assert remainingOuterContext != null;
                remainingOuterContext = remainingOuterContext.getParent();
                s = next;
            }
        }
        if (isAcceptState(s, state.useContext)) {
            if (s.predicates != null) {
                if (dfa_debug)
                    System.out.println("accept " + s);
            } else {
                if (dfa_debug)
                    System.out.println("accept; predict " + s.getPrediction() + " in state " + s.stateNumber);
            }
            // TODO: v3 dfa don't do this.
            break;
        }
        // t is not updated if one of these states is reached
        assert !isAcceptState(s, state.useContext);
        // if no edge, pop over to ATN interpreter, update DFA and return
        DFAState target = getExistingTargetState(s, t);
        if (target == null) {
            if (dfa_debug && t >= 0)
                System.out.println("no edge for " + parser.getVocabulary().getDisplayName(t));
            int alt;
            if (dfa_debug) {
                Interval interval = Interval.of(startIndex, parser.getInputStream().index());
                System.out.println("ATN exec upon " + parser.getInputStream().getText(interval) + " at DFA state " + s.stateNumber);
            }
            SimulatorState initialState = new SimulatorState(outerContext, s, state.useContext, remainingOuterContext);
            alt = execATN(dfa, input, startIndex, initialState);
            if (dfa_debug) {
                System.out.println("back from DFA update, alt=" + alt + ", dfa=\n" + dfa.toString(parser.getVocabulary(), parser.getRuleNames()));
            // dump(dfa);
            }
            // action already executed
            if (dfa_debug)
                System.out.println("DFA decision " + dfa.decision + " predicts " + alt);
            // we've updated DFA, exec'd action, and have our deepest answer
            return alt;
        } else if (target == ERROR) {
            SimulatorState errorState = new SimulatorState(outerContext, s, state.useContext, remainingOuterContext);
            return handleNoViableAlt(input, startIndex, errorState);
        }
        s = target;
        if (!isAcceptState(s, state.useContext) && t != IntStream.EOF) {
            input.consume();
            t = input.LA(1);
        }
    }
    if (!state.useContext && s.configs.getConflictInfo() != null) {
        if (dfa.atnStartState instanceof DecisionState) {
            if (!userWantsCtxSensitive || (!s.configs.getDipsIntoOuterContext() && s.configs.isExactConflict()) || (treat_sllk1_conflict_as_ambiguity && input.index() == startIndex)) {
            // we don't report the ambiguity again
            // if ( !acceptState.configset.hasSemanticContext() ) {
            // reportAmbiguity(dfa, acceptState, startIndex, input.index(), acceptState.configset.getConflictingAlts(), acceptState.configset);
            // }
            } else {
                assert !state.useContext;
                // Before attempting full context prediction, check to see if there are
                // disambiguating or validating predicates to evaluate which allow an
                // immediate decision
                BitSet conflictingAlts = null;
                DFAState.PredPrediction[] predicates = s.predicates;
                if (predicates != null) {
                    int conflictIndex = input.index();
                    if (conflictIndex != startIndex) {
                        input.seek(startIndex);
                    }
                    conflictingAlts = evalSemanticContext(predicates, outerContext, true);
                    if (conflictingAlts.cardinality() == 1) {
                        return conflictingAlts.nextSetBit(0);
                    }
                    if (conflictIndex != startIndex) {
                        // restore the index so reporting the fallback to full
                        // context occurs with the index at the correct spot
                        input.seek(conflictIndex);
                    }
                }
                if (reportAmbiguities) {
                    SimulatorState conflictState = new SimulatorState(outerContext, s, state.useContext, remainingOuterContext);
                    reportAttemptingFullContext(dfa, conflictingAlts, conflictState, startIndex, input.index());
                }
                input.seek(startIndex);
                return adaptivePredict(input, dfa.decision, outerContext, true);
            }
        }
    }
    // Before jumping to prediction, check to see if there are
    // disambiguating or validating predicates to evaluate
    DFAState.PredPrediction[] predicates = s.predicates;
    if (predicates != null) {
        int stopIndex = input.index();
        if (startIndex != stopIndex) {
            input.seek(startIndex);
        }
        BitSet alts = evalSemanticContext(predicates, outerContext, reportAmbiguities && predictionMode == PredictionMode.LL_EXACT_AMBIG_DETECTION);
        switch(alts.cardinality()) {
            case 0:
                throw noViableAlt(input, outerContext, s.configs, startIndex);
            case 1:
                return alts.nextSetBit(0);
            default:
                // set of ambig alts is reported.
                if (startIndex != stopIndex) {
                    input.seek(stopIndex);
                }
                reportAmbiguity(dfa, s, startIndex, stopIndex, s.configs.isExactConflict(), alts, s.configs);
                return alts.nextSetBit(0);
        }
    }
    if (dfa_debug)
        System.out.println("DFA decision " + dfa.decision + " predicts " + s.getPrediction());
    return s.getPrediction();
}

Example 27 with DFA

use of org.antlr.v4.runtime.dfa.DFA in project antlr4 by tunnelvisionlabs.

the class ParserATNSimulator method execATN.

/**
 * Performs ATN simulation to compute a predicted alternative based
 *  upon the remaining input, but also updates the DFA cache to avoid
 *  having to traverse the ATN again for the same input sequence.
 *
 *	 There are some key conditions we're looking for after computing a new
 *	 set of ATN configs (proposed DFA state):
 * if the set is empty, there is no viable alternative for current symbol
 * does the state uniquely predict an alternative?
 * does the state have a conflict that would prevent us from
 *	         putting it on the work list?
 * if in non-greedy decision is there a config at a rule stop state?
 *
 *	 We also have some key operations to do:
 * add an edge from previous DFA state to potentially new DFA state, D,
 *	         upon current symbol but only if adding to work list, which means in all
 *	         cases except no viable alternative (and possibly non-greedy decisions?)
 * collecting predicates and adding semantic context to DFA accept states
 * adding rule context to context-sensitive DFA accept states
 * consuming an input symbol
 * reporting a conflict
 * reporting an ambiguity
 * reporting a context sensitivity
 * reporting insufficient predicates
 *
 *	 We should isolate those operations, which are side-effecting, to the
 *	 main work loop. We can isolate lots of code into other functions, but
 *	 they should be side effect free. They can return package that
 *	 indicates whether we should report something, whether we need to add a
 *	 DFA edge, whether we need to augment accept state with semantic
 *	 context or rule invocation context. Actually, it seems like we always
 *	 add predicates if they exist, so that can simply be done in the main
 *	 loop for any accept state creation or modification request.
 *
 *	 cover these cases:
 *	    dead end
 *	    single alt
 *	    single alt + preds
 *	    conflict
 *	    conflict + preds
 *
 *	 TODO: greedy + those
 */
protected int execATN(@NotNull DFA dfa, @NotNull TokenStream input, int startIndex, @NotNull SimulatorState initialState) {
    if (debug)
        System.out.println("execATN decision " + dfa.decision + " exec LA(1)==" + getLookaheadName(input));
    final ParserRuleContext outerContext = initialState.outerContext;
    final boolean useContext = initialState.useContext;
    int t = input.LA(1);
    SimulatorState previous = initialState;
    PredictionContextCache contextCache = new PredictionContextCache();
    while (true) {
        // while more work
        SimulatorState nextState = computeReachSet(dfa, previous, t, contextCache);
        if (nextState == null) {
            addDFAEdge(previous.s0, input.LA(1), ERROR);
            return handleNoViableAlt(input, startIndex, previous);
        }
        DFAState D = nextState.s0;
        // predicted alt => accept state
        assert D.isAcceptState() || D.getPrediction() == ATN.INVALID_ALT_NUMBER;
        // conflicted => accept state
        assert D.isAcceptState() || D.configs.getConflictInfo() == null;
        if (isAcceptState(D, useContext)) {
            BitSet conflictingAlts = D.configs.getConflictingAlts();
            int predictedAlt = conflictingAlts == null ? D.getPrediction() : ATN.INVALID_ALT_NUMBER;
            if (predictedAlt != ATN.INVALID_ALT_NUMBER) {
                if (optimize_ll1 && input.index() == startIndex && !dfa.isPrecedenceDfa() && nextState.outerContext == nextState.remainingOuterContext && dfa.decision >= 0 && !D.configs.hasSemanticContext()) {
                    if (t >= 0 && t <= Short.MAX_VALUE) {
                        int key = (dfa.decision << 16) + t;
                        atn.LL1Table.put(key, predictedAlt);
                    }
                }
                if (useContext && always_try_local_context) {
                    reportContextSensitivity(dfa, predictedAlt, nextState, startIndex, input.index());
                }
            }
            predictedAlt = D.getPrediction();
            // int k = input.index() - startIndex + 1; // how much input we used
            // System.out.println("used k="+k);
            boolean attemptFullContext = conflictingAlts != null && userWantsCtxSensitive;
            if (attemptFullContext) {
                // Only exact conflicts are known to be ambiguous when local
                // prediction does not step out of the decision rule.
                attemptFullContext = !useContext && (D.configs.getDipsIntoOuterContext() || !D.configs.isExactConflict()) && (!treat_sllk1_conflict_as_ambiguity || input.index() != startIndex);
            }
            if (D.configs.hasSemanticContext()) {
                DFAState.PredPrediction[] predPredictions = D.predicates;
                if (predPredictions != null) {
                    int conflictIndex = input.index();
                    if (conflictIndex != startIndex) {
                        input.seek(startIndex);
                    }
                    // use complete evaluation here if we'll want to retry with full context if still ambiguous
                    conflictingAlts = evalSemanticContext(predPredictions, outerContext, attemptFullContext || reportAmbiguities);
                    switch(conflictingAlts.cardinality()) {
                        case 0:
                            throw noViableAlt(input, outerContext, D.configs, startIndex);
                        case 1:
                            return conflictingAlts.nextSetBit(0);
                        default:
                            break;
                    }
                    if (conflictIndex != startIndex) {
                        // restore the index so reporting the fallback to full
                        // context occurs with the index at the correct spot
                        input.seek(conflictIndex);
                    }
                }
            }
            if (!attemptFullContext) {
                if (conflictingAlts != null) {
                    if (reportAmbiguities && conflictingAlts.cardinality() > 1) {
                        reportAmbiguity(dfa, D, startIndex, input.index(), D.configs.isExactConflict(), conflictingAlts, D.configs);
                    }
                    predictedAlt = conflictingAlts.nextSetBit(0);
                }
                return predictedAlt;
            } else {
                assert !useContext;
                assert isAcceptState(D, false);
                if (debug)
                    System.out.println("RETRY with outerContext=" + outerContext);
                SimulatorState fullContextState = computeStartState(dfa, outerContext, true);
                if (reportAmbiguities) {
                    reportAttemptingFullContext(dfa, conflictingAlts, nextState, startIndex, input.index());
                }
                input.seek(startIndex);
                return execATN(dfa, input, startIndex, fullContextState);
            }
        }
        previous = nextState;
        if (t != IntStream.EOF) {
            input.consume();
            t = input.LA(1);
        }
    }
}

Example 28 with DFA

use of org.antlr.v4.runtime.dfa.DFA in project antlr4 by tunnelvisionlabs.

the class ParserATNSimulator method computeStartState.

@NotNull
protected SimulatorState computeStartState(DFA dfa, ParserRuleContext globalContext, boolean useContext) {
    DFAState s0 = dfa.isPrecedenceDfa() ? dfa.getPrecedenceStartState(parser.getPrecedence(), useContext) : useContext ? dfa.s0full.get() : dfa.s0.get();
    if (s0 != null) {
        if (!useContext) {
            return new SimulatorState(globalContext, s0, useContext, globalContext);
        }
        s0.setContextSensitive(atn);
    }
    final int decision = dfa.decision;
    @NotNull final ATNState p = dfa.atnStartState;
    int previousContext = 0;
    ParserRuleContext remainingGlobalContext = globalContext;
    // always at least the implicit call to start rule
    PredictionContext initialContext = useContext ? PredictionContext.EMPTY_FULL : PredictionContext.EMPTY_LOCAL;
    PredictionContextCache contextCache = new PredictionContextCache();
    if (useContext) {
        if (!enable_global_context_dfa) {
            while (remainingGlobalContext != null) {
                if (remainingGlobalContext.isEmpty()) {
                    previousContext = PredictionContext.EMPTY_FULL_STATE_KEY;
                    remainingGlobalContext = null;
                } else {
                    previousContext = getReturnState(remainingGlobalContext);
                    initialContext = initialContext.appendContext(previousContext, contextCache);
                    remainingGlobalContext = remainingGlobalContext.getParent();
                }
            }
        }
        while (s0 != null && s0.isContextSensitive() && remainingGlobalContext != null) {
            DFAState next;
            remainingGlobalContext = skipTailCalls(remainingGlobalContext);
            if (remainingGlobalContext.isEmpty()) {
                next = s0.getContextTarget(PredictionContext.EMPTY_FULL_STATE_KEY);
                previousContext = PredictionContext.EMPTY_FULL_STATE_KEY;
                remainingGlobalContext = null;
            } else {
                previousContext = getReturnState(remainingGlobalContext);
                next = s0.getContextTarget(previousContext);
                initialContext = initialContext.appendContext(previousContext, contextCache);
                remainingGlobalContext = remainingGlobalContext.getParent();
            }
            if (next == null) {
                break;
            }
            s0 = next;
        }
    }
    if (s0 != null && !s0.isContextSensitive()) {
        return new SimulatorState(globalContext, s0, useContext, remainingGlobalContext);
    }
    ATNConfigSet configs = new ATNConfigSet();
    while (true) {
        ATNConfigSet reachIntermediate = new ATNConfigSet();
        int n = p.getNumberOfTransitions();
        for (int ti = 0; ti < n; ti++) {
            // for each transition
            ATNState target = p.transition(ti).target;
            reachIntermediate.add(ATNConfig.create(target, ti + 1, initialContext));
        }
        boolean hasMoreContext = remainingGlobalContext != null;
        if (!hasMoreContext) {
            configs.setOutermostConfigSet(true);
        }
        final boolean collectPredicates = true;
        closure(reachIntermediate, configs, collectPredicates, hasMoreContext, contextCache, false);
        boolean stepIntoGlobal = configs.getDipsIntoOuterContext();
        DFAState next;
        if (useContext && !enable_global_context_dfa) {
            s0 = addDFAState(dfa, configs, contextCache);
            break;
        } else if (s0 == null) {
            if (!dfa.isPrecedenceDfa()) {
                AtomicReference<DFAState> reference = useContext ? dfa.s0full : dfa.s0;
                next = addDFAState(dfa, configs, contextCache);
                if (!reference.compareAndSet(null, next)) {
                    next = reference.get();
                }
            } else {
                /* If this is a precedence DFA, we use applyPrecedenceFilter
					 * to convert the computed start state to a precedence start
					 * state. We then use DFA.setPrecedenceStartState to set the
					 * appropriate start state for the precedence level rather
					 * than simply setting DFA.s0.
					 */
                configs = applyPrecedenceFilter(configs, globalContext, contextCache);
                next = addDFAState(dfa, configs, contextCache);
                dfa.setPrecedenceStartState(parser.getPrecedence(), useContext, next);
            }
        } else {
            if (dfa.isPrecedenceDfa()) {
                configs = applyPrecedenceFilter(configs, globalContext, contextCache);
            }
            next = addDFAState(dfa, configs, contextCache);
            s0.setContextTarget(previousContext, next);
        }
        s0 = next;
        if (!useContext || !stepIntoGlobal) {
            break;
        }
        // TODO: make sure it distinguishes empty stack states
        next.setContextSensitive(atn);
        configs.clear();
        remainingGlobalContext = skipTailCalls(remainingGlobalContext);
        int nextContextElement = getReturnState(remainingGlobalContext);
        if (remainingGlobalContext.isEmpty()) {
            remainingGlobalContext = null;
        } else {
            remainingGlobalContext = remainingGlobalContext.getParent();
        }
        if (nextContextElement != PredictionContext.EMPTY_FULL_STATE_KEY) {
            initialContext = initialContext.appendContext(nextContextElement, contextCache);
        }
        previousContext = nextContextElement;
    }
    return new SimulatorState(globalContext, s0, useContext, remainingGlobalContext);
}

Example 29 with DFA

use of org.antlr.v4.runtime.dfa.DFA in project antlr4 by tunnelvisionlabs.

the class ParserATNSimulator method adaptivePredict.

public int adaptivePredict(@NotNull TokenStream input, int decision, @Nullable ParserRuleContext outerContext, boolean useContext) {
    DFA dfa = atn.decisionToDFA[decision];
    assert dfa != null;
    if (optimize_ll1 && !dfa.isPrecedenceDfa() && !dfa.isEmpty()) {
        int ll_1 = input.LA(1);
        if (ll_1 >= 0 && ll_1 <= Short.MAX_VALUE) {
            int key = (decision << 16) + ll_1;
            Integer alt = atn.LL1Table.get(key);
            if (alt != null) {
                return alt;
            }
        }
    }
    this.dfa = dfa;
    if (force_global_context) {
        useContext = true;
    } else if (!always_try_local_context) {
        useContext |= dfa.isContextSensitive();
    }
    userWantsCtxSensitive = useContext || (predictionMode != PredictionMode.SLL && outerContext != null && !atn.decisionToState.get(decision).sll);
    if (outerContext == null) {
        outerContext = ParserRuleContext.emptyContext();
    }
    SimulatorState state = null;
    if (!dfa.isEmpty()) {
        state = getStartState(dfa, input, outerContext, useContext);
    }
    if (state == null) {
        if (outerContext == null)
            outerContext = ParserRuleContext.emptyContext();
        if (debug)
            System.out.println("ATN decision " + dfa.decision + " exec LA(1)==" + getLookaheadName(input) + ", outerContext=" + outerContext.toString(parser));
        state = computeStartState(dfa, outerContext, useContext);
    }
    int m = input.mark();
    int index = input.index();
    try {
        int alt = execDFA(dfa, input, index, state);
        if (debug)
            System.out.println("DFA after predictATN: " + dfa.toString(parser.getVocabulary(), parser.getRuleNames()));
        return alt;
    } finally {
        this.dfa = null;
        input.seek(index);
        input.release(m);
    }
}

Example 30 with DFA

use of org.antlr.v4.runtime.dfa.DFA in project antlr4 by tunnelvisionlabs.

the class ParserATNSimulator method addDFAEdge.

@NotNull
protected DFAState addDFAEdge(@NotNull DFA dfa, @NotNull DFAState fromState, int t, IntegerList contextTransitions, @NotNull ATNConfigSet toConfigs, PredictionContextCache contextCache) {
    assert contextTransitions == null || contextTransitions.isEmpty() || dfa.isContextSensitive();
    DFAState from = fromState;
    DFAState to = addDFAState(dfa, toConfigs, contextCache);
    if (contextTransitions != null) {
        for (int context : contextTransitions.toArray()) {
            if (context == PredictionContext.EMPTY_FULL_STATE_KEY) {
                if (from.configs.isOutermostConfigSet()) {
                    continue;
                }
            }
            from.setContextSensitive(atn);
            from.setContextSymbol(t);
            DFAState next = from.getContextTarget(context);
            if (next != null) {
                from = next;
                continue;
            }
            next = addDFAContextState(dfa, from.configs, context, contextCache);
            assert context != PredictionContext.EMPTY_FULL_STATE_KEY || next.configs.isOutermostConfigSet();
            from.setContextTarget(context, next);
            from = next;
        }
    }
    if (debug)
        System.out.println("EDGE " + from + " -> " + to + " upon " + getTokenName(t));
    addDFAEdge(from, t, to);
    if (debug)
        System.out.println("DFA=\n" + dfa.toString(parser != null ? parser.getVocabulary() : VocabularyImpl.EMPTY_VOCABULARY, parser != null ? parser.getRuleNames() : null));
    return to;
}