Examples with TokenStream org.antlr.v4.runtime.TokenStream used on opensource projects

Example 21 with TokenStream

use of org.antlr.v4.runtime.TokenStream in project antlr4 by antlr.

the class DefaultErrorStrategy method singleTokenDeletion.

/**
	 * This method implements the single-token deletion inline error recovery
	 * strategy. It is called by {@link #recoverInline} to attempt to recover
	 * from mismatched input. If this method returns null, the parser and error
	 * handler state will not have changed. If this method returns non-null,
	 * {@code recognizer} will <em>not</em> be in error recovery mode since the
	 * returned token was a successful match.
	 *
	 * <p>If the single-token deletion is successful, this method calls
	 * {@link #reportUnwantedToken} to report the error, followed by
	 * {@link Parser#consume} to actually "delete" the extraneous token. Then,
	 * before returning {@link #reportMatch} is called to signal a successful
	 * match.</p>
	 *
	 * @param recognizer the parser instance
	 * @return the successfully matched {@link Token} instance if single-token
	 * deletion successfully recovers from the mismatched input, otherwise
	 * {@code null}
	 */
protected Token singleTokenDeletion(Parser recognizer) {
    int nextTokenType = recognizer.getInputStream().LA(2);
    IntervalSet expecting = getExpectedTokens(recognizer);
    if (expecting.contains(nextTokenType)) {
        reportUnwantedToken(recognizer);
        /*
			System.err.println("recoverFromMismatchedToken deleting "+
							   ((TokenStream)recognizer.getInputStream()).LT(1)+
							   " since "+((TokenStream)recognizer.getInputStream()).LT(2)+
							   " is what we want");
			*/
        // simply delete extra token
        recognizer.consume();
        // we want to return the token we're actually matching
        Token matchedSymbol = recognizer.getCurrentToken();
        // we know current token is correct
        reportMatch(recognizer);
        return matchedSymbol;
    }
    return null;
}

Example 22 with TokenStream

use of org.antlr.v4.runtime.TokenStream in project antlr4 by antlr.

the class ParserATNSimulator method execATNWithFullContext.

// comes back with reach.uniqueAlt set to a valid alt
protected int execATNWithFullContext(DFA dfa, // how far we got in SLL DFA before failing over
DFAState D, ATNConfigSet s0, TokenStream input, int startIndex, ParserRuleContext outerContext) {
    if (debug || debug_list_atn_decisions) {
        System.out.println("execATNWithFullContext " + s0);
    }
    boolean fullCtx = true;
    boolean foundExactAmbig = false;
    ATNConfigSet reach = null;
    ATNConfigSet previous = s0;
    input.seek(startIndex);
    int t = input.LA(1);
    int predictedAlt;
    while (true) {
        // while more work
        //			System.out.println("LL REACH "+getLookaheadName(input)+
        //							   " from configs.size="+previous.size()+
        //							   " line "+input.LT(1).getLine()+":"+input.LT(1).getCharPositionInLine());
        reach = computeReachSet(previous, t, fullCtx);
        if (reach == null) {
            // if any configs in previous dipped into outer context, that
            // means that input up to t actually finished entry rule
            // at least for LL decision. Full LL doesn't dip into outer
            // so don't need special case.
            // We will get an error no matter what so delay until after
            // decision; better error message. Also, no reachable target
            // ATN states in SLL implies LL will also get nowhere.
            // If conflict in states that dip out, choose min since we
            // will get error no matter what.
            NoViableAltException e = noViableAlt(input, outerContext, previous, startIndex);
            input.seek(startIndex);
            int alt = getSynValidOrSemInvalidAltThatFinishedDecisionEntryRule(previous, outerContext);
            if (alt != ATN.INVALID_ALT_NUMBER) {
                return alt;
            }
            throw e;
        }
        Collection<BitSet> altSubSets = PredictionMode.getConflictingAltSubsets(reach);
        if (debug) {
            System.out.println("LL altSubSets=" + altSubSets + ", predict=" + PredictionMode.getUniqueAlt(altSubSets) + ", resolvesToJustOneViableAlt=" + PredictionMode.resolvesToJustOneViableAlt(altSubSets));
        }
        //			System.out.println("altSubSets: "+altSubSets);
        //			System.err.println("reach="+reach+", "+reach.conflictingAlts);
        reach.uniqueAlt = getUniqueAlt(reach);
        // unique prediction?
        if (reach.uniqueAlt != ATN.INVALID_ALT_NUMBER) {
            predictedAlt = reach.uniqueAlt;
            break;
        }
        if (mode != PredictionMode.LL_EXACT_AMBIG_DETECTION) {
            predictedAlt = PredictionMode.resolvesToJustOneViableAlt(altSubSets);
            if (predictedAlt != ATN.INVALID_ALT_NUMBER) {
                break;
            }
        } else {
            // Just keeps scarfing until we know what the conflict is
            if (PredictionMode.allSubsetsConflict(altSubSets) && PredictionMode.allSubsetsEqual(altSubSets)) {
                foundExactAmbig = true;
                predictedAlt = PredictionMode.getSingleViableAlt(altSubSets);
                break;
            }
        // else there are multiple non-conflicting subsets or
        // we're not sure what the ambiguity is yet.
        // So, keep going.
        }
        previous = reach;
        if (t != IntStream.EOF) {
            input.consume();
            t = input.LA(1);
        }
    }
    // not SLL.
    if (reach.uniqueAlt != ATN.INVALID_ALT_NUMBER) {
        reportContextSensitivity(dfa, predictedAlt, reach, startIndex, input.index());
        return predictedAlt;
    }
    // We do not check predicates here because we have checked them
    // on-the-fly when doing full context prediction.
    /*
		In non-exact ambiguity detection mode, we might	actually be able to
		detect an exact ambiguity, but I'm not going to spend the cycles
		needed to check. We only emit ambiguity warnings in exact ambiguity
		mode.

		For example, we might know that we have conflicting configurations.
		But, that does not mean that there is no way forward without a
		conflict. It's possible to have nonconflicting alt subsets as in:

		   LL altSubSets=[{1, 2}, {1, 2}, {1}, {1, 2}]

		from

		   [(17,1,[5 $]), (13,1,[5 10 $]), (21,1,[5 10 $]), (11,1,[$]),
			(13,2,[5 10 $]), (21,2,[5 10 $]), (11,2,[$])]

		In this case, (17,1,[5 $]) indicates there is some next sequence that
		would resolve this without conflict to alternative 1. Any other viable
		next sequence, however, is associated with a conflict.  We stop
		looking for input because no amount of further lookahead will alter
		the fact that we should predict alternative 1.  We just can't say for
		sure that there is an ambiguity without looking further.
		*/
    reportAmbiguity(dfa, D, startIndex, input.index(), foundExactAmbig, reach.getAlts(), reach);
    return predictedAlt;
}

Example 23 with TokenStream

use of org.antlr.v4.runtime.TokenStream in project antlr4 by antlr.

the class Grammar method createParserInterpreter.

public ParserInterpreter createParserInterpreter(TokenStream tokenStream) {
    if (this.isLexer()) {
        throw new IllegalStateException("A parser interpreter can only be created for a parser or combined grammar.");
    }
    char[] serializedAtn = ATNSerializer.getSerializedAsChars(atn);
    ATN deserialized = new ATNDeserializer().deserialize(serializedAtn);
    return new ParserInterpreter(fileName, getVocabulary(), Arrays.asList(getRuleNames()), deserialized, tokenStream);
}

Example 24 with TokenStream

use of org.antlr.v4.runtime.TokenStream in project antlr4 by antlr.

the class Grammar method createGrammarParserInterpreter.

/** @since 4.5.1 */
public GrammarParserInterpreter createGrammarParserInterpreter(TokenStream tokenStream) {
    if (this.isLexer()) {
        throw new IllegalStateException("A parser interpreter can only be created for a parser or combined grammar.");
    }
    char[] serializedAtn = ATNSerializer.getSerializedAsChars(atn);
    ATN deserialized = new ATNDeserializer().deserialize(serializedAtn);
    return new GrammarParserInterpreter(this, deserialized, tokenStream);
}

Example 25 with TokenStream

use of org.antlr.v4.runtime.TokenStream in project antlr4 by antlr.

the class TestRig method process.

public void process() throws Exception {
    //		System.out.println("exec "+grammarName+"."+startRuleName);
    String lexerName = grammarName + "Lexer";
    ClassLoader cl = Thread.currentThread().getContextClassLoader();
    Class<? extends Lexer> lexerClass = null;
    try {
        lexerClass = cl.loadClass(lexerName).asSubclass(Lexer.class);
    } catch (java.lang.ClassNotFoundException cnfe) {
        // might be pure lexer grammar; no Lexer suffix then
        lexerName = grammarName;
        try {
            lexerClass = cl.loadClass(lexerName).asSubclass(Lexer.class);
        } catch (ClassNotFoundException cnfe2) {
            System.err.println("Can't load " + lexerName + " as lexer or parser");
            return;
        }
    }
    Constructor<? extends Lexer> lexerCtor = lexerClass.getConstructor(CharStream.class);
    Lexer lexer = lexerCtor.newInstance((CharStream) null);
    Class<? extends Parser> parserClass = null;
    Parser parser = null;
    if (!startRuleName.equals(LEXER_START_RULE_NAME)) {
        String parserName = grammarName + "Parser";
        parserClass = cl.loadClass(parserName).asSubclass(Parser.class);
        Constructor<? extends Parser> parserCtor = parserClass.getConstructor(TokenStream.class);
        parser = parserCtor.newInstance((TokenStream) null);
    }
    Charset charset = (encoding == null ? Charset.defaultCharset() : Charset.forName(encoding));
    if (inputFiles.size() == 0) {
        CharStream charStream = CharStreams.fromStream(System.in, charset);
        process(lexer, parserClass, parser, charStream);
        return;
    }
    for (String inputFile : inputFiles) {
        CharStream charStream = CharStreams.fromPath(Paths.get(inputFile), charset);
        if (inputFiles.size() > 1) {
            System.err.println(inputFile);
        }
        process(lexer, parserClass, parser, charStream);
    }
}