Examples with Recognizer org.antlr.v4.runtime.Recognizer used on opensource projects

Example 66 with Recognizer

use of org.antlr.v4.runtime.Recognizer in project antlr4 by tunnelvisionlabs.

the class DefaultErrorStrategy method sync.

/**
 * The default implementation of {@link ANTLRErrorStrategy#sync} makes sure
 * that the current lookahead symbol is consistent with what were expecting
 * at this point in the ATN. You can call this anytime but ANTLR only
 * generates code to check before subrules/loops and each iteration.
 *
 * <p>Implements Jim Idle's magic sync mechanism in closures and optional
 * subrules. E.g.,</p>
 *
 * <pre>
 * a : sync ( stuff sync )* ;
 * sync : {consume to what can follow sync} ;
 * </pre>
 *
 * At the start of a sub rule upon error, {@link #sync} performs single
 * token deletion, if possible. If it can't do that, it bails on the current
 * rule and uses the default error recovery, which consumes until the
 * resynchronization set of the current rule.
 *
 * <p>If the sub rule is optional ({@code (...)?}, {@code (...)*}, or block
 * with an empty alternative), then the expected set includes what follows
 * the subrule.</p>
 *
 * <p>During loop iteration, it consumes until it sees a token that can start a
 * sub rule or what follows loop. Yes, that is pretty aggressive. We opt to
 * stay in the loop as long as possible.</p>
 *
 * <p><strong>ORIGINS</strong></p>
 *
 * <p>Previous versions of ANTLR did a poor job of their recovery within loops.
 * A single mismatch token or missing token would force the parser to bail
 * out of the entire rules surrounding the loop. So, for rule</p>
 *
 * <pre>
 * classDef : 'class' ID '{' member* '}'
 * </pre>
 *
 * input with an extra token between members would force the parser to
 * consume until it found the next class definition rather than the next
 * member definition of the current class.
 *
 * <p>This functionality cost a little bit of effort because the parser has to
 * compare token set at the start of the loop and at each iteration. If for
 * some reason speed is suffering for you, you can turn off this
 * functionality by simply overriding this method as a blank { }.</p>
 */
@Override
public void sync(Parser recognizer) throws RecognitionException {
    ATNState s = recognizer.getInterpreter().atn.states.get(recognizer.getState());
    // If already recovering, don't try to sync
    if (inErrorRecoveryMode(recognizer)) {
        return;
    }
    TokenStream tokens = recognizer.getInputStream();
    int la = tokens.LA(1);
    // try cheaper subset first; might get lucky. seems to shave a wee bit off
    IntervalSet nextTokens = recognizer.getATN().nextTokens(s);
    if (nextTokens.contains(la)) {
        // We are sure the token matches
        nextTokensContext = null;
        nextTokensState = ATNState.INVALID_STATE_NUMBER;
        return;
    }
    if (nextTokens.contains(Token.EPSILON)) {
        if (nextTokensContext == null) {
            // It's possible the next token won't match; information tracked
            // by sync is restricted for performance.
            nextTokensContext = recognizer.getContext();
            nextTokensState = recognizer.getState();
        }
        return;
    }
    switch(s.getStateType()) {
        case ATNState.BLOCK_START:
        case ATNState.STAR_BLOCK_START:
        case ATNState.PLUS_BLOCK_START:
        case ATNState.STAR_LOOP_ENTRY:
            // report error and recover if possible
            if (singleTokenDeletion(recognizer) != null) {
                return;
            }
            throw new InputMismatchException(recognizer);
        case ATNState.PLUS_LOOP_BACK:
        case ATNState.STAR_LOOP_BACK:
            // System.err.println("at loop back: "+s.getClass().getSimpleName());
            reportUnwantedToken(recognizer);
            IntervalSet expecting = recognizer.getExpectedTokens();
            IntervalSet whatFollowsLoopIterationOrRule = expecting.or(getErrorRecoverySet(recognizer));
            consumeUntil(recognizer, whatFollowsLoopIterationOrRule);
            break;
        default:
            // do nothing if we can't identify the exact kind of ATN state
            break;
    }
}

Example 67 with Recognizer

use of org.antlr.v4.runtime.Recognizer in project antlr4 by tunnelvisionlabs.

the class DefaultErrorStrategy method recover.

/**
 * {@inheritDoc}
 *
 * <p>The default implementation resynchronizes the parser by consuming tokens
 * until we find one in the resynchronization set--loosely the set of tokens
 * that can follow the current rule.</p>
 */
@Override
public void recover(Parser recognizer, RecognitionException e) {
    // ", states="+lastErrorStates);
    if (lastErrorIndex == recognizer.getInputStream().index() && lastErrorStates != null && lastErrorStates.contains(recognizer.getState())) {
        // uh oh, another error at same token index and previously-visited
        // state in ATN; must be a case where LT(1) is in the recovery
        // token set so nothing got consumed. Consume a single token
        // at least to prevent an infinite loop; this is a failsafe.
        // System.err.println("seen error condition before index="+
        // lastErrorIndex+", states="+lastErrorStates);
        // System.err.println("FAILSAFE consumes "+recognizer.getTokenNames()[recognizer.getInputStream().LA(1)]);
        recognizer.consume();
    }
    lastErrorIndex = recognizer.getInputStream().index();
    if (lastErrorStates == null)
        lastErrorStates = new IntervalSet();
    lastErrorStates.add(recognizer.getState());
    IntervalSet followSet = getErrorRecoverySet(recognizer);
    consumeUntil(recognizer, followSet);
}

Example 68 with Recognizer

use of org.antlr.v4.runtime.Recognizer in project antlr4 by tunnelvisionlabs.

the class DefaultErrorStrategy method singleTokenInsertion.

/**
 * This method implements the single-token insertion inline error recovery
 * strategy. It is called by {@link #recoverInline} if the single-token
 * deletion strategy fails to recover from the mismatched input. If this
 * method returns {@code true}, {@code recognizer} will be in error recovery
 * mode.
 *
 * <p>This method determines whether or not single-token insertion is viable by
 * checking if the {@code LA(1)} input symbol could be successfully matched
 * if it were instead the {@code LA(2)} symbol. If this method returns
 * {@code true}, the caller is responsible for creating and inserting a
 * token with the correct type to produce this behavior.</p>
 *
 * @param recognizer the parser instance
 * @return {@code true} if single-token insertion is a viable recovery
 * strategy for the current mismatched input, otherwise {@code false}
 */
protected boolean singleTokenInsertion(@NotNull Parser recognizer) {
    int currentSymbolType = recognizer.getInputStream().LA(1);
    // if current token is consistent with what could come after current
    // ATN state, then we know we're missing a token; error recovery
    // is free to conjure up and insert the missing token
    ATNState currentState = recognizer.getInterpreter().atn.states.get(recognizer.getState());
    ATNState next = currentState.transition(0).target;
    ATN atn = recognizer.getInterpreter().atn;
    IntervalSet expectingAtLL2 = atn.nextTokens(next, PredictionContext.fromRuleContext(atn, recognizer._ctx));
    // System.out.println("LT(2) set="+expectingAtLL2.toString(recognizer.getTokenNames()));
    if (expectingAtLL2.contains(currentSymbolType)) {
        reportMissingToken(recognizer);
        return true;
    }
    return false;
}

Example 69 with Recognizer

use of org.antlr.v4.runtime.Recognizer in project checkstyle by checkstyle.

the class JavadocDetailNodeParser method getFirstNonTightHtmlTag.

/**
 * This method is used to get the first non-tight HTML tag encountered while parsing javadoc.
 * This shall eventually be reflected by the {@link ParseStatus} object returned by
 * {@link #parseJavadocAsDetailNode(DetailAST)} method via the instance member
 * {@link ParseStatus#firstNonTightHtmlTag}, and checks not supposed to process non-tight HTML
 * or the ones which are supposed to log violation for non-tight javadocs can utilize that.
 *
 * @param javadocParser The ANTLR recognizer instance which has been used to parse the javadoc
 * @param javadocLineOffset The line number of beginning of the Javadoc comment
 * @return First non-tight HTML tag if one exists; null otherwise
 */
private static Token getFirstNonTightHtmlTag(JavadocParser javadocParser, int javadocLineOffset) {
    final CommonToken offendingToken;
    final ParserRuleContext nonTightTagStartContext = javadocParser.nonTightTagStartContext;
    if (nonTightTagStartContext == null) {
        offendingToken = null;
    } else {
        final Token token = ((TerminalNode) nonTightTagStartContext.getChild(1)).getSymbol();
        offendingToken = new CommonToken(token);
        offendingToken.setLine(offendingToken.getLine() + javadocLineOffset);
    }
    return offendingToken;
}

Example 70 with Recognizer

use of org.antlr.v4.runtime.Recognizer in project antlr4 by antlr.

the class BaseJavaTest method execClass.

public String execClass(String className) {
    if (TEST_IN_SAME_PROCESS) {
        try {
            ClassLoader loader = new URLClassLoader(new URL[] { getTempTestDir().toURI().toURL() }, ClassLoader.getSystemClassLoader());
            final Class<?> mainClass = (Class<?>) loader.loadClass(className);
            final Method mainMethod = mainClass.getDeclaredMethod("main", String[].class);
            PipedInputStream stdoutIn = new PipedInputStream();
            PipedInputStream stderrIn = new PipedInputStream();
            PipedOutputStream stdoutOut = new PipedOutputStream(stdoutIn);
            PipedOutputStream stderrOut = new PipedOutputStream(stderrIn);
            StreamVacuum stdoutVacuum = new StreamVacuum(stdoutIn);
            StreamVacuum stderrVacuum = new StreamVacuum(stderrIn);
            PrintStream originalOut = System.out;
            System.setOut(new PrintStream(stdoutOut));
            try {
                PrintStream originalErr = System.err;
                try {
                    System.setErr(new PrintStream(stderrOut));
                    stdoutVacuum.start();
                    stderrVacuum.start();
                    mainMethod.invoke(null, (Object) new String[] { new File(getTempTestDir(), "input").getAbsolutePath() });
                } finally {
                    System.setErr(originalErr);
                }
            } finally {
                System.setOut(originalOut);
            }
            stdoutOut.close();
            stderrOut.close();
            stdoutVacuum.join();
            stderrVacuum.join();
            String output = stdoutVacuum.toString();
            if (output.length() == 0) {
                output = null;
            }
            if (stderrVacuum.toString().length() > 0) {
                setParseErrors(stderrVacuum.toString());
            }
            return output;
        } catch (Exception ex) {
            throw new RuntimeException(ex);
        }
    }
    try {
        String[] args = new String[] { "java", "-classpath", getTempDirPath() + PATH_SEP + CLASSPATH, "-Dfile.encoding=UTF-8", className, new File(getTempTestDir(), "input").getAbsolutePath() };
        // String cmdLine = Utils.join(args, " ");
        // System.err.println("execParser: "+cmdLine);
        Process process = Runtime.getRuntime().exec(args, null, getTempTestDir());
        StreamVacuum stdoutVacuum = new StreamVacuum(process.getInputStream());
        StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream());
        stdoutVacuum.start();
        stderrVacuum.start();
        process.waitFor();
        stdoutVacuum.join();
        stderrVacuum.join();
        String output = stdoutVacuum.toString();
        if (output.length() == 0) {
            output = null;
        }
        if (stderrVacuum.toString().length() > 0) {
            setParseErrors(stderrVacuum.toString());
        }
        return output;
    } catch (Exception e) {
        System.err.println("can't exec recognizer");
        e.printStackTrace(System.err);
    }
    return null;
}