Examples with EOF org.antlr.v4.runtime.Recognizer.EOF used on opensource projects

Example 21 with EOF

use of org.antlr.v4.runtime.Recognizer.EOF in project antlr4 by antlr.

the class TestCodePointCharStream method consumingPastSingleCJKCodePointShouldThrow.

@Test
public void consumingPastSingleCJKCodePointShouldThrow() {
    CodePointCharStream s = CharStreams.fromString("愛");
    s.consume();
    thrown.expect(IllegalStateException.class);
    thrown.expectMessage("cannot consume EOF");
    s.consume();
}

Example 22 with EOF

use of org.antlr.v4.runtime.Recognizer.EOF in project antlr4 by antlr.

the class BaseBrowserTest method getTokenTypes.

public List<String> getTokenTypes(LexerGrammar lg, ATN atn, CharStream input) {
    LexerATNSimulator interp = new LexerATNSimulator(atn, new DFA[] { new DFA(atn.modeToStartState.get(Lexer.DEFAULT_MODE)) }, null);
    List<String> tokenTypes = new ArrayList<String>();
    int ttype;
    boolean hitEOF = false;
    do {
        if (hitEOF) {
            tokenTypes.add("EOF");
            break;
        }
        int t = input.LA(1);
        ttype = interp.match(input, Lexer.DEFAULT_MODE);
        if (ttype == Token.EOF) {
            tokenTypes.add("EOF");
        } else {
            tokenTypes.add(lg.typeToTokenList.get(ttype));
        }
        if (t == IntStream.EOF) {
            hitEOF = true;
        }
    } while (ttype != Token.EOF);
    return tokenTypes;
}

Example 23 with EOF

use of org.antlr.v4.runtime.Recognizer.EOF in project antlr4 by antlr.

the class BaseNodeTest method getTokenTypes.

public List<String> getTokenTypes(LexerGrammar lg, ATN atn, CharStream input) {
    LexerATNSimulator interp = new LexerATNSimulator(atn, new DFA[] { new DFA(atn.modeToStartState.get(Lexer.DEFAULT_MODE)) }, null);
    List<String> tokenTypes = new ArrayList<String>();
    int ttype;
    boolean hitEOF = false;
    do {
        if (hitEOF) {
            tokenTypes.add("EOF");
            break;
        }
        int t = input.LA(1);
        ttype = interp.match(input, Lexer.DEFAULT_MODE);
        if (ttype == Token.EOF) {
            tokenTypes.add("EOF");
        } else {
            tokenTypes.add(lg.typeToTokenList.get(ttype));
        }
        if (t == IntStream.EOF) {
            hitEOF = true;
        }
    } while (ttype != Token.EOF);
    return tokenTypes;
}

Example 24 with EOF

use of org.antlr.v4.runtime.Recognizer.EOF in project java by wavefrontHQ.

the class AbstractIngesterFormatter method getQueue.

protected Queue<Token> getQueue(String input) {
    DSWrapperLexer lexer = dsWrapperLexerThreadLocal.get();
    lexer.setInputStream(new ANTLRInputStream(input));
    CommonTokenStream commonTokenStream = new CommonTokenStream(lexer);
    commonTokenStream.fill();
    List<Token> tokens = commonTokenStream.getTokens();
    if (tokens.isEmpty()) {
        throw new RuntimeException("Could not parse: " + input);
    }
    // this is sensitive to the grammar in DSQuery.g4. We could just use the visitor but doing so
    // means we need to be creating the AST and instead we could just use the lexer. in any case,
    // we don't expect the graphite format to change anytime soon.
    // filter all EOF tokens first.
    Queue<Token> queue = tokens.stream().filter(t -> t.getType() != Lexer.EOF).collect(Collectors.toCollection(ArrayDeque::new));
    return queue;
}

Example 25 with EOF

use of org.antlr.v4.runtime.Recognizer.EOF in project ksql by confluentinc.

the class KsqlParserErrorStrategy method reportNoViableAlternative.

protected void reportNoViableAlternative(final Parser recognizer, final NoViableAltException e) {
    final TokenStream tokens = recognizer.getInputStream();
    final String input;
    if (tokens != null) {
        if (e.getStartToken().getType() == -1) {
            input = "<EOF>";
        } else {
            input = tokens.getText(e.getStartToken(), e.getOffendingToken());
        }
    } else {
        input = "<unknown input>";
    }
    final String msg = "no viable alternative at input " + this.escapeWSAndQuote(input);
    recognizer.notifyErrorListeners(e.getOffendingToken(), msg, e);
}