Examples with EOF org.antlr.v4.runtime.Recognizer.EOF used on opensource projects

Example 26 with EOF

use of org.antlr.v4.runtime.Recognizer.EOF in project cafebabepy by cafebabepy.

the class CafeBabePyLexer method nextToken.

@Override
public Token nextToken() {
    boolean atStartOfInput = getCharPositionInLine() == 0 && getLine() == 1;
    readNewLine();
    if (this.opened == 0 && (this.newLineBuilder.length() > 0 || atStartOfInput)) {
        int la = readSpaces();
        if (atStartOfInput && this.spacesBuilder.length() == 0) {
        // skip
        } else {
            if (la == '\r' || la == '\n' || la == '\f') {
                return nextToken();
            } else if (la == '#') {
                return super.nextToken();
            }
            String spaces = this.spacesBuilder.toString();
            int indent = getIndentCount(spaces);
            int previous = this.indents.isEmpty() ? 0 : this.indents.peekFirst();
            if (indent == previous) {
                newLine(false);
            // skip
            } else if (indent > previous) {
                if (la != EOF) {
                    newLine(false);
                    this.indents.addFirst(indent);
                    emitOnly(createCommonToken(INDENT, spaces, this.spaceCharIndex));
                }
            } else {
                newLine(false);
                while (!this.indents.isEmpty() && this.indents.peekFirst() > indent) {
                    CommonToken dedent = new CommonToken(DEDENT, "<DEDENT>");
                    dedent.setLine(getLine());
                    emitOnly(dedent);
                    this.indents.removeFirst();
                }
            }
        }
    }
    Token next;
    while ((next = super.nextToken()) == null) ;
    int la = this._input.LA(1);
    if (la == EOF) {
        if (!this.eof) {
            for (int i = this.tokens.size() - 1; i >= 0; i--) {
                if (this.tokens.get(i).getType() == EOF) {
                    this.tokens.remove(i);
                }
            }
            if (!this.indents.isEmpty()) {
                newLine(true);
                while (!this.indents.isEmpty()) {
                    CommonToken dedent = new CommonToken(DEDENT, "<DEDENT>");
                    int dedentLine = (this.lastToken != null) ? this.lastToken.getLine() : getLine();
                    dedent.setLine(dedentLine);
                    emitOnly(dedent);
                    this.indents.removeFirst();
                }
                newEof();
            } else {
                newLine(true);
                newEof();
            }
            this.eof = true;
        }
    }
    if (next.getChannel() == Token.DEFAULT_CHANNEL) {
        this.lastToken = next;
    }
    Token token = this.tokens.isEmpty() ? next : this.tokens.poll();
    return token;
}

Example 27 with EOF

use of org.antlr.v4.runtime.Recognizer.EOF in project contribution by checkstyle.

the class ExpectedParseTreeGenerator method walk.

public String walk(ParseTree t, String parentObjectName) {
    final String className = t.getClass().getSimpleName();
    String id = null;
    if (t instanceof TerminalNode) {
        final TerminalNodeImpl terminal = (TerminalNodeImpl) t;
        final int type = terminal.symbol.getType();
        String tokenType = "";
        if (type == -1) {
            tokenType = "EOF";
        } else {
            tokenType = JavadocUtils.getTokenName(type);
        }
        String text = terminal.getText();
        if ("\n".equals(text)) {
            text = "\\n";
        } else if ("\t".equals(text)) {
            text = "\\t";
        } else {
            text = text.replace("\"", "\\\"");
        }
        final int number = getVariableCounter(tokenType);
        id = CaseFormat.LOWER_UNDERSCORE.to(CaseFormat.LOWER_CAMEL, tokenType.toLowerCase()) + number;
        System.out.println("    CommonToken " + id + " = new CommonToken(JavadocTokenTypes." + tokenType + ", \"" + text + "\");");
    } else {
        int number = getVariableCounter(className);
        id = CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_CAMEL, className) + number++;
        System.out.println("    " + className + " " + id + " = new " + className + "(" + parentObjectName + ", 0);");
        final int n = t.getChildCount();
        for (int i = 0; i < n; i++) {
            final String childId = walk(t.getChild(i), id);
            System.out.println("    " + id + ".addChild(" + childId + ");");
        }
    }
    return id;
}

Example 28 with EOF

use of org.antlr.v4.runtime.Recognizer.EOF in project titan.EclipsePlug-ins by eclipse.

the class ModuleLevelTokenStreamTracker method getBlock.

private boolean getBlock(final Token first) {
    // return true if EOF hit
    Token t;
    TokenWithIndexAndSubTokens result;
    t = getTokenSource().nextToken();
    if (t instanceof WritableToken) {
        ((WritableToken) t).setTokenIndex(tokens.size());
    }
    List<Token> tokenList = new ArrayList<Token>();
    int nofUnclosedParanthesis = 1;
    while (t != null && t.getType() != Token.EOF) {
        if (t.getType() == Asn1Lexer.BEGINCHAR) {
            nofUnclosedParanthesis++;
        } else if (t.getType() == Asn1Lexer.ENDCHAR) {
            nofUnclosedParanthesis--;
            if (nofUnclosedParanthesis == 0) {
                result = new TokenWithIndexAndSubTokens(Asn1Lexer.BLOCK, tokenList, sourceFile);
                result.setCharPositionInLine(first.getCharPositionInLine());
                result.setLine(first.getLine());
                result.setStartIndex(((TokenWithIndexAndSubTokens) first).getStopIndex());
                result.setStopIndex(((TokenWithIndexAndSubTokens) t).getStopIndex());
                result.setText(makeString(tokenList));
                tokens.add(result);
                return false;
            }
        }
        if (!discardMask.contains(Integer.valueOf(t.getType()))) {
            tokenList.add(new TokenWithIndexAndSubTokens(t));
        }
        t = getTokenSource().nextToken();
    }
    result = new TokenWithIndexAndSubTokens(Asn1Lexer.BLOCK, tokenList, sourceFile);
    result.setCharPositionInLine(first.getCharPositionInLine());
    result.setLine(first.getLine());
    result.setStartIndex(((TokenWithIndexAndSubTokens) first).getStopIndex());
    if (t != null) {
        result.setStopIndex(((TokenWithIndexAndSubTokens) t).getStopIndex());
    }
    tokens.add(result);
    return true;
}

Example 29 with EOF

use of org.antlr.v4.runtime.Recognizer.EOF in project antlr4 by tunnelvisionlabs.

the class TestATNInterpreter method testMustTrackPreviousGoodAlt2WithEOF.

@Test(expected = NoViableAltException.class)
public void testMustTrackPreviousGoodAlt2WithEOF() throws Exception {
    LexerGrammar lg = new LexerGrammar("lexer grammar L;\n" + "A : 'a' ;\n" + "B : 'b' ;\n" + "C : 'c' ;\n" + "D : 'd' ;\n");
    Grammar g = new Grammar("parser grammar T;\n" + "a : (A | A B | A B C) EOF;");
    checkMatchedAlt(lg, g, "a", 1);
    checkMatchedAlt(lg, g, "ab", 2);
    checkMatchedAlt(lg, g, "abc", 3);
    try {
        checkMatchedAlt(lg, g, "abd", 1);
    } catch (NoViableAltException re) {
        assertEquals(2, re.getOffendingToken().getTokenIndex());
        assertEquals(4, re.getOffendingToken().getType());
        throw re;
    }
}

Example 30 with EOF

use of org.antlr.v4.runtime.Recognizer.EOF in project antlr4 by tunnelvisionlabs.

the class TestATNLexerInterpreter method testRecursiveLexerRuleRefWithWildcard.

@Test
public void testRecursiveLexerRuleRefWithWildcard() throws Exception {
    LexerGrammar lg = new LexerGrammar("lexer grammar L;\n" + "CMT : '/*' (CMT | .)*? '*/' ;\n" + "WS : (' '|'\\n')+ ;");
    String expecting = "CMT, WS, CMT, WS, EOF";
    checkLexerMatches(lg, "/* ick */\n" + "/* /* */\n" + "/* /*nested*/ */\n", expecting);
}