Examples with Token org.antlr.v4.runtime.Token used on opensource projects

Example 56 with Token

use of org.antlr.v4.runtime.Token in project antlr4 by antlr.

the class TestTokenPositionOptions method testLeftRecursionWithSet.

@Test
public void testLeftRecursionWithSet() throws Exception {
    Grammar g = new Grammar("grammar T;\n" + "s : e ';' ;\n" + "e : e op=('*'|'/') e\n" + "  | e '+' e\n" + "  | e '.' ID\n" + "  | '-' e\n" + "  | ID\n" + "  ;\n" + "ID : [a-z]+ ;\n");
    String expectedTree = "(COMBINED_GRAMMAR T (RULES (RULE s (BLOCK (ALT e ';'))) (RULE e (BLOCK (ALT (BLOCK (ALT {} ('-' (ELEMENT_OPTIONS (= tokenIndex 49))) (e (ELEMENT_OPTIONS (= tokenIndex 51) (= p 2)))) (ALT (ID (ELEMENT_OPTIONS (= tokenIndex 55))))) (* (BLOCK (ALT ({precpred(_ctx, 5)}? (ELEMENT_OPTIONS (= p 5))) (= op (SET ('*' (ELEMENT_OPTIONS (= tokenIndex 24))) ('/' (ELEMENT_OPTIONS (= tokenIndex 26))))) (e (ELEMENT_OPTIONS (= tokenIndex 29) (= p 6)))) (ALT ({precpred(_ctx, 4)}? (ELEMENT_OPTIONS (= p 4))) ('+' (ELEMENT_OPTIONS (= tokenIndex 35))) (e (ELEMENT_OPTIONS (= tokenIndex 37) (= p 5)))) (ALT ({precpred(_ctx, 3)}? (ELEMENT_OPTIONS (= p 3))) ('.' (ELEMENT_OPTIONS (= tokenIndex 43))) (ID (ELEMENT_OPTIONS (= tokenIndex 45)))))))))))";
    assertEquals(expectedTree, g.ast.toStringTree());
    String expectedElementTokens = "[@5,11:11='s',<57>,2:0]\n" + "[@9,15:15='e',<57>,2:4]\n" + "[@11,17:19='';'',<62>,2:6]\n" + "[@15,23:23='e',<57>,3:0]\n" + "[@49,73:75=''-'',<62>,6:4]\n" + "[@51,77:77='e',<57>,6:8]\n" + "[@55,83:84='ID',<66>,7:4]\n" + "[@24,33:35=''*'',<62>,3:10]\n" + "[@26,37:39=''/'',<62>,3:14]\n" + "[@29,42:42='e',<57>,3:19]\n" + "[@35,50:52=''+'',<62>,4:6]\n" + "[@37,54:54='e',<57>,4:10]\n" + "[@43,62:64=''.'',<62>,5:6]\n" + "[@45,66:67='ID',<66>,5:10]";
    IntervalSet types = new IntervalSet(ANTLRParser.TOKEN_REF, ANTLRParser.STRING_LITERAL, ANTLRParser.RULE_REF);
    List<GrammarAST> nodes = g.ast.getNodesWithTypePreorderDFS(types);
    List<Token> tokens = new ArrayList<Token>();
    for (GrammarAST node : nodes) {
        tokens.add(node.getToken());
    }
    assertEquals(expectedElementTokens, Utils.join(tokens.toArray(), "\n"));
}

Example 57 with Token

use of org.antlr.v4.runtime.Token in project antlr4 by antlr.

the class TestTokenTypeAssignment method testPredDoesNotHideNameToLiteralMapInLexer.

@Test
public void testPredDoesNotHideNameToLiteralMapInLexer() throws Exception {
    // 'x' is token and char in lexer rule
    Grammar g = new Grammar("grammar t;\n" + "a : 'x' X ; \n" + // must match as alias even with pred
    "X: 'x' {true}?;\n");
    assertEquals("{'x'=1}", g.stringLiteralToTypeMap.toString());
    assertEquals("{EOF=-1, X=1}", g.tokenNameToTypeMap.toString());
    // pushed in lexer from parser
    assertEquals("{'x'=1}", g.implicitLexer.stringLiteralToTypeMap.toString());
    assertEquals("{EOF=-1, X=1}", g.implicitLexer.tokenNameToTypeMap.toString());
}

Example 58 with Token

use of org.antlr.v4.runtime.Token in project elasticsearch by elastic.

the class EnhancedPainlessLexer method nextToken.

@Override
public Token nextToken() {
    if (stashedNext != null) {
        previous = stashedNext;
        stashedNext = null;
        return previous;
    }
    Token next = super.nextToken();
    if (insertSemicolon(previous, next)) {
        stashedNext = next;
        previous = _factory.create(new Pair<TokenSource, CharStream>(this, _input), PainlessLexer.SEMICOLON, ";", Lexer.DEFAULT_TOKEN_CHANNEL, next.getStartIndex(), next.getStopIndex(), next.getLine(), next.getCharPositionInLine());
        return previous;
    } else {
        previous = next;
        return next;
    }
}

Example 59 with Token

use of org.antlr.v4.runtime.Token in project aic-praise by aic-sri-international.

the class HOGMCodeArea method computeHighlighting.

private static StyleSpans<Collection<String>> computeHighlighting(String text) {
    StyleSpansBuilder<Collection<String>> spansBuilder = new StyleSpansBuilder<>();
    int lastTokenEnd = 0;
    ANTLRInputStream input = new ANTLRInputStream(text);
    HOGMLexer lexer = new HOGMLexer(input);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    tokens.fill();
    for (int i = 0; i < tokens.size(); i++) {
        Token t = tokens.get(i);
        if (t.getType() == Token.EOF) {
            break;
        }
        String styleClass;
        if (t.getType() == HOGMLexer.COMMENT || t.getType() == HOGMLexer.LINE_COMMENT) {
            styleClass = "hogmCodeComment";
        } else if (HOGMTerminalSymbols.isTerminalSymbol(t.getText())) {
            styleClass = "hogmCodeKeyword";
        } else {
            styleClass = "hogmCodeOther";
        }
        int spacing = t.getStartIndex() - lastTokenEnd;
        if (spacing > 0) {
            spansBuilder.add(Collections.emptyList(), spacing);
        }
        int stylesize = (t.getStopIndex() - t.getStartIndex()) + 1;
        spansBuilder.add(Collections.singleton(styleClass), stylesize);
        lastTokenEnd = t.getStopIndex() + 1;
    }
    return spansBuilder.create();
}

Example 60 with Token

use of org.antlr.v4.runtime.Token in project lucene-solr by apache.

the class JavascriptParserErrorStrategy method recoverInline.

/**
   * Ensures the ANTLR parser will throw an exception after the first error
   *
   * @param recognizer the parser being used
   * @return no actual return value
   * @throws RecognitionException not used as a ParseException wrapped in a RuntimeException is thrown instead
   */
@Override
public Token recoverInline(Parser recognizer) throws RecognitionException {
    Token token = recognizer.getCurrentToken();
    String message = "unexpected token " + getTokenErrorDisplay(token) + " on line (" + token.getLine() + ") position (" + token.getCharPositionInLine() + ")" + " was expecting one of " + recognizer.getExpectedTokens().toString(recognizer.getVocabulary());
    ParseException parseException = new ParseException(message, token.getStartIndex());
    throw new RuntimeException(parseException);
}