Examples with CommonToken org.antlr.runtime.CommonToken used on opensource projects

Example 11 with CommonToken

use of org.antlr.runtime.CommonToken in project smali by JesusFreke.

the class LexerTest method runTest.

public void runTest(String test, boolean discardHiddenTokens) {
    String smaliFile = String.format("LexerTest%s%s.smali", File.separatorChar, test);
    String tokensFile = String.format("LexerTest%s%s.tokens", File.separatorChar, test);
    org.jf.smali.expectedTokensTestGrammarLexer expectedTokensLexer = null;
    try {
        expectedTokensLexer = new org.jf.smali.expectedTokensTestGrammarLexer(new ANTLRInputStream(LexerTest.class.getClassLoader().getResourceAsStream(tokensFile)));
    } catch (IOException ex) {
        throw new RuntimeException(ex);
    }
    CommonTokenStream expectedTokensStream = new CommonTokenStream(expectedTokensLexer);
    org.jf.smali.expectedTokensTestGrammarParser expectedTokensParser = new org.jf.smali.expectedTokensTestGrammarParser(expectedTokensStream);
    try {
        expectedTokensParser.top();
    } catch (RecognitionException ex) {
        throw new RuntimeException(ex);
    }
    List<ExpectedToken> expectedTokens = expectedTokensParser.getExpectedTokens();
    InputStream smaliStream = LexerTest.class.getClassLoader().getResourceAsStream(smaliFile);
    if (smaliStream == null) {
        Assert.fail("Could not load " + smaliFile);
    }
    smaliFlexLexer lexer = new smaliFlexLexer(new InputStreamReader(smaliStream));
    lexer.setSourceFile(new File(test + ".smali"));
    lexer.setSuppressErrors(true);
    CommonTokenStream tokenStream = new CommonTokenStream(lexer);
    tokenStream.fill();
    List tokens = tokenStream.getTokens();
    int expectedTokenIndex = 0;
    CommonToken token;
    for (int i = 0; i < tokens.size() - 1; i++) {
        token = (CommonToken) tokens.get(i);
        if (discardHiddenTokens && token.getChannel() == smaliParser.HIDDEN) {
            continue;
        }
        if (expectedTokenIndex >= expectedTokens.size()) {
            Assert.fail("Too many tokens");
        }
        if (token.getType() == smaliParser.INVALID_TOKEN) {
            Assert.assertTrue("Encountered an INVALID_TOKEN not on the error channel", token.getChannel() == smaliParser.ERROR_CHANNEL);
        }
        ExpectedToken expectedToken = expectedTokens.get(expectedTokenIndex++);
        if (!tokenTypesByName.containsKey(expectedToken.tokenName)) {
            Assert.fail("Unknown token: " + expectedToken.tokenName);
        }
        int expectedTokenType = tokenTypesByName.get(expectedToken.tokenName);
        if (token.getType() != expectedTokenType) {
            Assert.fail(String.format("Invalid token at index %d. Expecting %s, got %s(%s)", expectedTokenIndex - 1, expectedToken.tokenName, getTokenName(token.getType()), token.getText()));
        }
        if (expectedToken.tokenText != null) {
            if (!expectedToken.tokenText.equals(token.getText())) {
                Assert.fail(String.format("Invalid token text at index %d. Expecting text \"%s\", got \"%s\"", expectedTokenIndex - 1, expectedToken.tokenText, token.getText()));
            }
        }
    }
    if (expectedTokenIndex < expectedTokens.size()) {
        Assert.fail(String.format("Not enough tokens. Expecting %d tokens, but got %d", expectedTokens.size(), expectedTokenIndex));
    }
}

Example 12 with CommonToken

use of org.antlr.runtime.CommonToken in project smali by JesusFreke.

the class PsiBuilderTokenStream method buildCurrentToken.

private void buildCurrentToken() {
    IElementType element = psiBuilder.getTokenType();
    if (element != null) {
        if (element instanceof SmaliLexicalElementType) {
            SmaliLexicalElementType elementType = (SmaliLexicalElementType) element;
            currentToken = new CommonToken(elementType.tokenId, psiBuilder.getTokenText());
        } else if (element == TokenType.BAD_CHARACTER) {
            currentToken = new InvalidToken("", psiBuilder.getTokenText());
        } else {
            throw new UnsupportedOperationException();
        }
    } else {
        currentToken = new CommonToken(Token.EOF);
    }
}

Example 13 with CommonToken

use of org.antlr.runtime.CommonToken in project ceylon-compiler by ceylon.

the class ClassOrPackageDoc method getSourceCode.

private String getSourceCode(PhasedUnit pu, Node node) throws IOException {
    int startIndex = ((CommonToken) node.getToken()).getStartIndex();
    int stopIndex = ((CommonToken) node.getEndToken()).getStopIndex();
    StringBuilder sourceCodeBuilder = new StringBuilder();
    BufferedReader sourceCodeReader = new BufferedReader(new InputStreamReader(pu.getUnitFile().getInputStream()));
    try {
        while (true) {
            int c = sourceCodeReader.read();
            if (c == -1 || sourceCodeBuilder.length() > stopIndex) {
                break;
            }
            sourceCodeBuilder.append((char) c);
        }
    } finally {
        sourceCodeReader.close();
    }
    String sourceCode = sourceCodeBuilder.substring(startIndex, stopIndex + 1);
    sourceCode = sourceCode.replaceAll("&", "&");
    sourceCode = sourceCode.replaceAll("<", "&lt;");
    sourceCode = sourceCode.replaceAll(">", "&gt;");
    return sourceCode;
}

Example 14 with CommonToken

use of org.antlr.runtime.CommonToken in project hive by apache.

the class SemanticAnalyzer method genSelectDIAST.

public static ASTNode genSelectDIAST(RowResolver rr) {
    LinkedHashMap<String, LinkedHashMap<String, ColumnInfo>> map = rr.getRslvMap();
    ASTNode selectDI = new ASTNode(new CommonToken(HiveParser.TOK_SELECTDI, "TOK_SELECTDI"));
    //       table, is deterministic, but undefined - RR stores them in the order of addition.
    for (String tabAlias : map.keySet()) {
        for (Entry<String, ColumnInfo> entry : map.get(tabAlias).entrySet()) {
            selectDI.addChild(buildSelExprSubTree(tabAlias, entry.getKey()));
        }
    }
    return selectDI;
}

Example 15 with CommonToken

use of org.antlr.runtime.CommonToken in project hive by apache.

the class SemanticAnalyzer method buildSelExprSubTree.

private static ASTNode buildSelExprSubTree(String tableAlias, String col) {
    ASTNode selexpr = new ASTNode(new CommonToken(HiveParser.TOK_SELEXPR, "TOK_SELEXPR"));
    ASTNode tableOrCol = new ASTNode(new CommonToken(HiveParser.TOK_TABLE_OR_COL, "TOK_TABLE_OR_COL"));
    ASTNode dot = new ASTNode(new CommonToken(HiveParser.DOT, "."));
    tableOrCol.addChild(new ASTNode(new CommonToken(HiveParser.Identifier, tableAlias)));
    dot.addChild(tableOrCol);
    dot.addChild(new ASTNode(new CommonToken(HiveParser.Identifier, col)));
    selexpr.addChild(dot);
    return selexpr;
}