Examples with CommonTokenStream org.antlr.runtime.CommonTokenStream used on opensource projects

Example 11 with CommonTokenStream

use of org.antlr.runtime.CommonTokenStream in project atlas by alibaba.

the class SmaliMod method assembleSmaliFile.

public static boolean assembleSmaliFile(File smaliFile, DexBuilder dexBuilder, boolean verboseErrors, boolean printTokens) throws IOException, RecognitionException {
    CommonTokenStream tokens;
    LexerErrorInterface lexer;
    InputStream is = new FileInputStream(smaliFile);
    InputStreamReader reader = new InputStreamReader(is, "UTF-8");
    lexer = new smaliFlexLexer(reader);
    ((smaliFlexLexer) lexer).setSourceFile(smaliFile);
    tokens = new CommonTokenStream((TokenSource) lexer);
    if (printTokens) {
        tokens.getTokens();
        for (int i = 0; i < tokens.size(); i++) {
            Token token = tokens.get(i);
            if (token.getChannel() == smaliParser.HIDDEN) {
                continue;
            }
            System.out.println(smaliParser.tokenNames[token.getType()] + ": " + token.getText());
        }
    }
    smaliParser parser = new smaliParser(tokens);
    parser.setVerboseErrors(verboseErrors);
    smaliParser.smali_file_return result = parser.smali_file();
    if (parser.getNumberOfSyntaxErrors() > 0 || lexer.getNumberOfSyntaxErrors() > 0) {
        return false;
    }
    CommonTree t = (CommonTree) result.getTree();
    CommonTreeNodeStream treeStream = new CommonTreeNodeStream(t);
    treeStream.setTokenStream(tokens);
    smaliTreeWalker dexGen = new smaliTreeWalker(treeStream);
    dexGen.setVerboseErrors(verboseErrors);
    dexGen.setDexBuilder(dexBuilder);
    dexGen.smali_file();
    return dexGen.getNumberOfSyntaxErrors() == 0;
}

Example 12 with CommonTokenStream

use of org.antlr.runtime.CommonTokenStream in project smali by JesusFreke.

the class Smali method assembleSmaliFile.

private static boolean assembleSmaliFile(File smaliFile, DexBuilder dexBuilder, SmaliOptions options) throws Exception {
    FileInputStream fis = null;
    try {
        fis = new FileInputStream(smaliFile);
        InputStreamReader reader = new InputStreamReader(fis, "UTF-8");
        LexerErrorInterface lexer = new smaliFlexLexer(reader);
        ((smaliFlexLexer) lexer).setSourceFile(smaliFile);
        CommonTokenStream tokens = new CommonTokenStream((TokenSource) lexer);
        if (options.printTokens) {
            tokens.getTokens();
            for (int i = 0; i < tokens.size(); i++) {
                Token token = tokens.get(i);
                if (token.getChannel() == smaliParser.HIDDEN) {
                    continue;
                }
                System.out.println(smaliParser.tokenNames[token.getType()] + ": " + token.getText());
            }
            System.out.flush();
        }
        smaliParser parser = new smaliParser(tokens);
        parser.setVerboseErrors(options.verboseErrors);
        parser.setAllowOdex(options.allowOdexOpcodes);
        parser.setApiLevel(options.apiLevel);
        smaliParser.smali_file_return result = parser.smali_file();
        if (parser.getNumberOfSyntaxErrors() > 0 || lexer.getNumberOfSyntaxErrors() > 0) {
            return false;
        }
        CommonTree t = result.getTree();
        CommonTreeNodeStream treeStream = new CommonTreeNodeStream(t);
        treeStream.setTokenStream(tokens);
        if (options.printTokens) {
            System.out.println(t.toStringTree());
        }
        smaliTreeWalker dexGen = new smaliTreeWalker(treeStream);
        dexGen.setApiLevel(options.apiLevel);
        dexGen.setVerboseErrors(options.verboseErrors);
        dexGen.setDexBuilder(dexBuilder);
        dexGen.smali_file();
        return dexGen.getNumberOfSyntaxErrors() == 0;
    } finally {
        if (fis != null) {
            fis.close();
        }
    }
}

Example 13 with CommonTokenStream

use of org.antlr.runtime.CommonTokenStream in project smali by JesusFreke.

the class LexerTest method runTest.

public void runTest(String test, boolean discardHiddenTokens) {
    String smaliFile = String.format("LexerTest%s%s.smali", File.separatorChar, test);
    String tokensFile = String.format("LexerTest%s%s.tokens", File.separatorChar, test);
    org.jf.smali.expectedTokensTestGrammarLexer expectedTokensLexer = null;
    try {
        expectedTokensLexer = new org.jf.smali.expectedTokensTestGrammarLexer(new ANTLRInputStream(LexerTest.class.getClassLoader().getResourceAsStream(tokensFile)));
    } catch (IOException ex) {
        throw new RuntimeException(ex);
    }
    CommonTokenStream expectedTokensStream = new CommonTokenStream(expectedTokensLexer);
    org.jf.smali.expectedTokensTestGrammarParser expectedTokensParser = new org.jf.smali.expectedTokensTestGrammarParser(expectedTokensStream);
    try {
        expectedTokensParser.top();
    } catch (RecognitionException ex) {
        throw new RuntimeException(ex);
    }
    List<ExpectedToken> expectedTokens = expectedTokensParser.getExpectedTokens();
    InputStream smaliStream = LexerTest.class.getClassLoader().getResourceAsStream(smaliFile);
    if (smaliStream == null) {
        Assert.fail("Could not load " + smaliFile);
    }
    smaliFlexLexer lexer = new smaliFlexLexer(new InputStreamReader(smaliStream));
    lexer.setSourceFile(new File(test + ".smali"));
    lexer.setSuppressErrors(true);
    CommonTokenStream tokenStream = new CommonTokenStream(lexer);
    tokenStream.fill();
    List tokens = tokenStream.getTokens();
    int expectedTokenIndex = 0;
    CommonToken token;
    for (int i = 0; i < tokens.size() - 1; i++) {
        token = (CommonToken) tokens.get(i);
        if (discardHiddenTokens && token.getChannel() == smaliParser.HIDDEN) {
            continue;
        }
        if (expectedTokenIndex >= expectedTokens.size()) {
            Assert.fail("Too many tokens");
        }
        if (token.getType() == smaliParser.INVALID_TOKEN) {
            Assert.assertTrue("Encountered an INVALID_TOKEN not on the error channel", token.getChannel() == smaliParser.ERROR_CHANNEL);
        }
        ExpectedToken expectedToken = expectedTokens.get(expectedTokenIndex++);
        if (!tokenTypesByName.containsKey(expectedToken.tokenName)) {
            Assert.fail("Unknown token: " + expectedToken.tokenName);
        }
        int expectedTokenType = tokenTypesByName.get(expectedToken.tokenName);
        if (token.getType() != expectedTokenType) {
            Assert.fail(String.format("Invalid token at index %d. Expecting %s, got %s(%s)", expectedTokenIndex - 1, expectedToken.tokenName, getTokenName(token.getType()), token.getText()));
        }
        if (expectedToken.tokenText != null) {
            if (!expectedToken.tokenText.equals(token.getText())) {
                Assert.fail(String.format("Invalid token text at index %d. Expecting text \"%s\", got \"%s\"", expectedTokenIndex - 1, expectedToken.tokenText, token.getText()));
            }
        }
    }
    if (expectedTokenIndex < expectedTokens.size()) {
        Assert.fail(String.format("Not enough tokens. Expecting %d tokens, but got %d", expectedTokens.size(), expectedTokenIndex));
    }
}

Example 14 with CommonTokenStream

use of org.antlr.runtime.CommonTokenStream in project eiger by wlloyd.

the class CliCompiler method compileQuery.

public static Tree compileQuery(String query) {
    Tree queryTree;
    try {
        ANTLRStringStream input = new ANTLRNoCaseStringStream(query);
        CliLexer lexer = new CliLexer(input);
        CommonTokenStream tokens = new CommonTokenStream(lexer);
        CliParser parser = new CliParser(tokens);
        // start parsing...
        queryTree = (Tree) (parser.root().getTree());
    // semantic analysis if any...
    //  [tbd]
    } catch (Exception e) {
        // if there was an exception we don't want to process request any further
        throw new RuntimeException(e.getMessage(), e);
    }
    return queryTree;
}

Example 15 with CommonTokenStream

use of org.antlr.runtime.CommonTokenStream in project ceylon-compiler by ceylon.

the class LanguageCompiler method ceylonParse.

private JCCompilationUnit ceylonParse(JavaFileObject filename, CharSequence readSource) {
    if (ceylonEnter.hasRun())
        throw new RunTwiceException("Trying to load new source file after CeylonEnter has been called: " + filename);
    try {
        ModuleManager moduleManager = phasedUnits.getModuleManager();
        ModuleSourceMapper moduleSourceMapper = phasedUnits.getModuleSourceMapper();
        File sourceFile = new File(filename.getName());
        // FIXME: temporary solution
        VirtualFile file = vfs.getFromFile(sourceFile);
        VirtualFile srcDir = vfs.getFromFile(getSrcDir(sourceFile));
        String source = readSource.toString();
        char[] chars = source.toCharArray();
        LineMap map = Position.makeLineMap(chars, chars.length, false);
        PhasedUnit phasedUnit = null;
        PhasedUnit externalPhasedUnit = compilerDelegate.getExternalSourcePhasedUnit(srcDir, file);
        String suppressWarnings = options.get(OptionName.CEYLONSUPPRESSWARNINGS);
        final EnumSet<Warning> suppressedWarnings;
        if (suppressWarnings != null) {
            if (suppressWarnings.trim().isEmpty()) {
                suppressedWarnings = EnumSet.allOf(Warning.class);
            } else {
                suppressedWarnings = EnumSet.noneOf(Warning.class);
                for (String name : suppressWarnings.trim().split(" *, *")) {
                    suppressedWarnings.add(Warning.valueOf(name));
                }
            }
        } else {
            suppressedWarnings = EnumSet.noneOf(Warning.class);
        }
        if (externalPhasedUnit != null) {
            phasedUnit = new CeylonPhasedUnit(externalPhasedUnit, filename, map);
            phasedUnit.setSuppressedWarnings(suppressedWarnings);
            phasedUnits.addPhasedUnit(externalPhasedUnit.getUnitFile(), phasedUnit);
            gen.setMap(map);
            String pkgName = phasedUnit.getPackage().getQualifiedNameString();
            if ("".equals(pkgName)) {
                pkgName = null;
            }
            return gen.makeJCCompilationUnitPlaceholder(phasedUnit.getCompilationUnit(), filename, pkgName, phasedUnit);
        }
        if (phasedUnit == null) {
            ANTLRStringStream input = new NewlineFixingStringStream(source);
            CeylonLexer lexer = new CeylonLexer(input);
            CommonTokenStream tokens = new CommonTokenStream(lexer);
            CeylonParser parser = new CeylonParser(tokens);
            CompilationUnit cu = parser.compilationUnit();
            java.util.List<LexError> lexerErrors = lexer.getErrors();
            for (LexError le : lexerErrors) {
                printError(le, le.getMessage(), "ceylon.lexer", map);
            }
            java.util.List<ParseError> parserErrors = parser.getErrors();
            for (ParseError pe : parserErrors) {
                printError(pe, pe.getMessage(), "ceylon.parser", map);
            }
            // if we continue and it's not a descriptor, we don't care about errors
            if ((options.get(OptionName.CEYLONCONTINUE) != null && !ModuleManager.MODULE_FILE.equals(sourceFile.getName()) && !ModuleManager.PACKAGE_FILE.equals(sourceFile.getName())) || // otherwise we care about errors
            (lexerErrors.size() == 0 && parserErrors.size() == 0)) {
                // FIXME: this is bad in many ways
                String pkgName = getPackage(filename);
                // make a Package with no module yet, we will resolve them later
                /*
                     * Stef: see javadoc for findOrCreateModulelessPackage() for why this is here.
                     */
                com.redhat.ceylon.model.typechecker.model.Package p = modelLoader.findOrCreateModulelessPackage(pkgName == null ? "" : pkgName);
                phasedUnit = new CeylonPhasedUnit(file, srcDir, cu, p, moduleManager, moduleSourceMapper, ceylonContext, filename, map);
                phasedUnit.setSuppressedWarnings(suppressedWarnings);
                phasedUnits.addPhasedUnit(file, phasedUnit);
                gen.setMap(map);
                return gen.makeJCCompilationUnitPlaceholder(cu, filename, pkgName, phasedUnit);
            }
        }
    } catch (RuntimeException e) {
        throw e;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
    JCCompilationUnit result = make.TopLevel(List.<JCAnnotation>nil(), null, List.<JCTree>of(make.Erroneous()));
    result.sourcefile = filename;
    return result;
}