Examples with ANTLRInputStream org.antlr.runtime.ANTLRInputStream used on opensource projects

Example 1 with ANTLRInputStream

use of org.antlr.runtime.ANTLRInputStream in project smali by JesusFreke.

the class LexerTest method runTest.

public void runTest(String test, boolean discardHiddenTokens) {
    String smaliFile = String.format("LexerTest%s%s.smali", File.separatorChar, test);
    String tokensFile = String.format("LexerTest%s%s.tokens", File.separatorChar, test);
    org.jf.smali.expectedTokensTestGrammarLexer expectedTokensLexer = null;
    try {
        expectedTokensLexer = new org.jf.smali.expectedTokensTestGrammarLexer(new ANTLRInputStream(LexerTest.class.getClassLoader().getResourceAsStream(tokensFile)));
    } catch (IOException ex) {
        throw new RuntimeException(ex);
    }
    CommonTokenStream expectedTokensStream = new CommonTokenStream(expectedTokensLexer);
    org.jf.smali.expectedTokensTestGrammarParser expectedTokensParser = new org.jf.smali.expectedTokensTestGrammarParser(expectedTokensStream);
    try {
        expectedTokensParser.top();
    } catch (RecognitionException ex) {
        throw new RuntimeException(ex);
    }
    List<ExpectedToken> expectedTokens = expectedTokensParser.getExpectedTokens();
    InputStream smaliStream = LexerTest.class.getClassLoader().getResourceAsStream(smaliFile);
    if (smaliStream == null) {
        Assert.fail("Could not load " + smaliFile);
    }
    smaliFlexLexer lexer = new smaliFlexLexer(new InputStreamReader(smaliStream));
    lexer.setSourceFile(new File(test + ".smali"));
    lexer.setSuppressErrors(true);
    CommonTokenStream tokenStream = new CommonTokenStream(lexer);
    tokenStream.fill();
    List tokens = tokenStream.getTokens();
    int expectedTokenIndex = 0;
    CommonToken token;
    for (int i = 0; i < tokens.size() - 1; i++) {
        token = (CommonToken) tokens.get(i);
        if (discardHiddenTokens && token.getChannel() == smaliParser.HIDDEN) {
            continue;
        }
        if (expectedTokenIndex >= expectedTokens.size()) {
            Assert.fail("Too many tokens");
        }
        if (token.getType() == smaliParser.INVALID_TOKEN) {
            Assert.assertTrue("Encountered an INVALID_TOKEN not on the error channel", token.getChannel() == smaliParser.ERROR_CHANNEL);
        }
        ExpectedToken expectedToken = expectedTokens.get(expectedTokenIndex++);
        if (!tokenTypesByName.containsKey(expectedToken.tokenName)) {
            Assert.fail("Unknown token: " + expectedToken.tokenName);
        }
        int expectedTokenType = tokenTypesByName.get(expectedToken.tokenName);
        if (token.getType() != expectedTokenType) {
            Assert.fail(String.format("Invalid token at index %d. Expecting %s, got %s(%s)", expectedTokenIndex - 1, expectedToken.tokenName, getTokenName(token.getType()), token.getText()));
        }
        if (expectedToken.tokenText != null) {
            if (!expectedToken.tokenText.equals(token.getText())) {
                Assert.fail(String.format("Invalid token text at index %d. Expecting text \"%s\", got \"%s\"", expectedTokenIndex - 1, expectedToken.tokenText, token.getText()));
            }
        }
    }
    if (expectedTokenIndex < expectedTokens.size()) {
        Assert.fail(String.format("Not enough tokens. Expecting %d tokens, but got %d", expectedTokens.size(), expectedTokenIndex));
    }
}

Example 2 with ANTLRInputStream

use of org.antlr.runtime.ANTLRInputStream in project ceylon by eclipse.

the class PhasedUnits method parseFile.

protected void parseFile(VirtualFile file, VirtualFile srcDir) throws Exception {
    if (file.getName().endsWith(".ceylon") && (sourceFiles.isEmpty() || sourceFiles.contains(file))) {
        // System.out.println("Parsing " + file.getName());
        CeylonLexer lexer = new CeylonLexer(new ANTLRInputStream(file.getInputStream(), getEncoding()));
        CommonTokenStream tokenStream = new CommonTokenStream(new CeylonInterpolatingLexer(lexer));
        CeylonParser parser = new CeylonParser(tokenStream);
        Tree.CompilationUnit cu = parser.compilationUnit();
        PhasedUnit phasedUnit = new PhasedUnit(file, srcDir, cu, moduleSourceMapper.getCurrentPackage(), moduleManager, moduleSourceMapper, context, new ArrayList<Token>(tokenStream.getTokens()));
        addPhasedUnit(file, phasedUnit);
        List<LexError> lexerErrors = lexer.getErrors();
        for (LexError le : lexerErrors) {
            // System.out.println("Lexer error in " + file.getName() + ": " + le.getMessage());
            cu.addLexError(le);
        }
        lexerErrors.clear();
        List<ParseError> parserErrors = parser.getErrors();
        for (ParseError pe : parserErrors) {
            // System.out.println("Parser error in " + file.getName() + ": " + pe.getMessage());
            cu.addParseError(pe);
        }
        parserErrors.clear();
    }
}

Example 3 with ANTLRInputStream

use of org.antlr.runtime.ANTLRInputStream in project ceylon by eclipse.

the class Generate method tree.

private static void tree(File file) throws Exception {
    InputStream is = new FileInputStream(file);
    ANTLRInputStream input = new ANTLRInputStream(is);
    TreegenLexer lexer = new TreegenLexer(input);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    TreegenParser parser = new TreegenParser(tokens);
    File out = new File(GENERATED_PACKAGE_DIR + "Tree.java");
    out.createNewFile();
    Util.out = new PrintStream(out);
    parser.nodeList();
}

Example 4 with ANTLRInputStream

use of org.antlr.runtime.ANTLRInputStream in project ceylon by eclipse.

the class Generate method visitorAdaptor.

private static void visitorAdaptor(File file) throws Exception {
    InputStream is = new FileInputStream(file);
    ANTLRInputStream input = new ANTLRInputStream(is);
    VisitorAdaptorgenLexer lexer = new VisitorAdaptorgenLexer(input);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    VisitorAdaptorgenParser parser = new VisitorAdaptorgenParser(tokens);
    File out = new File(GENERATED_PACKAGE_DIR + "VisitorAdaptor.java");
    out.createNewFile();
    Util.out = new PrintStream(out);
    parser.nodeList();
}

Example 5 with ANTLRInputStream

use of org.antlr.runtime.ANTLRInputStream in project ceylon by eclipse.

the class Generate method walker.

/*private static void builder(File file) throws Exception {
        InputStream is = new FileInputStream( file );
        ANTLRInputStream input = new ANTLRInputStream(is);
        BuildergenLexer lexer = new BuildergenLexer(input);
        CommonTokenStream tokens = new CommonTokenStream(lexer);
        BuildergenParser parser = new BuildergenParser(tokens);
        File out = new File( GENERATED_PACKAGE_DIR + "Builder.java" );
        out.createNewFile();
        AnalyzerUtil.out=new PrintStream(out);
        parser.nodeList();
    }*/
private static void walker(File file) throws Exception {
    InputStream is = new FileInputStream(file);
    ANTLRInputStream input = new ANTLRInputStream(is);
    WalkergenLexer lexer = new WalkergenLexer(input);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    WalkergenParser parser = new WalkergenParser(tokens);
    File out = new File(GENERATED_PACKAGE_DIR + "Walker.java");
    out.createNewFile();
    Util.out = new PrintStream(out);
    parser.nodeList();
}