Search in sources :

Example 1 with CommonTokenStream

use of org.antlr.runtime.CommonTokenStream in project freud by LMAX-Exchange.

the class JavaSourceJdomParser method parseJavaSourceToDocument.

private static Document parseJavaSourceToDocument(final Reader javaSourceReader) throws RecognitionException, IOException {
    JavaParser parser = new JavaParser(new CommonTokenStream(new JavaLexer(new ANTLRReaderStream(javaSourceReader))));
    final JdomTreeAdaptor treeAdaptor = new JdomTreeAdaptor(JAVA_SOURCE_ROOT_ELEMENT_NAME, JAVA_SOURCE_TOKEN_TYPES);
    parser.setTreeAdaptor(treeAdaptor);
    parser.compilationUnit();
    return treeAdaptor.getDocument();
}
Also used : JavaParser(org.freud.analysed.javasource.parser.JavaParser) CommonTokenStream(org.antlr.runtime.CommonTokenStream) JdomTreeAdaptor(org.freud.core.parser.JdomTreeAdaptor) JavaLexer(org.freud.analysed.javasource.parser.JavaLexer) ANTLRReaderStream(org.antlr.runtime.ANTLRReaderStream)

Example 2 with CommonTokenStream

use of org.antlr.runtime.CommonTokenStream in project smali by JesusFreke.

the class LexerTest method runTest.

public void runTest(String test, boolean discardHiddenTokens) {
    String smaliFile = String.format("LexerTest%s%s.smali", File.separatorChar, test);
    String tokensFile = String.format("LexerTest%s%s.tokens", File.separatorChar, test);
    org.jf.smali.expectedTokensTestGrammarLexer expectedTokensLexer = null;
    try {
        expectedTokensLexer = new org.jf.smali.expectedTokensTestGrammarLexer(new ANTLRInputStream(LexerTest.class.getClassLoader().getResourceAsStream(tokensFile)));
    } catch (IOException ex) {
        throw new RuntimeException(ex);
    }
    CommonTokenStream expectedTokensStream = new CommonTokenStream(expectedTokensLexer);
    org.jf.smali.expectedTokensTestGrammarParser expectedTokensParser = new org.jf.smali.expectedTokensTestGrammarParser(expectedTokensStream);
    try {
        expectedTokensParser.top();
    } catch (RecognitionException ex) {
        throw new RuntimeException(ex);
    }
    List<ExpectedToken> expectedTokens = expectedTokensParser.getExpectedTokens();
    InputStream smaliStream = LexerTest.class.getClassLoader().getResourceAsStream(smaliFile);
    if (smaliStream == null) {
        Assert.fail("Could not load " + smaliFile);
    }
    smaliFlexLexer lexer = new smaliFlexLexer(new InputStreamReader(smaliStream));
    lexer.setSourceFile(new File(test + ".smali"));
    lexer.setSuppressErrors(true);
    CommonTokenStream tokenStream = new CommonTokenStream(lexer);
    tokenStream.fill();
    List tokens = tokenStream.getTokens();
    int expectedTokenIndex = 0;
    CommonToken token;
    for (int i = 0; i < tokens.size() - 1; i++) {
        token = (CommonToken) tokens.get(i);
        if (discardHiddenTokens && token.getChannel() == smaliParser.HIDDEN) {
            continue;
        }
        if (expectedTokenIndex >= expectedTokens.size()) {
            Assert.fail("Too many tokens");
        }
        if (token.getType() == smaliParser.INVALID_TOKEN) {
            Assert.assertTrue("Encountered an INVALID_TOKEN not on the error channel", token.getChannel() == smaliParser.ERROR_CHANNEL);
        }
        ExpectedToken expectedToken = expectedTokens.get(expectedTokenIndex++);
        if (!tokenTypesByName.containsKey(expectedToken.tokenName)) {
            Assert.fail("Unknown token: " + expectedToken.tokenName);
        }
        int expectedTokenType = tokenTypesByName.get(expectedToken.tokenName);
        if (token.getType() != expectedTokenType) {
            Assert.fail(String.format("Invalid token at index %d. Expecting %s, got %s(%s)", expectedTokenIndex - 1, expectedToken.tokenName, getTokenName(token.getType()), token.getText()));
        }
        if (expectedToken.tokenText != null) {
            if (!expectedToken.tokenText.equals(token.getText())) {
                Assert.fail(String.format("Invalid token text at index %d. Expecting text \"%s\", got \"%s\"", expectedTokenIndex - 1, expectedToken.tokenText, token.getText()));
            }
        }
    }
    if (expectedTokenIndex < expectedTokens.size()) {
        Assert.fail(String.format("Not enough tokens. Expecting %d tokens, but got %d", expectedTokens.size(), expectedTokenIndex));
    }
}
Also used : CommonTokenStream(org.antlr.runtime.CommonTokenStream) InputStreamReader(java.io.InputStreamReader) ANTLRInputStream(org.antlr.runtime.ANTLRInputStream) InputStream(java.io.InputStream) ExpectedToken(org.jf.smali.expectedTokensTestGrammarParser.ExpectedToken) IOException(java.io.IOException) List(java.util.List) CommonToken(org.antlr.runtime.CommonToken) File(java.io.File) ANTLRInputStream(org.antlr.runtime.ANTLRInputStream) RecognitionException(org.antlr.runtime.RecognitionException)

Example 3 with CommonTokenStream

use of org.antlr.runtime.CommonTokenStream in project SQLWindowing by hbutani.

the class ParseUtils method parse.

public static QuerySpec parse(String query) throws WindowingException {
    Windowing2Lexer lexer;
    CommonTokenStream tokens;
    Windowing2Parser parser = null;
    CommonTree t;
    CommonTreeNodeStream nodes;
    QSpecBuilder2 qSpecBldr = null;
    String err;
    try {
        lexer = new Windowing2Lexer(new ANTLRStringStream(query));
        tokens = new CommonTokenStream(lexer);
        parser = new Windowing2Parser(tokens);
        parser.setTreeAdaptor(TranslateUtils.adaptor);
        t = (CommonTree) parser.query().getTree();
        err = parser.getWindowingParseErrors();
        if (err != null) {
            throw new WindowingException(err);
        }
    } catch (WindowingException we) {
        throw we;
    } catch (Throwable te) {
        err = parser.getWindowingParseErrors();
        if (err != null) {
            throw new WindowingException(err);
        }
        throw new WindowingException("Parse Error:" + te.toString(), te);
    }
    TranslateUtils.unescapeStringLiterals((ASTNode) t);
    try {
        nodes = new CommonTreeNodeStream(t);
        nodes.setTokenStream(tokens);
        qSpecBldr = new QSpecBuilder2(nodes);
        qSpecBldr.query();
        err = qSpecBldr.getWindowingParseErrors();
        if (err != null) {
            throw new WindowingException(err);
        }
        return qSpecBldr.getQuerySpec();
    } catch (WindowingException we) {
        throw we;
    } catch (Throwable te) {
        err = qSpecBldr.getWindowingParseErrors();
        if (err != null) {
            throw new WindowingException(err);
        }
        throw new WindowingException("Parse Error:" + te.toString(), te);
    }
}
Also used : ANTLRStringStream(org.antlr.runtime.ANTLRStringStream) CommonTokenStream(org.antlr.runtime.CommonTokenStream) CommonTree(org.antlr.runtime.tree.CommonTree) WindowingException(com.sap.hadoop.windowing.WindowingException) CommonTreeNodeStream(org.antlr.runtime.tree.CommonTreeNodeStream)

Example 4 with CommonTokenStream

use of org.antlr.runtime.CommonTokenStream in project nifi by apache.

the class ExpressionCompiler method compileTree.

private Tree compileTree(final String expression) throws AttributeExpressionLanguageParsingException {
    try {
        final CharStream input = new ANTLRStringStream(expression);
        final AttributeExpressionLexer lexer = new AttributeExpressionLexer(input);
        final CommonTokenStream lexerTokenStream = new CommonTokenStream(lexer);
        final AttributeExpressionParser parser = new AttributeExpressionParser(lexerTokenStream);
        final Tree ast = (Tree) parser.query().getTree();
        final Tree tree = ast.getChild(0);
        // ensure that we are able to build the evaluators, so that we validate syntax
        final Evaluator<?> evaluator = buildEvaluator(tree);
        verifyMappingEvaluatorReduced(evaluator);
        return tree;
    } catch (final AttributeExpressionLanguageParsingException e) {
        throw e;
    } catch (final Exception e) {
        throw new AttributeExpressionLanguageParsingException(e);
    }
}
Also used : ANTLRStringStream(org.antlr.runtime.ANTLRStringStream) CommonTokenStream(org.antlr.runtime.CommonTokenStream) AttributeExpressionParser(org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser) Tree(org.antlr.runtime.tree.Tree) AttributeExpressionLexer(org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionLexer) AttributeExpressionLanguageParsingException(org.apache.nifi.attribute.expression.language.exception.AttributeExpressionLanguageParsingException) CharStream(org.antlr.runtime.CharStream) AttributeExpressionLanguageException(org.apache.nifi.attribute.expression.language.exception.AttributeExpressionLanguageException) UnknownHostException(java.net.UnknownHostException) AttributeExpressionLanguageParsingException(org.apache.nifi.attribute.expression.language.exception.AttributeExpressionLanguageParsingException)

Example 5 with CommonTokenStream

use of org.antlr.runtime.CommonTokenStream in project nifi by apache.

the class HL7Query method compile.

public static HL7Query compile(final String query) {
    try {
        final CommonTokenStream lexerTokenStream = createTokenStream(query);
        final HL7QueryParser parser = new HL7QueryParser(lexerTokenStream);
        final Tree tree = (Tree) parser.query().getTree();
        return new HL7Query(tree, query);
    } catch (final HL7QueryParsingException e) {
        throw e;
    } catch (final Exception e) {
        throw new HL7QueryParsingException(e);
    }
}
Also used : CommonTokenStream(org.antlr.runtime.CommonTokenStream) HL7QueryParser(org.apache.nifi.hl7.query.antlr.HL7QueryParser) HL7QueryParsingException(org.apache.nifi.hl7.query.exception.HL7QueryParsingException) Tree(org.antlr.runtime.tree.Tree) HL7QueryParsingException(org.apache.nifi.hl7.query.exception.HL7QueryParsingException)

Aggregations

CommonTokenStream (org.antlr.runtime.CommonTokenStream)93 ANTLRStringStream (org.antlr.runtime.ANTLRStringStream)62 RecognitionException (org.antlr.runtime.RecognitionException)25 CharStream (org.antlr.runtime.CharStream)22 CommonTree (org.antlr.runtime.tree.CommonTree)21 TokenStream (org.antlr.runtime.TokenStream)17 File (java.io.File)12 Test (org.junit.Test)12 CommonToken (org.antlr.runtime.CommonToken)10 CommonTreeNodeStream (org.antlr.runtime.tree.CommonTreeNodeStream)10 ExprLexer (org.apache.drill.common.expression.parser.ExprLexer)10 ExprParser (org.apache.drill.common.expression.parser.ExprParser)10 FileInputStream (java.io.FileInputStream)9 InputStream (java.io.InputStream)8 ANTLRInputStream (org.antlr.runtime.ANTLRInputStream)8 InternalSimpleExpressionsTestLanguageLexer (org.eclipse.xtext.testlanguages.parser.antlr.internal.InternalSimpleExpressionsTestLanguageLexer)8 InputStreamReader (java.io.InputStreamReader)6 Token (org.antlr.runtime.Token)6 JPA2Lexer (com.haulmont.cuba.core.sys.jpql.antlr2.JPA2Lexer)5 JPA2Parser (com.haulmont.cuba.core.sys.jpql.antlr2.JPA2Parser)5