use of org.antlr.runtime.CommonTokenStream in project freud by LMAX-Exchange.
the class JavaSourceJdomParser method parseJavaSourceToDocument.
private static Document parseJavaSourceToDocument(final Reader javaSourceReader) throws RecognitionException, IOException {
JavaParser parser = new JavaParser(new CommonTokenStream(new JavaLexer(new ANTLRReaderStream(javaSourceReader))));
final JdomTreeAdaptor treeAdaptor = new JdomTreeAdaptor(JAVA_SOURCE_ROOT_ELEMENT_NAME, JAVA_SOURCE_TOKEN_TYPES);
parser.setTreeAdaptor(treeAdaptor);
parser.compilationUnit();
return treeAdaptor.getDocument();
}
use of org.antlr.runtime.CommonTokenStream in project smali by JesusFreke.
the class LexerTest method runTest.
public void runTest(String test, boolean discardHiddenTokens) {
String smaliFile = String.format("LexerTest%s%s.smali", File.separatorChar, test);
String tokensFile = String.format("LexerTest%s%s.tokens", File.separatorChar, test);
org.jf.smali.expectedTokensTestGrammarLexer expectedTokensLexer = null;
try {
expectedTokensLexer = new org.jf.smali.expectedTokensTestGrammarLexer(new ANTLRInputStream(LexerTest.class.getClassLoader().getResourceAsStream(tokensFile)));
} catch (IOException ex) {
throw new RuntimeException(ex);
}
CommonTokenStream expectedTokensStream = new CommonTokenStream(expectedTokensLexer);
org.jf.smali.expectedTokensTestGrammarParser expectedTokensParser = new org.jf.smali.expectedTokensTestGrammarParser(expectedTokensStream);
try {
expectedTokensParser.top();
} catch (RecognitionException ex) {
throw new RuntimeException(ex);
}
List<ExpectedToken> expectedTokens = expectedTokensParser.getExpectedTokens();
InputStream smaliStream = LexerTest.class.getClassLoader().getResourceAsStream(smaliFile);
if (smaliStream == null) {
Assert.fail("Could not load " + smaliFile);
}
smaliFlexLexer lexer = new smaliFlexLexer(new InputStreamReader(smaliStream));
lexer.setSourceFile(new File(test + ".smali"));
lexer.setSuppressErrors(true);
CommonTokenStream tokenStream = new CommonTokenStream(lexer);
tokenStream.fill();
List tokens = tokenStream.getTokens();
int expectedTokenIndex = 0;
CommonToken token;
for (int i = 0; i < tokens.size() - 1; i++) {
token = (CommonToken) tokens.get(i);
if (discardHiddenTokens && token.getChannel() == smaliParser.HIDDEN) {
continue;
}
if (expectedTokenIndex >= expectedTokens.size()) {
Assert.fail("Too many tokens");
}
if (token.getType() == smaliParser.INVALID_TOKEN) {
Assert.assertTrue("Encountered an INVALID_TOKEN not on the error channel", token.getChannel() == smaliParser.ERROR_CHANNEL);
}
ExpectedToken expectedToken = expectedTokens.get(expectedTokenIndex++);
if (!tokenTypesByName.containsKey(expectedToken.tokenName)) {
Assert.fail("Unknown token: " + expectedToken.tokenName);
}
int expectedTokenType = tokenTypesByName.get(expectedToken.tokenName);
if (token.getType() != expectedTokenType) {
Assert.fail(String.format("Invalid token at index %d. Expecting %s, got %s(%s)", expectedTokenIndex - 1, expectedToken.tokenName, getTokenName(token.getType()), token.getText()));
}
if (expectedToken.tokenText != null) {
if (!expectedToken.tokenText.equals(token.getText())) {
Assert.fail(String.format("Invalid token text at index %d. Expecting text \"%s\", got \"%s\"", expectedTokenIndex - 1, expectedToken.tokenText, token.getText()));
}
}
}
if (expectedTokenIndex < expectedTokens.size()) {
Assert.fail(String.format("Not enough tokens. Expecting %d tokens, but got %d", expectedTokens.size(), expectedTokenIndex));
}
}
use of org.antlr.runtime.CommonTokenStream in project SQLWindowing by hbutani.
the class ParseUtils method parse.
public static QuerySpec parse(String query) throws WindowingException {
Windowing2Lexer lexer;
CommonTokenStream tokens;
Windowing2Parser parser = null;
CommonTree t;
CommonTreeNodeStream nodes;
QSpecBuilder2 qSpecBldr = null;
String err;
try {
lexer = new Windowing2Lexer(new ANTLRStringStream(query));
tokens = new CommonTokenStream(lexer);
parser = new Windowing2Parser(tokens);
parser.setTreeAdaptor(TranslateUtils.adaptor);
t = (CommonTree) parser.query().getTree();
err = parser.getWindowingParseErrors();
if (err != null) {
throw new WindowingException(err);
}
} catch (WindowingException we) {
throw we;
} catch (Throwable te) {
err = parser.getWindowingParseErrors();
if (err != null) {
throw new WindowingException(err);
}
throw new WindowingException("Parse Error:" + te.toString(), te);
}
TranslateUtils.unescapeStringLiterals((ASTNode) t);
try {
nodes = new CommonTreeNodeStream(t);
nodes.setTokenStream(tokens);
qSpecBldr = new QSpecBuilder2(nodes);
qSpecBldr.query();
err = qSpecBldr.getWindowingParseErrors();
if (err != null) {
throw new WindowingException(err);
}
return qSpecBldr.getQuerySpec();
} catch (WindowingException we) {
throw we;
} catch (Throwable te) {
err = qSpecBldr.getWindowingParseErrors();
if (err != null) {
throw new WindowingException(err);
}
throw new WindowingException("Parse Error:" + te.toString(), te);
}
}
use of org.antlr.runtime.CommonTokenStream in project nifi by apache.
the class ExpressionCompiler method compileTree.
private Tree compileTree(final String expression) throws AttributeExpressionLanguageParsingException {
try {
final CharStream input = new ANTLRStringStream(expression);
final AttributeExpressionLexer lexer = new AttributeExpressionLexer(input);
final CommonTokenStream lexerTokenStream = new CommonTokenStream(lexer);
final AttributeExpressionParser parser = new AttributeExpressionParser(lexerTokenStream);
final Tree ast = (Tree) parser.query().getTree();
final Tree tree = ast.getChild(0);
// ensure that we are able to build the evaluators, so that we validate syntax
final Evaluator<?> evaluator = buildEvaluator(tree);
verifyMappingEvaluatorReduced(evaluator);
return tree;
} catch (final AttributeExpressionLanguageParsingException e) {
throw e;
} catch (final Exception e) {
throw new AttributeExpressionLanguageParsingException(e);
}
}
use of org.antlr.runtime.CommonTokenStream in project nifi by apache.
the class HL7Query method compile.
public static HL7Query compile(final String query) {
try {
final CommonTokenStream lexerTokenStream = createTokenStream(query);
final HL7QueryParser parser = new HL7QueryParser(lexerTokenStream);
final Tree tree = (Tree) parser.query().getTree();
return new HL7Query(tree, query);
} catch (final HL7QueryParsingException e) {
throw e;
} catch (final Exception e) {
throw new HL7QueryParsingException(e);
}
}
Aggregations