use of org.antlr.v4.runtime.UnbufferedTokenStream in project pinot by linkedin.
the class Pql2Compiler method compileToBrokerRequest.
@Override
public BrokerRequest compileToBrokerRequest(String expression) throws Pql2CompilationException {
try {
//
CharStream charStream = new ANTLRInputStream(expression);
PQL2Lexer lexer = new PQL2Lexer(charStream);
lexer.setTokenFactory(new CommonTokenFactory(true));
TokenStream tokenStream = new UnbufferedTokenStream<CommonToken>(lexer);
PQL2Parser parser = new PQL2Parser(tokenStream);
parser.setErrorHandler(new BailErrorStrategy());
// Parse
ParseTree parseTree = parser.root();
ParseTreeWalker walker = new ParseTreeWalker();
Pql2AstListener listener = new Pql2AstListener(expression);
walker.walk(listener, parseTree);
AstNode rootNode = listener.getRootNode();
BrokerRequest brokerRequest = new BrokerRequest();
rootNode.updateBrokerRequest(brokerRequest);
return brokerRequest;
} catch (Pql2CompilationException e) {
throw e;
} catch (Exception e) {
throw new Pql2CompilationException(e.getMessage());
}
}
use of org.antlr.v4.runtime.UnbufferedTokenStream in project pinot by linkedin.
the class Pql2Compiler method compileToExpressionTree.
@Override
public TransformExpressionTree compileToExpressionTree(String expression) {
CharStream charStream = new ANTLRInputStream(expression);
PQL2Lexer lexer = new PQL2Lexer(charStream);
lexer.setTokenFactory(new CommonTokenFactory(true));
TokenStream tokenStream = new UnbufferedTokenStream<CommonToken>(lexer);
PQL2Parser parser = new PQL2Parser(tokenStream);
parser.setErrorHandler(new BailErrorStrategy());
// Parse
ParseTree parseTree = parser.expression();
ParseTreeWalker walker = new ParseTreeWalker();
Pql2AstListener listener = new Pql2AstListener(expression);
walker.walk(listener, parseTree);
final AstNode rootNode = listener.getRootNode();
return TransformExpressionTree.buildTree(rootNode);
}
use of org.antlr.v4.runtime.UnbufferedTokenStream in project antlr4 by antlr.
the class TestUnbufferedTokenStream method testLookahead.
@Test
public void testLookahead() throws Exception {
LexerGrammar g = new LexerGrammar("lexer grammar t;\n" + "ID : 'a'..'z'+;\n" + "INT : '0'..'9'+;\n" + "SEMI : ';';\n" + "ASSIGN : '=';\n" + "PLUS : '+';\n" + "MULT : '*';\n" + "WS : ' '+;\n");
// Tokens: 012345678901234567
// Input: x = 302;
CharStream input = new ANTLRInputStream(new StringReader("x = 302;"));
LexerInterpreter lexEngine = g.createLexerInterpreter(input);
TokenStream tokens = new UnbufferedTokenStream<Token>(lexEngine);
assertEquals("x", tokens.LT(1).getText());
assertEquals(" ", tokens.LT(2).getText());
assertEquals("=", tokens.LT(3).getText());
assertEquals(" ", tokens.LT(4).getText());
assertEquals("302", tokens.LT(5).getText());
assertEquals(";", tokens.LT(6).getText());
}
use of org.antlr.v4.runtime.UnbufferedTokenStream in project Alpha by alpha-asp.
the class Main method parseVisit.
public static ParsedProgram parseVisit(ANTLRInputStream is) throws IOException {
/*
// In order to require less memory: use unbuffered streams and avoid constructing a full parse tree.
ASPCore2Lexer lexer = new ASPCore2Lexer(new UnbufferedCharStream(is));
lexer.setTokenFactory(new CommonTokenFactory(true));
final ASPCore2Parser parser = new ASPCore2Parser(new UnbufferedTokenStream<>(lexer));
parser.setBuildParseTree(false);
*/
CommonTokenStream tokens = new CommonTokenStream(new ASPCore2Lexer(is));
final ASPCore2Parser parser = new ASPCore2Parser(tokens);
// Try SLL parsing mode (faster but may terminate incorrectly).
parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
parser.removeErrorListeners();
parser.setErrorHandler(new BailErrorStrategy());
final CustomErrorListener errorListener = new CustomErrorListener(is.getSourceName());
ASPCore2Parser.ProgramContext programContext;
try {
// Parse program
programContext = parser.program();
} catch (ParseCancellationException e) {
// retry with LL parser and DefaultErrorStrategy printing errors to console.
if (e.getCause() instanceof RecognitionException) {
tokens.reset();
parser.addErrorListener(errorListener);
parser.setErrorHandler(new DefaultErrorStrategy());
parser.getInterpreter().setPredictionMode(PredictionMode.LL);
// Re-run parse.
programContext = parser.program();
} else {
throw e;
}
}
// is attempted) and the user will only see the first error encountered.
if (errorListener.getRecognitionException() != null) {
throw errorListener.getRecognitionException();
}
// Construct internal program representation.
ParsedTreeVisitor visitor = new ParsedTreeVisitor();
return (ParsedProgram) visitor.visitProgram(programContext);
}
Aggregations