Examples with TokenStream org.antlr.runtime.TokenStream used on opensource projects

Example 1 with TokenStream

use of org.antlr.runtime.TokenStream in project cassandra by apache.

the class CQLFragmentParser method parseAnyUnhandled.

/**
     * Just call a parser method in {@link CqlParser} - does not do any error handling.
     */
public static <R> R parseAnyUnhandled(CQLParserFunction<R> parserFunction, String input) throws RecognitionException {
    // Lexer and parser
    ErrorCollector errorCollector = new ErrorCollector(input);
    CharStream stream = new ANTLRStringStream(input);
    CqlLexer lexer = new CqlLexer(stream);
    lexer.addErrorListener(errorCollector);
    TokenStream tokenStream = new CommonTokenStream(lexer);
    CqlParser parser = new CqlParser(tokenStream);
    parser.addErrorListener(errorCollector);
    // Parse the query string to a statement instance
    R r = parserFunction.parse(parser);
    // The errorCollector has queue up any errors that the lexer and parser may have encountered
    // along the way, if necessary, we turn the last error into exceptions here.
    errorCollector.throwFirstSyntaxError();
    return r;
}

Example 2 with TokenStream

use of org.antlr.runtime.TokenStream in project cassandra by apache.

the class CqlParserTest method testRemoveErrorListener.

@Test
public void testRemoveErrorListener() throws Exception {
    SyntaxErrorCounter firstCounter = new SyntaxErrorCounter();
    SyntaxErrorCounter secondCounter = new SyntaxErrorCounter();
    CharStream stream = new ANTLRStringStream("SELECT * FORM test;");
    CqlLexer lexer = new CqlLexer(stream);
    TokenStream tokenStream = new CommonTokenStream(lexer);
    CqlParser parser = new CqlParser(tokenStream);
    parser.addErrorListener(firstCounter);
    parser.addErrorListener(secondCounter);
    parser.removeErrorListener(secondCounter);
    parser.query();
    assertEquals(1, firstCounter.count);
    assertEquals(0, secondCounter.count);
}

Example 3 with TokenStream

use of org.antlr.runtime.TokenStream in project cassandra by apache.

the class ErrorCollector method appendQuerySnippet.

/**
     * Appends a query snippet to the message to help the user to understand the problem.
     *
     * @param parser the parser used to parse the query
     * @param builder the <code>StringBuilder</code> used to build the error message
     */
private void appendQuerySnippet(Parser parser, StringBuilder builder) {
    TokenStream tokenStream = parser.getTokenStream();
    int index = tokenStream.index();
    int size = tokenStream.size();
    Token from = tokenStream.get(getSnippetFirstTokenIndex(index));
    Token to = tokenStream.get(getSnippetLastTokenIndex(index, size));
    Token offending = tokenStream.get(getOffendingTokenIndex(index, size));
    appendSnippet(builder, from, to, offending);
}

Example 4 with TokenStream

use of org.antlr.runtime.TokenStream in project cassandra by apache.

the class CqlParserTest method testAddErrorListener.

@Test
public void testAddErrorListener() throws Exception {
    SyntaxErrorCounter firstCounter = new SyntaxErrorCounter();
    SyntaxErrorCounter secondCounter = new SyntaxErrorCounter();
    CharStream stream = new ANTLRStringStream("SELECT * FORM FROM test");
    CqlLexer lexer = new CqlLexer(stream);
    TokenStream tokenStream = new CommonTokenStream(lexer);
    CqlParser parser = new CqlParser(tokenStream);
    parser.addErrorListener(firstCounter);
    parser.addErrorListener(secondCounter);
    // By default CqlParser should recover from the syntax error by removing FORM
    // but as recoverFromMismatchedToken and recover have been overloaded, it will not
    // and the returned ParsedStatement will be null.
    assertNull(parser.query());
    // Only one error must be reported (mismatched: FORM).
    assertEquals(1, firstCounter.count);
    assertEquals(1, secondCounter.count);
}