Examples with ANTLRStringStream org.antlr.runtime.ANTLRStringStream used on opensource projects

Example 46 with ANTLRStringStream

use of org.antlr.runtime.ANTLRStringStream in project drill by apache.

the class ExecTest method parseExpr.

protected LogicalExpression parseExpr(String expr) throws RecognitionException {
    final ExprLexer lexer = new ExprLexer(new ANTLRStringStream(expr));
    final CommonTokenStream tokens = new CommonTokenStream(lexer);
    final ExprParser parser = new ExprParser(tokens);
    final ExprParser.parse_return ret = parser.parse();
    return ret.e;
}

Example 47 with ANTLRStringStream

use of org.antlr.runtime.ANTLRStringStream in project xtext-xtend by eclipse.

the class AbstractSmokeTest method testSkipTokensInBetween.

@Test
public void testSkipTokensInBetween() throws Exception {
    for (String string : smokeTestModels) {
        List<CommonToken> tokenList = Lists.newArrayList();
        {
            Lexer lexer = lexerProvider.get();
            lexer.setCharStream(new ANTLRStringStream(string));
            Token token = lexer.nextToken();
            while (token != Token.EOF_TOKEN) {
                tokenList.add((CommonToken) token);
                token = lexer.nextToken();
            }
        }
        for (CommonToken token : tokenList) {
            int start = token.getStartIndex();
            int length = token.getText().length();
            logProgress(token);
            doParseAndCheckForSmoke(string.substring(0, start) + string.substring(start + length));
        }
    }
}

Example 48 with ANTLRStringStream

use of org.antlr.runtime.ANTLRStringStream in project xtext-xtend by eclipse.

the class LexerBenchmark method timeLexing.

public int timeLexing(int reps) throws Exception {
    int result = reps;
    for (int i = 0; i < reps; i++) {
        Token tok = null;
        lexer.setCharStream(new ANTLRStringStream(contentToParse));
        while (tok != Token.EOF_TOKEN) {
            tok = lexer.nextToken();
            result++;
        }
        result += reps;
    }
    return result;
}

Example 49 with ANTLRStringStream

use of org.antlr.runtime.ANTLRStringStream in project xtext-core by eclipse.

the class PartialContentAssistContextFactory method createContextsForLastCompleteNode.

@Override
protected void createContextsForLastCompleteNode(EObject previousModel, boolean strict) {
    String currentNodePrefix = getPrefix(currentNode);
    if (!Strings.isEmpty(currentNodePrefix) && !currentNode.getText().equals(currentNodePrefix)) {
        lexer.setCharStream(new ANTLRStringStream(currentNodePrefix));
        Token token = lexer.nextToken();
        if (token == Token.EOF_TOKEN) {
            return;
        }
        while (token != Token.EOF_TOKEN) {
            if (isErrorToken(token)) {
                return;
            }
            token = lexer.nextToken();
        }
    }
    String prefix = "";
    Collection<FollowElement> followElements = parseFollowElements(completionOffset, strict);
    doCreateContexts(lastCompleteNode, currentNode, prefix, previousModel, followElements);
}

Example 50 with ANTLRStringStream

use of org.antlr.runtime.ANTLRStringStream in project xtext-core by eclipse.

the class LexerErrorTest method testLexerError_01.

@Test
public void testLexerError_01() throws Exception {
    String model = "a /* incomplete comment *";
    InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
    lexer.setCharStream(new ANTLRStringStream(model));
    CommonTokenStream stream = new CommonTokenStream(lexer);
    @SuppressWarnings("unchecked") List<CommonToken> tokens = stream.getTokens();
    assertEquals(tokens.toString(), 3, tokens.size());
    assertEquals("a", tokens.get(0).getText());
    assertEquals(" ", tokens.get(1).getText());
    assertEquals("/* incomplete comment *", tokens.get(2).getText());
    assertEquals(0, tokens.get(2).getType());
}