Examples with CommonTokenStream org.antlr.runtime.CommonTokenStream used on opensource projects

Example 76 with CommonTokenStream

use of org.antlr.runtime.CommonTokenStream in project xtext-core by eclipse.

the class LexerErrorTest method testLexerError_05.

@Test
public void testLexerError_05() throws Exception {
    String model = "a 'incomplete string \\'";
    InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
    lexer.setCharStream(new ANTLRStringStream(model));
    CommonTokenStream stream = new CommonTokenStream(lexer);
    @SuppressWarnings("unchecked") List<CommonToken> tokens = stream.getTokens();
    assertEquals(tokens.toString(), 3, tokens.size());
    assertEquals("a", tokens.get(0).getText());
    assertEquals(" ", tokens.get(1).getText());
    assertEquals("'incomplete string \\'", tokens.get(2).getText());
    assertEquals(0, tokens.get(2).getType());
}

Example 77 with CommonTokenStream

use of org.antlr.runtime.CommonTokenStream in project xtext-core by eclipse.

the class LexerErrorTest method testLexerError_01.

@Test
public void testLexerError_01() throws Exception {
    String model = "a /* incomplete comment *";
    InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
    lexer.setCharStream(new ANTLRStringStream(model));
    CommonTokenStream stream = new CommonTokenStream(lexer);
    @SuppressWarnings("unchecked") List<CommonToken> tokens = stream.getTokens();
    assertEquals(tokens.toString(), 3, tokens.size());
    assertEquals("a", tokens.get(0).getText());
    assertEquals(" ", tokens.get(1).getText());
    assertEquals("/* incomplete comment *", tokens.get(2).getText());
    assertEquals(0, tokens.get(2).getType());
}

Example 78 with CommonTokenStream

use of org.antlr.runtime.CommonTokenStream in project xtext-core by eclipse.

the class LexerErrorTest method testLexerError_06.

@Test
public void testLexerError_06() throws Exception {
    String model = "a '";
    InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
    lexer.setCharStream(new ANTLRStringStream(model));
    CommonTokenStream stream = new CommonTokenStream(lexer);
    @SuppressWarnings("unchecked") List<CommonToken> tokens = stream.getTokens();
    assertEquals(tokens.toString(), 3, tokens.size());
    assertEquals("a", tokens.get(0).getText());
    assertEquals(" ", tokens.get(1).getText());
    assertEquals("'", tokens.get(2).getText());
    assertEquals(InternalSimpleExpressionsTestLanguageParser.RULE_ANY_OTHER, tokens.get(2).getType());
}

Example 79 with CommonTokenStream

use of org.antlr.runtime.CommonTokenStream in project drools by kiegroup.

the class DSLTokenizedMappingFile method buildFileMappingWalker.

private DSLMapWalker buildFileMappingWalker(final List<ParserError> errors, CharStream stream) throws RecognitionException {
    DSLMapLexer lexer = new DSLMapLexer(stream);
    CommonTokenStream tokens = new CommonTokenStream();
    tokens.setTokenSource(lexer);
    DSLMapParser parser = new DSLMapParser(tokens);
    DSLMapParser.mapping_file_return example = parser.mapping_file();
    CommonTree tree = (CommonTree) example.getTree();
    // logger.info(tree.toStringTree());
    CommonTreeNodeStream nodes = new CommonTreeNodeStream(tree);
    DSLMapWalker walker = new DSLMapWalker(nodes);
    errors.addAll(lexer.getErrors());
    errors.addAll(parser.getErrors());
    return walker;
}

Example 80 with CommonTokenStream

use of org.antlr.runtime.CommonTokenStream in project drools by kiegroup.

the class DrlExprParser method parse.

/**
 * Parse an expression from text
 */
public ConstraintConnectiveDescr parse(final String text) {
    ConstraintConnectiveDescr constraint = null;
    try {
        DRLLexer lexer = DRLFactory.getDRLLexer(new ANTLRStringStream(text), languageLevel);
        CommonTokenStream input = new CommonTokenStream(lexer);
        RecognizerSharedState state = new RecognizerSharedState();
        helper = new ParserHelper(input, state, languageLevel);
        DRLExpressions parser = DRLFactory.getDRLExpressions(input, state, helper, languageLevel);
        parser.setBuildDescr(true);
        // setting initial value just in case
        parser.setLeftMostExpr(null);
        BaseDescr expr = parser.conditionalOrExpression();
        if (expr != null && !parser.hasErrors()) {
            constraint = ConstraintConnectiveDescr.newAnd();
            constraint.addOrMerge(expr);
        }
    } catch (RecognitionException e) {
        helper.reportError(e);
    }
    return constraint;
}