Examples with CommonTokenStream org.antlr.runtime.CommonTokenStream used on opensource projects

Example 41 with CommonTokenStream

use of org.antlr.runtime.CommonTokenStream in project drools by kiegroup.

the class DSLTokenizedMappingFile method buildFileMappingWalker.

private DSLMapWalker buildFileMappingWalker(final List<ParserError> errors, CharStream stream) throws RecognitionException {
    DSLMapLexer lexer = new DSLMapLexer(stream);
    CommonTokenStream tokens = new CommonTokenStream();
    tokens.setTokenSource(lexer);
    DSLMapParser parser = new DSLMapParser(tokens);
    DSLMapParser.mapping_file_return example = parser.mapping_file();
    CommonTree tree = (CommonTree) example.getTree();
    // logger.info(tree.toStringTree());
    CommonTreeNodeStream nodes = new CommonTreeNodeStream(tree);
    DSLMapWalker walker = new DSLMapWalker(nodes);
    errors.addAll(lexer.getErrors());
    errors.addAll(parser.getErrors());
    return walker;
}

Example 42 with CommonTokenStream

use of org.antlr.runtime.CommonTokenStream in project xtext-core by eclipse.

the class LexerErrorTest method testLexerError_01.

@Test
public void testLexerError_01() throws Exception {
    String model = "a /* incomplete comment *";
    InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
    lexer.setCharStream(new ANTLRStringStream(model));
    CommonTokenStream stream = new CommonTokenStream(lexer);
    @SuppressWarnings("unchecked") List<CommonToken> tokens = stream.getTokens();
    assertEquals(tokens.toString(), 3, tokens.size());
    assertEquals("a", tokens.get(0).getText());
    assertEquals(" ", tokens.get(1).getText());
    assertEquals("/* incomplete comment *", tokens.get(2).getText());
    assertEquals(0, tokens.get(2).getType());
}

Example 43 with CommonTokenStream

use of org.antlr.runtime.CommonTokenStream in project xtext-core by eclipse.

the class LexerErrorTest method testLexerError_06.

@Test
public void testLexerError_06() throws Exception {
    String model = "a '";
    InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
    lexer.setCharStream(new ANTLRStringStream(model));
    CommonTokenStream stream = new CommonTokenStream(lexer);
    @SuppressWarnings("unchecked") List<CommonToken> tokens = stream.getTokens();
    assertEquals(tokens.toString(), 3, tokens.size());
    assertEquals("a", tokens.get(0).getText());
    assertEquals(" ", tokens.get(1).getText());
    assertEquals("'", tokens.get(2).getText());
    assertEquals(InternalSimpleExpressionsTestLanguageParser.RULE_ANY_OTHER, tokens.get(2).getType());
}

Example 44 with CommonTokenStream

use of org.antlr.runtime.CommonTokenStream in project xtext-core by eclipse.

the class LexerErrorTest method testLexerError_05.

@Test
public void testLexerError_05() throws Exception {
    String model = "a 'incomplete string \\'";
    InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
    lexer.setCharStream(new ANTLRStringStream(model));
    CommonTokenStream stream = new CommonTokenStream(lexer);
    @SuppressWarnings("unchecked") List<CommonToken> tokens = stream.getTokens();
    assertEquals(tokens.toString(), 3, tokens.size());
    assertEquals("a", tokens.get(0).getText());
    assertEquals(" ", tokens.get(1).getText());
    assertEquals("'incomplete string \\'", tokens.get(2).getText());
    assertEquals(0, tokens.get(2).getType());
}

Example 45 with CommonTokenStream

use of org.antlr.runtime.CommonTokenStream in project xtext-core by eclipse.

the class LexerSLComment method testSlComment.

@Test
public void testSlComment() {
    String model = "//sl comment\na";
    InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
    lexer.setCharStream(new ANTLRStringStream(model));
    CommonTokenStream stream = new CommonTokenStream(lexer);
    Object inLineComment = stream.getTokens().get(0);
    assertTrue(inLineComment instanceof CommonToken);
    assertEquals(InternalSimpleExpressionsTestLanguageLexer.RULE_SL_COMMENT, ((CommonToken) inLineComment).getType());
}