Examples with ANTLRStringStream org.antlr.runtime.ANTLRStringStream used on opensource projects

Example 51 with ANTLRStringStream

use of org.antlr.runtime.ANTLRStringStream in project xtext-core by eclipse.

the class LexerErrorTest method testLexerError_06.

@Test
public void testLexerError_06() throws Exception {
    String model = "a '";
    InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
    lexer.setCharStream(new ANTLRStringStream(model));
    CommonTokenStream stream = new CommonTokenStream(lexer);
    @SuppressWarnings("unchecked") List<CommonToken> tokens = stream.getTokens();
    assertEquals(tokens.toString(), 3, tokens.size());
    assertEquals("a", tokens.get(0).getText());
    assertEquals(" ", tokens.get(1).getText());
    assertEquals("'", tokens.get(2).getText());
    assertEquals(InternalSimpleExpressionsTestLanguageParser.RULE_ANY_OTHER, tokens.get(2).getType());
}

Example 52 with ANTLRStringStream

use of org.antlr.runtime.ANTLRStringStream in project xtext-core by eclipse.

the class LexerErrorTest method testLexerError_05.

@Test
public void testLexerError_05() throws Exception {
    String model = "a 'incomplete string \\'";
    InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
    lexer.setCharStream(new ANTLRStringStream(model));
    CommonTokenStream stream = new CommonTokenStream(lexer);
    @SuppressWarnings("unchecked") List<CommonToken> tokens = stream.getTokens();
    assertEquals(tokens.toString(), 3, tokens.size());
    assertEquals("a", tokens.get(0).getText());
    assertEquals(" ", tokens.get(1).getText());
    assertEquals("'incomplete string \\'", tokens.get(2).getText());
    assertEquals(0, tokens.get(2).getType());
}

Example 53 with ANTLRStringStream

use of org.antlr.runtime.ANTLRStringStream in project xtext-core by eclipse.

the class LexerSLComment method testSlComment.

@Test
public void testSlComment() {
    String model = "//sl comment\na";
    InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
    lexer.setCharStream(new ANTLRStringStream(model));
    CommonTokenStream stream = new CommonTokenStream(lexer);
    Object inLineComment = stream.getTokens().get(0);
    assertTrue(inLineComment instanceof CommonToken);
    assertEquals(InternalSimpleExpressionsTestLanguageLexer.RULE_SL_COMMENT, ((CommonToken) inLineComment).getType());
}

Example 54 with ANTLRStringStream

use of org.antlr.runtime.ANTLRStringStream in project xtext-core by eclipse.

the class LexerSLComment method testSlCommentEOF.

/**
 * see BUG 234135: Comments on EOF not detected
 * https://bugs.eclipse.org/bugs/show_bug.cgi?id=234135
 */
@Test
public void testSlCommentEOF() {
    String model = "a\n//sl comment";
    InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
    lexer.setCharStream(new ANTLRStringStream(model));
    CommonTokenStream stream = new CommonTokenStream(lexer);
    Object eofLineComment = stream.getTokens().get(2);
    assertTrue(eofLineComment instanceof CommonToken);
    CommonToken commonToken = (CommonToken) eofLineComment;
    int type = commonToken.getType();
    assertEquals(InternalSimpleExpressionsTestLanguageLexer.RULE_SL_COMMENT, type);
}

Example 55 with ANTLRStringStream

use of org.antlr.runtime.ANTLRStringStream in project xtext-core by eclipse.

the class AbstractLexerBasedConverter method getTokenSource.

protected TokenSource getTokenSource(String escapedValue) {
    Lexer result = getLexer();
    if (result == null)
        return null;
    result.setCharStream(new ANTLRStringStream(escapedValue));
    return result;
}