Examples with ANTLRStringStream org.antlr.runtime.ANTLRStringStream used on opensource projects

Example 26 with ANTLRStringStream

use of org.antlr.runtime.ANTLRStringStream in project xtext-core by eclipse.

the class LexerErrorTest method testLexerError_02.

@Test
public void testLexerError_02() throws Exception {
    String model = "a 'incomplete string";
    InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
    lexer.setCharStream(new ANTLRStringStream(model));
    CommonTokenStream stream = new CommonTokenStream(lexer);
    @SuppressWarnings("unchecked") List<CommonToken> tokens = stream.getTokens();
    assertEquals(tokens.toString(), 3, tokens.size());
    assertEquals("a", tokens.get(0).getText());
    assertEquals(" ", tokens.get(1).getText());
    assertEquals("'incomplete string", tokens.get(2).getText());
    assertEquals(0, tokens.get(2).getType());
}

Example 27 with ANTLRStringStream

use of org.antlr.runtime.ANTLRStringStream in project xtext-core by eclipse.

the class LexerErrorTest method testLexerError_03.

@Test
public void testLexerError_03() throws Exception {
    String model = "a '\\ incomplete string with bad escape sequence";
    InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
    lexer.setCharStream(new ANTLRStringStream(model));
    CommonTokenStream stream = new CommonTokenStream(lexer);
    @SuppressWarnings("unchecked") List<CommonToken> tokens = stream.getTokens();
    assertEquals(tokens.toString(), 3, tokens.size());
    assertEquals("a", tokens.get(0).getText());
    assertEquals(" ", tokens.get(1).getText());
    assertEquals("'\\ incomplete string with bad escape sequence", tokens.get(2).getText());
    assertEquals(0, tokens.get(2).getType());
}

Example 28 with ANTLRStringStream

use of org.antlr.runtime.ANTLRStringStream in project xtext-core by eclipse.

the class LexerErrorTest method testLexerError_04.

@Test
public void testLexerError_04() throws Exception {
    String model = "a 'incomplete string with bad escape sequence \\";
    InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
    lexer.setCharStream(new ANTLRStringStream(model));
    CommonTokenStream stream = new CommonTokenStream(lexer);
    @SuppressWarnings("unchecked") List<CommonToken> tokens = stream.getTokens();
    assertEquals(tokens.toString(), 3, tokens.size());
    assertEquals("a", tokens.get(0).getText());
    assertEquals(" ", tokens.get(1).getText());
    assertEquals("'incomplete string with bad escape sequence \\", tokens.get(2).getText());
    assertEquals(0, tokens.get(2).getType());
}

Example 29 with ANTLRStringStream

use of org.antlr.runtime.ANTLRStringStream in project xtext-core by eclipse.

the class TokenRegionProviderTest method getTokens.

protected List<CommonToken> getTokens(String model) {
    Lexer lexer = get(Lexer.class);
    lexer.setCharStream(new ANTLRStringStream(model));
    List<CommonToken> tokens = newArrayList();
    CommonToken currentToken;
    do {
        currentToken = (CommonToken) lexer.nextToken();
        // System.out.println(currentToken);
        tokens.add(currentToken);
    } while (currentToken != Token.EOF_TOKEN);
    return tokens;
}

Example 30 with ANTLRStringStream

use of org.antlr.runtime.ANTLRStringStream in project xtext-core by eclipse.

the class TokenRegionProvider method getTokenRegion.

public ITextRegion getTokenRegion(String text, ITextRegion region) {
    Lexer lexer = lexerProvider.get();
    lexer.setCharStream(new ANTLRStringStream(text));
    int currentStart = 0;
    int currentEnd = 0;
    CommonToken nextToken = (CommonToken) lexer.nextToken();
    int regionStart = region.getOffset();
    int regionEnd = regionStart + region.getLength();
    while (nextToken != Token.EOF_TOKEN && currentEnd <= regionStart) {
        currentStart = nextToken.getStartIndex();
        currentEnd = nextToken.getStopIndex() + 1;
        nextToken = (CommonToken) lexer.nextToken();
    }
    // nextToken is either EOF or the first token that follows the start of the given region
    while (nextToken != Token.EOF_TOKEN && currentEnd < regionEnd) {
        currentEnd = nextToken.getStopIndex() + 1;
        nextToken = (CommonToken) lexer.nextToken();
    }
    if (currentStart != regionStart || currentEnd != regionEnd)
        return new TextRegion(currentStart, currentEnd - currentStart);
    else
        return region;
}