Examples with Lexer org.eclipse.xtext.parser.antlr.Lexer used on opensource projects

Example 6 with Lexer

use of org.eclipse.xtext.parser.antlr.Lexer in project xtext-core by eclipse.

the class AbstractSmokeTest method testSkipTokensInBetween.

@Test
public void testSkipTokensInBetween() throws Exception {
    for (String string : getSmokeTestModels()) {
        List<CommonToken> tokenList = Lists.newArrayList();
        {
            Lexer lexer = lexerProvider.get();
            lexer.setCharStream(new ANTLRStringStream(string));
            Token token = lexer.nextToken();
            while (token != Token.EOF_TOKEN) {
                tokenList.add((CommonToken) token);
                token = lexer.nextToken();
            }
        }
        for (CommonToken token : tokenList) {
            int start = token.getStartIndex();
            int length = token.getText().length();
            logProgress(token);
            processModel(string.substring(0, start) + string.substring(start + length));
        }
    }
}

Example 7 with Lexer

use of org.eclipse.xtext.parser.antlr.Lexer in project xtext-core by eclipse.

the class AbstractSmokeTest method testSkipTokensInBetweenWithoutResourceSet.

@Test
public void testSkipTokensInBetweenWithoutResourceSet() throws Exception {
    for (String string : getSmokeTestModels()) {
        List<CommonToken> tokenList = Lists.newArrayList();
        {
            Lexer lexer = lexerProvider.get();
            lexer.setCharStream(new ANTLRStringStream(string));
            Token token = lexer.nextToken();
            while (token != Token.EOF_TOKEN) {
                tokenList.add((CommonToken) token);
                token = lexer.nextToken();
            }
        }
        for (CommonToken token : tokenList) {
            int start = token.getStartIndex();
            int length = token.getText().length();
            logProgress(token);
            processModelWithoutResourceSet(string.substring(0, start) + string.substring(start + length));
        }
    }
}

Example 8 with Lexer

use of org.eclipse.xtext.parser.antlr.Lexer in project xtext-core by eclipse.

the class AntlrProposalConflictHelper method initTokenSource.

protected void initTokenSource(String text, TokenSource tokenSource, ContentAssistContext context) {
    Lexer lexer = (Lexer) tokenSource;
    CharStream stream = new ANTLRStringStream(text);
    lexer.setCharStream(stream);
}

Example 9 with Lexer

use of org.eclipse.xtext.parser.antlr.Lexer in project xtext-core by eclipse.

the class TokenRegionProviderTest method getTokens.

protected List<CommonToken> getTokens(String model) {
    Lexer lexer = get(Lexer.class);
    lexer.setCharStream(new ANTLRStringStream(model));
    List<CommonToken> tokens = newArrayList();
    CommonToken currentToken;
    do {
        currentToken = (CommonToken) lexer.nextToken();
        // System.out.println(currentToken);
        tokens.add(currentToken);
    } while (currentToken != Token.EOF_TOKEN);
    return tokens;
}

Example 10 with Lexer

use of org.eclipse.xtext.parser.antlr.Lexer in project xtext-core by eclipse.

the class TokenRegionProvider method getTokenRegion.

public ITextRegion getTokenRegion(String text, ITextRegion region) {
    Lexer lexer = lexerProvider.get();
    lexer.setCharStream(new ANTLRStringStream(text));
    int currentStart = 0;
    int currentEnd = 0;
    CommonToken nextToken = (CommonToken) lexer.nextToken();
    int regionStart = region.getOffset();
    int regionEnd = regionStart + region.getLength();
    while (nextToken != Token.EOF_TOKEN && currentEnd <= regionStart) {
        currentStart = nextToken.getStartIndex();
        currentEnd = nextToken.getStopIndex() + 1;
        nextToken = (CommonToken) lexer.nextToken();
    }
    // nextToken is either EOF or the first token that follows the start of the given region
    while (nextToken != Token.EOF_TOKEN && currentEnd < regionEnd) {
        currentEnd = nextToken.getStopIndex() + 1;
        nextToken = (CommonToken) lexer.nextToken();
    }
    if (currentStart != regionStart || currentEnd != regionEnd)
        return new TextRegion(currentStart, currentEnd - currentStart);
    else
        return region;
}