Examples with FlexTokenSource org.eclipse.xtend.core.parser.antlr.internal.FlexTokenSource used on opensource projects

Example 1 with FlexTokenSource

use of org.eclipse.xtend.core.parser.antlr.internal.FlexTokenSource in project xtext-xtend by eclipse.

the class FlexTokenRegionProviderTest method getTokens.

protected List<CommonToken> getTokens(String model) {
    FlexTokenSource tokenSource = flexerFactory.createTokenSource(new StringReader(model));
    List<CommonToken> tokens = newArrayList();
    CommonToken currentToken;
    do {
        currentToken = (CommonToken) tokenSource.nextToken();
        tokens.add(currentToken);
    } while (currentToken != Token.EOF_TOKEN);
    return tokens;
}

Example 2 with FlexTokenSource

use of org.eclipse.xtend.core.parser.antlr.internal.FlexTokenSource in project xtext-xtend by eclipse.

the class FlexerBasedTemplateBodyHighlighter method doProvideHighlightingFor.

@Override
public void doProvideHighlightingFor(final String body, final IHighlightedPositionAcceptor acceptor) {
    StringReader _stringReader = new StringReader(body);
    final FlexTokenSource tokenSource = this._flexerFactory.createTokenSource(_stringReader);
    Token token = tokenSource.nextToken();
    while ((!Objects.equal(token, Token.EOF_TOKEN))) {
        {
            final String id = this._abstractAntlrTokenToAttributeIdMapper.getId(token.getType());
            final int offset = TokenTool.getOffset(token);
            final int length = TokenTool.getLength(token);
            acceptor.addPosition(offset, length, id);
            token = tokenSource.nextToken();
        }
    }
}

Example 3 with FlexTokenSource

use of org.eclipse.xtend.core.parser.antlr.internal.FlexTokenSource in project xtext-xtend by eclipse.

the class FlexingTest method assertLexing.

@Override
protected void assertLexing(String input, Pair<String, String>... expectedTokens) {
    FlexTokenSource tokenSource = new FlexerFactory().createTokenSource(new StringReader(input));
    XtextTokenStream tokenStream = new XtextTokenStream(tokenSource, getTokenDefProvider());
    List<?> tokens = tokenStream.getTokens();
    assertEquals(input + " / " + tokens, expectedTokens.length, tokens.size());
    for (int i = 0; i < tokens.size(); i++) {
        Token token = (Token) tokens.get(i);
        assertEquals(token.toString(), expectedTokens[i].getFirst(), token.getText());
        final String expected = expectedTokens[i].getSecond();
        String actual = getTokenDefProvider().getTokenDefMap().get(token.getType());
        assertEquals("expected " + expected + " but was " + actual, expected, actual);
    }
}