Examples with TokenScanner org.eclipse.xtext.ui.editor.syntaxcoloring.TokenScanner used on opensource projects

Example 1 with TokenScanner

use of org.eclipse.xtext.ui.editor.syntaxcoloring.TokenScanner in project xtext-eclipse by eclipse.

the class TokenScannerTest method testTrailingRange.

@Test
public void testTrailingRange() throws Exception {
    TokenScanner scanner = getTokenScanner(3, 4, 3, 2);
    scanner.setRange(null, 7, 5);
    scanner.nextToken();
    assertEquals(3, scanner.getTokenLength());
    assertEquals(7, scanner.getTokenOffset());
    scanner.nextToken();
    assertEquals(2, scanner.getTokenLength());
    assertEquals(10, scanner.getTokenOffset());
    assertEquals(Token.EOF, scanner.nextToken());
}

Example 2 with TokenScanner

use of org.eclipse.xtext.ui.editor.syntaxcoloring.TokenScanner in project xtext-eclipse by eclipse.

the class TokenScannerTest method testMidRange.

@Test
public void testMidRange() throws Exception {
    TokenScanner scanner = getTokenScanner(3, 4, 3, 2);
    scanner.setRange(null, 3, 7);
    scanner.nextToken();
    assertEquals(4, scanner.getTokenLength());
    assertEquals(3, scanner.getTokenOffset());
    scanner.nextToken();
    assertEquals(3, scanner.getTokenLength());
    assertEquals(7, scanner.getTokenOffset());
    assertEquals(Token.EOF, scanner.nextToken());
}

Example 3 with TokenScanner

use of org.eclipse.xtext.ui.editor.syntaxcoloring.TokenScanner in project xtext-eclipse by eclipse.

the class TokenScannerTest method getTokenScanner.

public TokenScanner getTokenScanner(int... lengths) throws Exception {
    final List<ILexerTokenRegion> tokens = Lists.newArrayList();
    int offset = 0;
    for (final int length : lengths) {
        final int currentOffset = offset;
        tokens.add(new ILexerTokenRegion() {

            @Override
            public int getLength() {
                return length;
            }

            @Override
            public int getLexerTokenType() {
                return 4711;
            }

            @Override
            public int getOffset() {
                return currentOffset;
            }
        });
        offset += length;
    }
    TokenScanner tokenScanner = new TokenScanner() {

        @Override
        protected Iterable<ILexerTokenRegion> getTokens(IDocument document) {
            return tokens;
        }

        @Override
        protected IToken createToken(ILexerTokenRegion token) {
            return Token.UNDEFINED;
        }
    };
    return tokenScanner;
}

Example 4 with TokenScanner

use of org.eclipse.xtext.ui.editor.syntaxcoloring.TokenScanner in project xtext-eclipse by eclipse.

the class TokenScannerTest method testFullRange.

@Test
public void testFullRange() throws Exception {
    TokenScanner scanner = getTokenScanner(3, 4, 3, 2);
    scanner.setRange(null, 0, 12);
    scanner.nextToken();
    assertEquals(3, scanner.getTokenLength());
    assertEquals(0, scanner.getTokenOffset());
    scanner.nextToken();
    assertEquals(4, scanner.getTokenLength());
    assertEquals(3, scanner.getTokenOffset());
    scanner.nextToken();
    assertEquals(3, scanner.getTokenLength());
    assertEquals(7, scanner.getTokenOffset());
    scanner.nextToken();
    assertEquals(2, scanner.getTokenLength());
    assertEquals(10, scanner.getTokenOffset());
    assertEquals(Token.EOF, scanner.nextToken());
}

Example 5 with TokenScanner

use of org.eclipse.xtext.ui.editor.syntaxcoloring.TokenScanner in project xtext-eclipse by eclipse.

the class TokenScannerTest method testBeginning.

@Test
public void testBeginning() throws Exception {
    TokenScanner scanner = getTokenScanner(3, 4, 3, 2);
    scanner.setRange(null, 0, 7);
    scanner.nextToken();
    assertEquals(3, scanner.getTokenLength());
    assertEquals(0, scanner.getTokenOffset());
    scanner.nextToken();
    assertEquals(4, scanner.getTokenLength());
    assertEquals(3, scanner.getTokenOffset());
    assertEquals(Token.EOF, scanner.nextToken());
}