Examples with ILexerTokenRegion org.eclipse.xtext.ui.editor.model.ILexerTokenRegion used on opensource projects

Example 6 with ILexerTokenRegion

use of org.eclipse.xtext.ui.editor.model.ILexerTokenRegion in project xtext-xtend by eclipse.

the class RichStringAwareTokenScanner method nextToken.

@Override
public IToken nextToken() {
    if (currentRichTextToken != null) {
        if (currentRichTextToken.hasNext())
            return currentRichTextToken.nextToken();
        else
            currentRichTextToken = null;
    }
    if (!getIterator().hasNext())
        return Token.EOF;
    ILexerTokenRegion next = getIterator().next();
    int tokenType = next.getLexerTokenType();
    if (tokenType >= 0 && allTokenTypesAsString[tokenType] != null) {
        currentRichTextToken = createRichTextToken(allTokenTypesAsString[tokenType], next);
        return currentRichTextToken.nextToken();
    } else {
        setCurrentToken(next);
        return createToken(next);
    }
}

Example 7 with ILexerTokenRegion

use of org.eclipse.xtext.ui.editor.model.ILexerTokenRegion in project xtext-eclipse by eclipse.

the class TokenScannerTest method getTokenScanner.

public TokenScanner getTokenScanner(int... lengths) throws Exception {
    final List<ILexerTokenRegion> tokens = Lists.newArrayList();
    int offset = 0;
    for (final int length : lengths) {
        final int currentOffset = offset;
        tokens.add(new ILexerTokenRegion() {

            @Override
            public int getLength() {
                return length;
            }

            @Override
            public int getLexerTokenType() {
                return 4711;
            }

            @Override
            public int getOffset() {
                return currentOffset;
            }
        });
        offset += length;
    }
    TokenScanner tokenScanner = new TokenScanner() {

        @Override
        protected Iterable<ILexerTokenRegion> getTokens(IDocument document) {
            return tokens;
        }

        @Override
        protected IToken createToken(ILexerTokenRegion token) {
            return Token.UNDEFINED;
        }
    };
    return tokenScanner;
}

Example 8 with ILexerTokenRegion

use of org.eclipse.xtext.ui.editor.model.ILexerTokenRegion in project xtext-eclipse by eclipse.

the class LexerTokenAndCharacterPairAwareStrategy method findWord.

@Override
protected IRegion findWord(IDocument document, int offset) {
    Iterable<ILexerTokenRegion> tokenIterable = tokenSourceAccess.getTokens(document, true);
    if (tokenIterable == null) {
        return super.findWord(document, offset);
    }
    Iterator<ILexerTokenRegion> tokenIterator = tokenIterable.iterator();
    ILexerTokenRegion leadingToken = null;
    ILexerTokenRegion trailingToken = null;
    while (tokenIterator.hasNext()) {
        ILexerTokenRegion token = tokenIterator.next();
        if (token.getOffset() <= offset && token.getOffset() + token.getLength() >= offset) {
            if (leadingToken != null)
                trailingToken = token;
            else
                leadingToken = token;
        }
        if (token.getOffset() > offset)
            break;
    }
    if (leadingToken != null) {
        try {
            if (leadingToken.getLength() > 1 && (trailingToken == null || !Character.isLetter(document.getChar(trailingToken.getOffset())))) {
                return new Region(leadingToken.getOffset(), leadingToken.getLength());
            } else if (trailingToken != null) {
                return new Region(trailingToken.getOffset(), trailingToken.getLength());
            }
        } catch (BadLocationException ignore) {
        }
    }
    return super.findWord(document, offset);
}

Example 9 with ILexerTokenRegion

use of org.eclipse.xtext.ui.editor.model.ILexerTokenRegion in project xtext-eclipse by eclipse.

the class DocumentTokenSourceTest method testInsertComment.

@Test
public void testInsertComment() throws Exception {
    document.set("bar 345 grammar : so 'baz & so'");
    IRegion region = tokenSource.getLastDamagedRegion();
    ArrayList<ILexerTokenRegion> list = Lists.newArrayList(tokenSource.getTokenInfos());
    document.replace(8, 7, "/*grammar*/");
    IRegion region2 = tokenSource.getLastDamagedRegion();
    assertTrue(!region.equals(region2));
    ArrayList<ILexerTokenRegion> list2 = Lists.newArrayList(tokenSource.getTokenInfos());
    assertTrue(!list.equals(list2));
}

Example 10 with ILexerTokenRegion

use of org.eclipse.xtext.ui.editor.model.ILexerTokenRegion in project xtext-eclipse by eclipse.

the class PresentationDamager method doComputeIntersection.

private IRegion doComputeIntersection(ITypedRegion partition, DocumentEvent e, IDocument document) {
    Iterable<ILexerTokenRegion> tokensInPartition = Iterables.filter(tokenSourceAccess.getTokens(document, false), Regions.overlaps(partition.getOffset(), partition.getLength()));
    Iterator<ILexerTokenRegion> tokens = Iterables.filter(tokensInPartition, Regions.overlaps(e.getOffset(), e.getLength())).iterator();
    if (tokens.hasNext()) {
        ILexerTokenRegion first = tokens.next();
        ILexerTokenRegion last = first;
        while (tokens.hasNext()) last = tokens.next();
        return new Region(first.getOffset(), last.getOffset() + last.getLength() - first.getOffset());
    }
    // this shouldn't happen, but just in case return the whole partition
    return partition;
}