Examples with ILexerTokenRegion org.eclipse.xtext.ui.editor.model.ILexerTokenRegion used on opensource projects

Example 1 with ILexerTokenRegion

use of org.eclipse.xtext.ui.editor.model.ILexerTokenRegion in project xtext-eclipse by eclipse.

the class DocumentTokenSourceTest method testConcurrentModification.

@Test
public void testConcurrentModification() throws Exception {
    document.set("foo bar");
    Iterator<ILexerTokenRegion> iterator = tokenSource.getTokenInfos().iterator();
    ILexerTokenRegion region = iterator.next();
    region = iterator.next();
    document.set("barfoo");
    region = iterator.next();
    assertEquals(3, region.getLength());
}

Example 2 with ILexerTokenRegion

use of org.eclipse.xtext.ui.editor.model.ILexerTokenRegion in project xtext-eclipse by eclipse.

the class PartitionTokenScannerTest method getPartitionTokenScanner.

public PartitionTokenScanner getPartitionTokenScanner(ILexerTokenRegion... tokenDescs) throws Exception {
    final List<ILexerTokenRegion> tokens = Arrays.asList(tokenDescs);
    int offset = 0;
    for (ILexerTokenRegion token : tokens) {
        ((LexerTokenRegion) token).setOffset(offset);
        offset += token.getLength();
    }
    PartitionTokenScanner tokenScanner = new PartitionTokenScanner() {

        @Override
        protected Iterable<ILexerTokenRegion> getTokens(IDocument document) {
            return tokens;
        }

        @Override
        protected boolean shouldMergePartitions(String contentType) {
            return "3".equals(contentType);
        }
    };
    tokenScanner.setMapper(new ITokenTypeToPartitionTypeMapper() {

        @Override
        public String[] getSupportedPartitionTypes() {
            throw new UnsupportedOperationException();
        }

        @Override
        public String getPartitionType(int antlrTokenType) {
            return "" + antlrTokenType;
        }
    });
    return tokenScanner;
}

Example 3 with ILexerTokenRegion

use of org.eclipse.xtext.ui.editor.model.ILexerTokenRegion in project xtext-eclipse by eclipse.

the class PresentationDamager method computeInterSection.

/**
 * @return the common region of the given partition and the changed region in the DocumentEvent based on the underlying tokens.
 */
protected IRegion computeInterSection(ITypedRegion partition, DocumentEvent e, XtextDocument document) {
    Iterable<ILexerTokenRegion> tokensInPartition = Iterables.filter(document.getTokens(), Regions.overlaps(partition.getOffset(), partition.getLength()));
    Iterator<ILexerTokenRegion> tokens = Iterables.filter(tokensInPartition, Regions.overlaps(e.getOffset(), e.getLength())).iterator();
    if (tokens.hasNext()) {
        ILexerTokenRegion first = tokens.next();
        ILexerTokenRegion last = first;
        while (tokens.hasNext()) last = tokens.next();
        return new Region(first.getOffset(), last.getOffset() + last.getLength() - first.getOffset());
    }
    // this shouldn't happen, but just in case return the whole partition
    return partition;
}

Example 4 with ILexerTokenRegion

use of org.eclipse.xtext.ui.editor.model.ILexerTokenRegion in project statecharts by Yakindu.

the class StyleRanges method getRanges.

public List<StyleRange> getRanges(String expression) {
    final List<StyleRange> ranges = Lists.newArrayList();
    DocumentEvent event = new DocumentEvent();
    event.fDocument = new DummyDocument(expression);
    DocumentTokenSource tokenSource = tokenSourceProvider.get();
    tokenSource.updateStructure(event);
    Iterator<ILexerTokenRegion> iterator = tokenSource.getTokenInfos().iterator();
    while (iterator.hasNext()) {
        ILexerTokenRegion next = iterator.next();
        TextAttribute attribute = attributeProvider.getAttribute(tokenTypeMapper.getId(next.getLexerTokenType()));
        StyleRange range = new StyleRange(next.getOffset(), next.getLength(), attribute.getForeground(), attribute.getBackground());
        range.font = attribute.getFont();
        range.fontStyle = attribute.getStyle();
        ranges.add(range);
    }
    return ranges;
}

Example 5 with ILexerTokenRegion

use of org.eclipse.xtext.ui.editor.model.ILexerTokenRegion in project xtext-xtend by eclipse.

the class RichStringAwareTokenScanner method nextToken.

@Override
public IToken nextToken() {
    if (currentRichTextToken != null) {
        if (currentRichTextToken.hasNext())
            return currentRichTextToken.nextToken();
        else
            currentRichTextToken = null;
    }
    if (!getIterator().hasNext())
        return Token.EOF;
    ILexerTokenRegion next = getIterator().next();
    int tokenType = next.getLexerTokenType();
    if (tokenType >= 0 && allTokenTypesAsString[tokenType] != null) {
        currentRichTextToken = createRichTextToken(allTokenTypesAsString[tokenType], next);
        return currentRichTextToken.nextToken();
    } else {
        setCurrentToken(next);
        return createToken(next);
    }
}