Examples with IToken org.eclipse.jface.text.rules.IToken used on opensource projects

Example 36 with IToken

use of org.eclipse.jface.text.rules.IToken in project xtext-eclipse by eclipse.

the class DocumentPartitioner method documentChanged2.

/**
 * {@inheritDoc}
 * <p>
 * May be extended by subclasses.
 * </p>
 *
 * @since 2.2
 */
@Override
public synchronized IRegion documentChanged2(DocumentEvent e) {
    if (!fIsInitialized)
        return null;
    try {
        Assert.isTrue(e.getDocument() == fDocument);
        Position[] category = getPositions();
        int reparseStart = e.getOffset();
        String contentType = null;
        int newLength = e.getText() == null ? 0 : e.getText().length();
        int oldPosition = fDocument.computeIndexInCategory(fPositionCategory, reparseStart);
        if (oldPosition > 0) {
            Position oldPartition = category[oldPosition - 1];
            if (oldPartition.offset + oldPartition.length > reparseStart)
                reparseStart = category[oldPosition - 1].offset;
        }
        fScanner.setPartialRange(fDocument, reparseStart, fDocument.getLength() - reparseStart, /* ignore contentType */
        null, reparseStart);
        int behindLastScannedPosition = reparseStart;
        IToken token = fScanner.nextToken();
        int actualReparseStart = fScanner.getTokenOffset();
        int partitionStart = -1;
        int first = fDocument.computeIndexInCategory(fPositionCategory, actualReparseStart);
        if (first > 0) {
            TypedPosition partition = (TypedPosition) category[first - 1];
            if (partition.includes(actualReparseStart)) {
                partitionStart = partition.getOffset();
                contentType = partition.getType();
                if (e.getOffset() == partition.getOffset() + partition.getLength())
                    actualReparseStart = partitionStart;
                --first;
            } else if (actualReparseStart == e.getOffset() && actualReparseStart == partition.getOffset() + partition.getLength()) {
                partitionStart = partition.getOffset();
                contentType = partition.getType();
                actualReparseStart = partitionStart;
                --first;
            } else {
                partitionStart = partition.getOffset() + partition.getLength();
                contentType = IDocument.DEFAULT_CONTENT_TYPE;
                if (actualReparseStart != partitionStart) {
                    String message = String.format("Detected unexpected state in document partitioner. Please file a bug with the following information attached:%n" + "Document content after the event was applied:%n" + ">>>%s<<<%n" + "Document event: %s", fDocument.get(), String.valueOf(e));
                    log.error(message);
                }
            }
        }
        fScanner.setPartialRange(fDocument, actualReparseStart, fDocument.getLength() - actualReparseStart, contentType, actualReparseStart);
        behindLastScannedPosition = actualReparseStart;
        token = fScanner.nextToken();
        fPositionUpdater.update(e);
        for (int i = first; i < category.length; i++) {
            Position p = category[i];
            if (p.isDeleted) {
                rememberDeletedOffset(e.getOffset());
                break;
            }
        }
        clearPositionCache();
        category = getPositions();
        while (!token.isEOF()) {
            contentType = getTokenContentType(token);
            if (!isSupportedContentType(contentType)) {
                token = fScanner.nextToken();
                continue;
            }
            int start = fScanner.getTokenOffset();
            int length = fScanner.getTokenLength();
            behindLastScannedPosition = start + length;
            int lastScannedPosition = behindLastScannedPosition - 1;
            // remove all affected positions
            while (first < category.length) {
                TypedPosition p = (TypedPosition) category[first];
                if (lastScannedPosition >= p.offset + p.length || (p.overlapsWith(start, length) && (!fDocument.containsPosition(fPositionCategory, start, length) || !contentType.equals(p.getType())))) {
                    rememberRegion(p.offset, p.length);
                    fDocument.removePosition(fPositionCategory, p);
                    ++first;
                } else
                    break;
            }
            // area covered by the event, we are done
            if (fDocument.containsPosition(fPositionCategory, start, length)) {
                if (lastScannedPosition >= e.getOffset() + newLength)
                    return createRegion();
                ++first;
            } else {
                // insert the new type position
                try {
                    fDocument.addPosition(fPositionCategory, new TypedPosition(start, length, contentType));
                    rememberRegion(start, length);
                } catch (BadPositionCategoryException x) {
                } catch (BadLocationException x) {
                }
            }
            token = fScanner.nextToken();
        }
        first = fDocument.computeIndexInCategory(fPositionCategory, behindLastScannedPosition);
        clearPositionCache();
        category = getPositions();
        TypedPosition p;
        while (first < category.length) {
            p = (TypedPosition) category[first++];
            fDocument.removePosition(fPositionCategory, p);
            rememberRegion(p.offset, p.length);
        }
    } catch (BadPositionCategoryException x) {
    // should never happen on connected documents
    } catch (BadLocationException x) {
    } finally {
        clearPositionCache();
    }
    return createRegion();
}

Example 37 with IToken

use of org.eclipse.jface.text.rules.IToken in project xtext-eclipse by eclipse.

the class TokenScanner method createToken.

protected IToken createToken(ILexerTokenRegion currentToken) {
    String id = tokenIdMapper.getId(currentToken.getLexerTokenType());
    Token token = new Token(getAttribute(id));
    return token;
}

Example 38 with IToken

use of org.eclipse.jface.text.rules.IToken in project translationstudio8 by heartsome.

the class InnerTagRule method evaluate.

@SuppressWarnings("unchecked")
@Override
public IToken evaluate(ICharacterScanner scanner) {
    int c = scanner.read();
    if (c != ICharacterScanner.EOF && fDetector.isWordStart((char) c)) {
        if (fColumn == UNDEFINED || (fColumn == scanner.getColumn() - 1)) {
            fBuffer.setLength(0);
            do {
                fBuffer.append((char) c);
                c = scanner.read();
            } while (fDetector.isWordPart((char) c) && c != ICharacterScanner.EOF);
            scanner.unread();
            String buffer = fBuffer.toString();
            IToken token = null;
            Set<String> set = fWords.keySet();
            for (String re : set) {
                if (re != null && buffer.matches(re)) {
                    token = (IToken) fWords.get(re);
                    break;
                }
            }
            if (token != null) {
                // 匹配并得到样式
                return token;
            }
            if (fDefaultToken.isUndefined()) {
                unreadBuffer(scanner);
            }
            return fDefaultToken;
        }
    } else {
    }
    scanner.unread();
    return Token.UNDEFINED;
}

Example 39 with IToken

use of org.eclipse.jface.text.rules.IToken in project translationstudio8 by heartsome.

the class PresentationRepairer method createPresentation.

public void createPresentation(TextPresentation presentation, ITypedRegion region) {
    if (fScanner == null) {
        // will be removed if deprecated constructor will be removed
        addRange(presentation, region.getOffset(), region.getLength(), fDefaultTextStyle);
        return;
    }
    int lastStart = region.getOffset();
    int length = 0;
    boolean firstToken = true;
    IToken lastToken = Token.UNDEFINED;
    TextStyle lastTextStyle = getTokenTextStyle(lastToken);
    fScanner.setRange(fDocument, lastStart, region.getLength());
    while (true) {
        IToken token = fScanner.nextToken();
        if (token.isEOF())
            break;
        TextStyle textStyle = getTokenTextStyle(token);
        if (lastTextStyle != null && lastTextStyle.equals(textStyle)) {
            length += fScanner.getTokenLength();
            firstToken = false;
        } else {
            if (!firstToken)
                addRange(presentation, lastStart, length, lastTextStyle);
            firstToken = false;
            lastToken = token;
            lastTextStyle = textStyle;
            lastStart = fScanner.getTokenOffset();
            length = fScanner.getTokenLength();
        }
    }
    addRange(presentation, lastStart, length, lastTextStyle);
}

Example 40 with IToken

use of org.eclipse.jface.text.rules.IToken in project translationstudio8 by heartsome.

the class TagStyleConfigurator method configure.

public static void configure(TextLayout textLayout) {
    String text = textLayout.getText();
    Document doc = new Document(text);
    ITokenScanner scanner = getRecipeScanner(doc);
    scanner.setRange(doc, 0, doc.getLength());
    IToken token;
    while ((token = scanner.nextToken()) != Token.EOF) {
        int offset = scanner.getTokenOffset();
        int length = scanner.getTokenLength();
        Object data = token.getData();
        if (data != null && data instanceof TextStyle) {
            TextStyle textStyle = (TextStyle) data;
            textLayout.setStyle(textStyle, offset, offset + length - 1);
        }
    }
}