Examples with IToken dyvilx.tools.parsing.token.IToken used on opensource projects

Example 16 with IToken

use of dyvilx.tools.parsing.token.IToken in project Dyvil by Dyvil.

the class ParserManager method split.

@Override
public IToken split(IToken token, int length) {
    final String stringValue = token.stringValue();
    if (length == stringValue.length()) {
        // the second part would be empty, so it stays a single token
        return token;
    }
    final int line = token.startLine();
    final int startIndex = token.startColumn();
    final IToken prev = token.prev();
    final IToken token1 = this.toToken(stringValue.substring(0, length), startIndex, line);
    final IToken token2 = this.toToken(stringValue.substring(length), startIndex + length, line);
    final IToken next = token.next();
    // Re-link the tokens
    prev.setNext(token1);
    token1.setPrev(prev);
    token1.setNext(token2);
    token2.setPrev(token1);
    token2.setNext(next);
    next.setPrev(token2);
    return token1;
}

Example 17 with IToken

use of dyvilx.tools.parsing.token.IToken in project Dyvil by Dyvil.

the class TokenIterator method remove.

@Override
public void remove() {
    IToken prev = this.lastReturned;
    if (prev == null) {
        throw new IllegalStateException();
    }
    IToken next = this.next.next();
    prev.setNext(next);
    next.setPrev(prev);
    this.lastReturned = null;
    this.next = next;
}

Example 18 with IToken

use of dyvilx.tools.parsing.token.IToken in project Dyvil by Dyvil.

the class TryParserManager method split.

@Override
public IToken split(IToken token, int length) {
    final IToken split = super.split(token, length);
    if (split == token) {
        return token;
    }
    if (this.splitTokens == null) {
        this.splitTokens = new ArrayList<>();
    }
    this.splitTokens.add(token);
    return split;
}