Examples with Token org.develnext.jphp.core.tokenizer.token.Token used on opensource projects

Example 46 with Token

use of org.develnext.jphp.core.tokenizer.token.Token in project jphp by jphp-compiler.

the class TokenizerTest method testComments.

@Test
public void testComments() throws IOException {
    Tokenizer tokenizer = new Tokenizer(new Context("/** FOO BAR \n\r100500 */"));
    Token token = tokenizer.nextToken();
    assertTrue(token instanceof CommentToken);
    assertEquals(CommentToken.Kind.DOCTYPE, ((CommentToken) token).getKind());
    assertEquals("FOO BAR\n\n100500", ((CommentToken) token).getComment());
    // simple
    tokenizer = new Tokenizer(new Context("// foobar \n $x"));
    token = tokenizer.nextToken();
    assertTrue(token instanceof CommentToken);
    assertEquals(CommentToken.Kind.SIMPLE, ((CommentToken) token).getKind());
    assertEquals(" foobar ", ((CommentToken) token).getComment());
    assertTrue(tokenizer.nextToken() instanceof VariableExprToken);
    assertNull(tokenizer.nextToken());
    tokenizer = new Tokenizer(new Context("# // foobar \n $x"));
    token = tokenizer.nextToken();
    assertTrue(token instanceof CommentToken);
    assertEquals(CommentToken.Kind.SIMPLE, ((CommentToken) token).getKind());
    assertEquals(" // foobar ", ((CommentToken) token).getComment());
    assertTrue(tokenizer.nextToken() instanceof VariableExprToken);
    assertNull(tokenizer.nextToken());
    // block
    tokenizer = new Tokenizer(new Context("/* foobar \n */"));
    token = tokenizer.nextToken();
    assertTrue(token instanceof CommentToken);
    assertEquals(CommentToken.Kind.BLOCK, ((CommentToken) token).getKind());
}

Example 47 with Token

use of org.develnext.jphp.core.tokenizer.token.Token in project jphp by jphp-compiler.

the class Generator method checkUnexpectedStart.

protected void checkUnexpectedStart(ListIterator<Token> iterator) {
    if (!iterator.hasPrevious()) {
        iterator.next();
        Token current = iterator.previous();
        unexpectedToken(current);
    }
}

Example 48 with Token

use of org.develnext.jphp.core.tokenizer.token.Token in project jphp by jphp-compiler.

the class Generator method nextTokenAndPrev.

protected Token nextTokenAndPrev(ListIterator<Token> iterator) {
    checkUnexpectedEnd(iterator);
    Token result = iterator.next();
    if (result instanceof CommentToken) {
        return nextTokenAndPrev(iterator);
    }
    iterator.previous();
    return result;
}

Example 49 with Token

use of org.develnext.jphp.core.tokenizer.token.Token in project jphp by jphp-compiler.

the class TokenMeta method of.

public static TokenMeta of(List<? extends Token> tokens) {
    int startPosition = 0, startLine = 0, endPosition = 0, endLine = 0, startIndex = -1, endIndex = -1;
    StringBuilder builder = new StringBuilder();
    int i = 0;
    int size = tokens.size();
    for (Token token : tokens) {
        if (token == null)
            continue;
        if (i == 0) {
            startIndex = token.getMeta().startIndex;
            startPosition = token.getMeta().startPosition;
            startLine = token.getMeta().startLine;
        }
        builder.append(token.getMeta().word);
        if (i == size - 1) {
            endPosition = token.getMeta().endPosition;
            endLine = token.getMeta().endLine;
            endIndex = token.getMeta().endIndex;
        }
        i++;
    }
    TokenMeta meta = new TokenMeta(builder.toString(), startLine, endLine, startPosition, endPosition);
    meta.setStartIndex(startIndex);
    meta.setEndIndex(endIndex);
    return meta;
}

Example 50 with Token

use of org.develnext.jphp.core.tokenizer.token.Token in project jphp by jphp-compiler.

the class Tokenizer method nextToken.

public Token nextToken() {
    boolean init = false;
    char ch = '\0';
    char prev_ch = '\0';
    int startPosition = currentPosition + 1;
    startRelativePosition = relativePosition;
    int startLine = currentLine;
    StringExprToken.Quote string = null;
    CommentToken.Kind comment = null;
    if (codeLength == 0) {
        return null;
    }
    boolean first = true;
    while (currentPosition < codeLength) {
        currentPosition++;
        relativePosition++;
        if (currentPosition == codeLength) {
            break;
        }
        ch = code.charAt(currentPosition);
        if (currentPosition > 0 && init) {
            prev_ch = code.charAt(currentPosition - 1);
        }
        checkNewLine(ch);
        if (rawMode) {
            if (TokenizeGrammarUtils.isOpenTag(prev_ch, ch)) {
                TokenMeta meta = new TokenMeta(code.substring(startPosition, currentPosition - 1), startLine, currentLine, startRelativePosition, relativePosition);
                rawMode = false;
                startLine = currentLine;
                startRelativePosition = relativePosition;
                EchoRawToken token = buildToken(EchoRawToken.class, meta);
                if (codeLength >= currentPosition + 4 && code.substring(currentPosition + 1, currentPosition + 4).equals("php")) {
                    relativePosition += 4;
                    currentPosition += 3;
                    token.setShort(false);
                } else {
                    token.setShort(true);
                }
                return token;
            } else {
                init = true;
                first = true;
                continue;
            }
        }
        if (ch == '=' && prevToken != null && prevToken instanceof EchoRawToken && ((EchoRawToken) prevToken).isShort()) {
            return buildToken(OpenEchoTagToken.class, buildMeta(startPosition, startLine));
        }
        if (first && (!init || prevToken == null)) {
            // numbers: integers, doubles, hex
            if (Character.isDigit(ch) || (ch == '.' && prevToken == null && currentPosition + 1 < codeLength && Character.isDigit(code.charAt(currentPosition + 1)))) {
                return readNumber(startPosition, startLine);
            }
            // comments
            comment = CommentToken.Kind.isComment(ch, prev_ch);
            if (comment != null) {
                return readComment(comment, startPosition, startLine);
            }
            // strings, herdoc, etc.
            string = TokenizeGrammarUtils.isQuote(ch);
            if (string != null) {
                return readString(string, startPosition, startLine);
            }
        }
        init = true;
        first = false;
        if (TokenizeGrammarUtils.isDelimiter(ch)) {
            if (startPosition == currentPosition && TokenizeGrammarUtils.isSpace(ch)) {
                startPosition = currentPosition + 1;
                startLine = currentLine;
                startRelativePosition = relativePosition;
                prevToken = null;
                first = true;
                continue;
            }
            if (startPosition == currentPosition) {
                Token token = tryNextToken();
                if (token instanceof BreakToken) {
                    rawMode = true;
                }
                if (token instanceof CommentToken) {
                    comment = ((CommentToken) token).getKind();
                    return readComment(comment, startPosition, startLine);
                // continue;
                }
                if (token instanceof StringStartDocToken) {
                    string = StringExprToken.Quote.DOC;
                    return readString(string, startPosition, startLine);
                }
                if (token != null) {
                    return token;
                }
            }
            break;
        } else if (TokenizeGrammarUtils.isVariableChar(ch)) {
            if (TokenizeGrammarUtils.isVariableChar(prev_ch)) {
                currentPosition -= 1;
                break;
            }
        }
    }
    TokenMeta meta = buildMeta(startPosition, startLine);
    if (currentPosition != startPosition && TokenizeGrammarUtils.isDelimiter(ch)) {
        checkNewLine(ch, true);
        currentPosition -= 1;
        relativePosition -= 1;
    }
    if (meta == null)
        return null;
    // currentPosition -= 1;
    Class<? extends Token> tokenClazz = rawMode ? EchoRawToken.class : tokenFinder.find(meta);
    if (tokenClazz == null) {
        return prevToken = new Token(meta, TokenType.T_J_CUSTOM);
    } else {
        return buildToken(tokenClazz, meta);
    }
}