Examples with StringStartDocToken org.develnext.jphp.core.tokenizer.token.StringStartDocToken used on opensource projects

Example 1 with StringStartDocToken

use of org.develnext.jphp.core.tokenizer.token.StringStartDocToken in project jphp by jphp-compiler.

the class Tokenizer method nextToken.

public Token nextToken() {
    boolean init = false;
    char ch = '\0';
    char prev_ch = '\0';
    int startPosition = currentPosition + 1;
    startRelativePosition = relativePosition;
    int startLine = currentLine;
    StringExprToken.Quote string = null;
    CommentToken.Kind comment = null;
    if (codeLength == 0) {
        return null;
    }
    boolean first = true;
    while (currentPosition < codeLength) {
        currentPosition++;
        relativePosition++;
        if (currentPosition == codeLength) {
            break;
        }
        ch = code.charAt(currentPosition);
        if (currentPosition > 0 && init) {
            prev_ch = code.charAt(currentPosition - 1);
        }
        checkNewLine(ch);
        if (rawMode) {
            if (TokenizeGrammarUtils.isOpenTag(prev_ch, ch)) {
                TokenMeta meta = new TokenMeta(code.substring(startPosition, currentPosition - 1), startLine, currentLine, startRelativePosition, relativePosition);
                rawMode = false;
                startLine = currentLine;
                startRelativePosition = relativePosition;
                EchoRawToken token = buildToken(EchoRawToken.class, meta);
                if (codeLength >= currentPosition + 4 && code.substring(currentPosition + 1, currentPosition + 4).equals("php")) {
                    relativePosition += 4;
                    currentPosition += 3;
                    token.setShort(false);
                } else {
                    token.setShort(true);
                }
                return token;
            } else {
                init = true;
                first = true;
                continue;
            }
        }
        if (ch == '=' && prevToken != null && prevToken instanceof EchoRawToken && ((EchoRawToken) prevToken).isShort()) {
            return buildToken(OpenEchoTagToken.class, buildMeta(startPosition, startLine));
        }
        if (first && (!init || prevToken == null)) {
            // numbers: integers, doubles, hex
            if (Character.isDigit(ch) || (ch == '.' && prevToken == null && currentPosition + 1 < codeLength && Character.isDigit(code.charAt(currentPosition + 1)))) {
                return readNumber(startPosition, startLine);
            }
            // comments
            comment = CommentToken.Kind.isComment(ch, prev_ch);
            if (comment != null) {
                return readComment(comment, startPosition, startLine);
            }
            // strings, herdoc, etc.
            string = TokenizeGrammarUtils.isQuote(ch);
            if (string != null) {
                return readString(string, startPosition, startLine);
            }
        }
        init = true;
        first = false;
        if (TokenizeGrammarUtils.isDelimiter(ch)) {
            if (startPosition == currentPosition && TokenizeGrammarUtils.isSpace(ch)) {
                startPosition = currentPosition + 1;
                startLine = currentLine;
                startRelativePosition = relativePosition;
                prevToken = null;
                first = true;
                continue;
            }
            if (startPosition == currentPosition) {
                Token token = tryNextToken();
                if (token instanceof BreakToken) {
                    rawMode = true;
                }
                if (token instanceof CommentToken) {
                    comment = ((CommentToken) token).getKind();
                    return readComment(comment, startPosition, startLine);
                // continue;
                }
                if (token instanceof StringStartDocToken) {
                    string = StringExprToken.Quote.DOC;
                    return readString(string, startPosition, startLine);
                }
                if (token != null) {
                    return token;
                }
            }
            break;
        } else if (TokenizeGrammarUtils.isVariableChar(ch)) {
            if (TokenizeGrammarUtils.isVariableChar(prev_ch)) {
                currentPosition -= 1;
                break;
            }
        }
    }
    TokenMeta meta = buildMeta(startPosition, startLine);
    if (currentPosition != startPosition && TokenizeGrammarUtils.isDelimiter(ch)) {
        checkNewLine(ch, true);
        currentPosition -= 1;
        relativePosition -= 1;
    }
    if (meta == null)
        return null;
    // currentPosition -= 1;
    Class<? extends Token> tokenClazz = rawMode ? EchoRawToken.class : tokenFinder.find(meta);
    if (tokenClazz == null) {
        return prevToken = new Token(meta, TokenType.T_J_CUSTOM);
    } else {
        return buildToken(tokenClazz, meta);
    }
}