Examples with BreakToken org.develnext.jphp.core.tokenizer.token.BreakToken used on opensource projects

Example 1 with BreakToken

use of org.develnext.jphp.core.tokenizer.token.BreakToken in project jphp by jphp-compiler.

the class Tokenizer method nextToken.

public Token nextToken() {
    boolean init = false;
    char ch = '\0';
    char prev_ch = '\0';
    int startPosition = currentPosition + 1;
    startRelativePosition = relativePosition;
    int startLine = currentLine;
    StringExprToken.Quote string = null;
    CommentToken.Kind comment = null;
    if (codeLength == 0) {
        return null;
    }
    boolean first = true;
    while (currentPosition < codeLength) {
        currentPosition++;
        relativePosition++;
        if (currentPosition == codeLength) {
            break;
        }
        ch = code.charAt(currentPosition);
        if (currentPosition > 0 && init) {
            prev_ch = code.charAt(currentPosition - 1);
        }
        checkNewLine(ch);
        if (rawMode) {
            if (TokenizeGrammarUtils.isOpenTag(prev_ch, ch)) {
                TokenMeta meta = new TokenMeta(code.substring(startPosition, currentPosition - 1), startLine, currentLine, startRelativePosition, relativePosition);
                rawMode = false;
                startLine = currentLine;
                startRelativePosition = relativePosition;
                EchoRawToken token = buildToken(EchoRawToken.class, meta);
                if (codeLength >= currentPosition + 4 && code.substring(currentPosition + 1, currentPosition + 4).equals("php")) {
                    relativePosition += 4;
                    currentPosition += 3;
                    token.setShort(false);
                } else {
                    token.setShort(true);
                }
                return token;
            } else {
                init = true;
                first = true;
                continue;
            }
        }
        if (ch == '=' && prevToken != null && prevToken instanceof EchoRawToken && ((EchoRawToken) prevToken).isShort()) {
            return buildToken(OpenEchoTagToken.class, buildMeta(startPosition, startLine));
        }
        if (first && (!init || prevToken == null)) {
            // numbers: integers, doubles, hex
            if (Character.isDigit(ch) || (ch == '.' && prevToken == null && currentPosition + 1 < codeLength && Character.isDigit(code.charAt(currentPosition + 1)))) {
                return readNumber(startPosition, startLine);
            }
            // comments
            comment = CommentToken.Kind.isComment(ch, prev_ch);
            if (comment != null) {
                return readComment(comment, startPosition, startLine);
            }
            // strings, herdoc, etc.
            string = TokenizeGrammarUtils.isQuote(ch);
            if (string != null) {
                return readString(string, startPosition, startLine);
            }
        }
        init = true;
        first = false;
        if (TokenizeGrammarUtils.isDelimiter(ch)) {
            if (startPosition == currentPosition && TokenizeGrammarUtils.isSpace(ch)) {
                startPosition = currentPosition + 1;
                startLine = currentLine;
                startRelativePosition = relativePosition;
                prevToken = null;
                first = true;
                continue;
            }
            if (startPosition == currentPosition) {
                Token token = tryNextToken();
                if (token instanceof BreakToken) {
                    rawMode = true;
                }
                if (token instanceof CommentToken) {
                    comment = ((CommentToken) token).getKind();
                    return readComment(comment, startPosition, startLine);
                // continue;
                }
                if (token instanceof StringStartDocToken) {
                    string = StringExprToken.Quote.DOC;
                    return readString(string, startPosition, startLine);
                }
                if (token != null) {
                    return token;
                }
            }
            break;
        } else if (TokenizeGrammarUtils.isVariableChar(ch)) {
            if (TokenizeGrammarUtils.isVariableChar(prev_ch)) {
                currentPosition -= 1;
                break;
            }
        }
    }
    TokenMeta meta = buildMeta(startPosition, startLine);
    if (currentPosition != startPosition && TokenizeGrammarUtils.isDelimiter(ch)) {
        checkNewLine(ch, true);
        currentPosition -= 1;
        relativePosition -= 1;
    }
    if (meta == null)
        return null;
    // currentPosition -= 1;
    Class<? extends Token> tokenClazz = rawMode ? EchoRawToken.class : tokenFinder.find(meta);
    if (tokenClazz == null) {
        return prevToken = new Token(meta, TokenType.T_J_CUSTOM);
    } else {
        return buildToken(tokenClazz, meta);
    }
}

Example 2 with BreakToken

use of org.develnext.jphp.core.tokenizer.token.BreakToken in project jphp by jphp-compiler.

the class SimpleExprGenerator method getToken.

@SuppressWarnings("unchecked")
public ExprStmtToken getToken(Token current, ListIterator<Token> iterator, Separator separator, BraceExprToken.Kind closedBraceKind, Callback<Boolean, Token> breakCallback) {
    isRef = false;
    List<Token> tokens = new ArrayList<>();
    Token previous = null;
    Token next = iterator.hasNext() ? iterator.next() : null;
    if (next != null)
        iterator.previous();
    int braceOpened = 0;
    boolean needBreak = false;
    do {
        if (breakCallback != null && current != null && breakCallback.call(current)) {
            break;
        }
        if (isOpenedBrace(current, SIMPLE)) {
            boolean isFunc = false;
            if (previous instanceof NameToken && "array".equalsIgnoreCase(previous.getMeta().getWord())) {
                iterator.previous();
                tokens.set(tokens.size() - 1, current = processNewArray(previous, iterator));
            } else {
                if (previous instanceof NameToken || previous instanceof VariableExprToken || previous instanceof ClosureStmtToken || previous instanceof ArrayGetExprToken || previous instanceof CallExprToken)
                    isFunc = true;
                else if (previous instanceof StaticAccessExprToken) {
                    // !((StaticAccessExprToken)previous).isGetStaticField(); TODO check it!
                    isFunc = true;
                } else if (previous instanceof DynamicAccessExprToken) {
                    isFunc = true;
                } else if (isClosedBrace(previous, SIMPLE)) {
                    isFunc = true;
                }
                if (isFunc) {
                    CallExprToken call = processCall(previous, current, iterator);
                    if (call.getName() != null) {
                        current = call;
                        tokens.set(tokens.size() - 1, call);
                    } else {
                        tokens.add(current = new CallOperatorToken(call));
                    }
                } else {
                    if (needBreak)
                        unexpectedToken(current);
                    braceOpened += 1;
                    tokens.add(current);
                }
            }
        } else if (braceOpened > 0 && isClosedBrace(current, SIMPLE)) {
            braceOpened -= 1;
            tokens.add(current);
            if (isOpenedBrace(previous, SIMPLE))
                unexpectedToken(current);
        } else if (isOpenedBrace(current, ARRAY) || isOpenedBrace(current, BLOCK)) {
            if (isTokenClass(previous, NameToken.class, VariableExprToken.class, GetVarExprToken.class, CallExprToken.class, ArrayGetExprToken.class, DynamicAccessExprToken.class, StringExprToken.class, StringBuilderExprToken.class, CallOperatorToken.class, ArrayPushExprToken.class, YieldExprToken.class) || (previous instanceof StaticAccessExprToken) || isClosedBrace(previous, SIMPLE)) {
                // array
                current = processArrayToken(previous, current, iterator);
                if (previous instanceof DynamicAccessExprToken && current instanceof ArrayGetRefExprToken) {
                    tokens.set(tokens.size() - 1, new DynamicAccessGetRefExprToken((DynamicAccessExprToken) previous));
                }
                tokens.add(current);
            } else if (previous instanceof OperatorExprToken || previous == null || isOpenedBrace(previous, SIMPLE) || isOpenedBrace(previous, BLOCK)) {
                int currIndex = iterator.nextIndex() - 1;
                ArrayExprToken arrToken = (ArrayExprToken) processNewArray(current, iterator);
                boolean nextAssign = nextTokenAndPrev(iterator) instanceof AssignExprToken;
                if (nextAssign) {
                    int doneIndex = iterator.nextIndex();
                    int prevCount = doneIndex - currIndex;
                    for (int i = 0; i < prevCount; i++) {
                        current = iterator.previous();
                    }
                    iterator.next();
                    tokens.add(processList(current, iterator, null, closedBraceKind, braceOpened));
                } else {
                    if (arrToken.isListSyntax()) {
                        unexpectedToken(nextToken(iterator), "=");
                    }
                    tokens.add(current = arrToken);
                }
            } else {
                unexpectedToken(current);
            }
        } else if (braceOpened == 0 && isClosedBrace(current, ARRAY)) {
            if (separator == Separator.ARRAY)
                break;
            if (closedBraceKind == ARRAY || closedBraceKind == BraceExprToken.Kind.ANY) {
                // if (tokens.isEmpty())
                iterator.previous();
                break;
            }
            unexpectedToken(current);
        } else if (separator == Separator.ARRAY_BLOCK && braceOpened == 0 && isClosedBrace(current, BLOCK)) {
            break;
        } else if (current instanceof StaticExprToken && nextTokenAndPrev(iterator) instanceof FunctionStmtToken) {
            current = nextToken(iterator);
            current = processClosure(current, next, iterator, true);
            tokens.add(current);
        } else if (current instanceof FunctionStmtToken) {
            current = processClosure(current, next, iterator, false);
            tokens.add(current);
        } else if (current instanceof StaticExprToken && nextTokenAndPrev(iterator) instanceof LambdaStmtToken) {
            current = nextToken(iterator);
            current = processLambda(current, iterator, separator, closedBraceKind, true);
            tokens.add(current);
        } else if (current instanceof LambdaStmtToken) {
            current = processLambda(current, iterator, separator, closedBraceKind, false);
            tokens.add(current);
        } else if (current instanceof ListExprToken && isOpenedBrace(next, SIMPLE)) {
            current = processList(current, iterator, null, closedBraceKind, braceOpened);
            tokens.add(current);
        } else if (current instanceof DieExprToken) {
            processDie(current, next, iterator);
            tokens.add(current);
        } else if (current instanceof EmptyExprToken) {
            processEmpty(current, iterator);
            tokens.add(current);
        } else if (current instanceof IssetExprToken) {
            processIsset(previous, current, iterator);
            tokens.add(current);
        } else if (current instanceof UnsetExprToken) {
            if (isOpenedBrace(previous, SIMPLE) && isClosedBrace(next, SIMPLE)) {
                current = new UnsetCastExprToken(current.getMeta());
                tokens.set(tokens.size() - 1, current);
                iterator.next();
                braceOpened--;
            } else {
                if (previous != null)
                    unexpectedToken(current);
                processUnset(previous, current, iterator);
                tokens.add(current);
                needBreak = true;
            }
        } else if (current instanceof CommaToken) {
            if (separator == Separator.COMMA || separator == Separator.COMMA_OR_SEMICOLON) {
                if (tokens.isEmpty())
                    unexpectedToken(current);
                break;
            } else {
                unexpectedToken(current);
            }
        } else if (current instanceof AsStmtToken) {
            if (separator == Separator.AS)
                break;
            unexpectedToken(current);
        } else if (isClosedBrace(current, closedBraceKind)) {
            iterator.previous();
            break;
        } else if (current instanceof BreakToken) {
            break;
        } else if (current instanceof ColonToken) {
            if (separator == Separator.COLON) {
                /*if (tokens.isEmpty()) see: issues/93
                        unexpectedToken(current);*/
                break;
            }
            unexpectedToken(current);
        } else if (current instanceof SemicolonToken) {
            // TODO refactor!
            if (separator == Separator.SEMICOLON || separator == Separator.COMMA_OR_SEMICOLON) {
                /*if (tokens.isEmpty()) see: issues/94
                        unexpectedToken(current);*/
                break;
            }
            if (separator == Separator.COMMA || closedBraceKind != null || tokens.isEmpty())
                unexpectedToken(current);
            break;
        } else if (current instanceof BraceExprToken) {
            if (closedBraceKind == BraceExprToken.Kind.ANY && isClosedBrace(current)) {
                iterator.previous();
                break;
            }
            unexpectedToken(current);
        } else if (current instanceof ArrayExprToken) {
            if (needBreak)
                unexpectedToken(current);
            tokens.add(current = processNewArray(current, iterator));
        } else if (current instanceof ExprToken) {
            if (needBreak)
                unexpectedToken(current);
            CastExprToken cast = null;
            if (current instanceof NameToken && isOpenedBrace(previous, SIMPLE) && isClosedBrace(next, SIMPLE)) {
                cast = CastExprToken.valueOf(((NameToken) current).getName(), current.getMeta());
                if (cast != null) {
                    current = cast;
                    iterator.next();
                    braceOpened--;
                    tokens.set(tokens.size() - 1, current);
                }
            }
            if (cast == null) {
                Token token = processSimpleToken(current, previous, next, iterator, closedBraceKind, braceOpened, separator);
                if (token != null)
                    current = token;
                tokens.add(current);
            }
        } else
            unexpectedToken(current);
        previous = current;
        if (iterator.hasNext()) {
            current = nextToken(iterator);
            next = iterator.hasNext() ? iterator.next() : null;
            if (next != null)
                iterator.previous();
        } else
            current = null;
        if (current == null)
            nextToken(iterator);
    } while (current != null);
    if (tokens.isEmpty())
        return null;
    if (braceOpened != 0)
        unexpectedToken(iterator.previous());
    ExprStmtToken exprStmtToken = new ExprStmtToken(analyzer.getEnvironment(), analyzer.getContext(), tokens);
    return exprStmtToken;
}