Examples with Token org.develnext.jphp.core.tokenizer.token.Token used on opensource projects

Example 71 with Token

use of org.develnext.jphp.core.tokenizer.token.Token in project jphp by jphp-compiler.

the class SimpleExprGenerator method processString.

protected Token processString(StringExprToken string) {
    if (string.getSegments().isEmpty()) {
        if (string.getQuote() == StringExprToken.Quote.SHELL) {
            return new ShellExecExprToken(string.getMeta(), Arrays.<Token>asList(string));
        }
        return string;
    }
    List<Token> tokens = new ArrayList<Token>();
    int i = 0;
    String value = string.getValue();
    TokenMeta meta = string.getMeta();
    for (StringExprToken.Segment segment : string.getSegments()) {
        String prev = value.substring(i, segment.from);
        if (!prev.isEmpty()) {
            StringExprToken item = new StringExprToken(new TokenMeta(prev, meta.getStartLine() + i, meta.getEndLine(), meta.getStartLine(), meta.getEndLine()), StringExprToken.Quote.SINGLE);
            tokens.add(item);
        }
        String dynamic = value.substring(segment.from, segment.to);
        if (!segment.isVariable)
            dynamic = dynamic.substring(1, dynamic.length() - 1);
        Tokenizer tokenizer = new Tokenizer(dynamic + ";", analyzer.getContext());
        try {
            SyntaxAnalyzer syntaxAnalyzer = new SyntaxAnalyzer(analyzer.getEnvironment(), tokenizer, analyzer.getFunction());
            List<Token> tree = syntaxAnalyzer.getTree();
            analyzer.getScope().addVariables(syntaxAnalyzer.getScope().getVariables());
            assert tree.size() > 0;
            Token item = tree.get(0);
            if (!(item instanceof ExprStmtToken))
                unexpectedToken(item);
            ExprStmtToken expr = (ExprStmtToken) item;
            if (expr.isSingle()) {
                tokens.add(expr.getSingle());
            } else
                tokens.add(expr);
        } catch (ParseException e) {
            TraceInfo oldTrace = e.getTraceInfo();
            e.setTraceInfo(new TraceInfo(analyzer.getContext(), meta.getStartLine() + oldTrace.getStartLine(), meta.getEndLine() + oldTrace.getEndLine(), meta.getStartLine() + oldTrace.getStartLine(), meta.getEndLine() + oldTrace.getEndLine()));
            throw e;
        }
        i = segment.to;
    }
    String prev = value.substring(i);
    if (!prev.isEmpty()) {
        StringExprToken item = new StringExprToken(new TokenMeta(prev, meta.getStartLine() + i, meta.getEndLine(), meta.getStartLine(), meta.getEndLine()), StringExprToken.Quote.SINGLE);
        tokens.add(item);
    }
    if (string.getQuote() == StringExprToken.Quote.SHELL) {
        return new ShellExecExprToken(meta, tokens);
    }
    StringBuilderExprToken result = new StringBuilderExprToken(meta, tokens);
    result.setBinary(string.isBinary());
    return result;
}

Example 72 with Token

use of org.develnext.jphp.core.tokenizer.token.Token in project jphp by jphp-compiler.

the class SimpleExprGenerator method processDynamicAccess.

protected DynamicAccessExprToken processDynamicAccess(Token current, Token next, ListIterator<Token> iterator, BraceExprToken.Kind closedBraceKind, int braceOpened) {
    if (next != null && next.isNamedToken() && !(next instanceof NameToken))
        next = new NameToken(next.getMeta());
    DynamicAccessExprToken result = (DynamicAccessExprToken) current;
    if (next instanceof NameToken || next instanceof VariableExprToken) {
        result.setField((ValueExprToken) next);
        next = iterator.next();
        if (result.getField() instanceof VariableExprToken) {
            if (isOpenedBrace(nextTokenAndPrev(iterator), ARRAY)) {
                Token arr = processArrayToken(next, nextToken(iterator), iterator);
                result.setFieldExpr(new ExprStmtToken(analyzer.getEnvironment(), analyzer.getContext(), result.getField(), arr));
                result.setField(null);
            }
        }
    } else if (isOpenedBrace(next, BLOCK)) {
        ExprStmtToken name = analyzer.generator(ExprGenerator.class).getInBraces(BLOCK, iterator);
        result.setFieldExpr(name);
    }
    if (iterator.hasNext()) {
        next = iterator.next();
        if (next instanceof AssignableOperatorToken) {
            DynamicAccessAssignExprToken dResult = new DynamicAccessAssignExprToken(result);
            dResult.setAssignOperator(next);
            ExprStmtToken value = analyzer.generator(SimpleExprGenerator.class).setCanStartByReference(true).getNextExpression(nextToken(iterator), iterator, BraceExprToken.Kind.ANY);
            dResult.setValue(value);
            return dResult;
        }
        iterator.previous();
    }
    return result;
}

Example 73 with Token

use of org.develnext.jphp.core.tokenizer.token.Token in project jphp by jphp-compiler.

the class SimpleExprGenerator method processIsset.

protected IssetExprToken processIsset(Token previous, Token current, ListIterator<Token> iterator) {
    Token next = nextTokenAndPrev(iterator);
    if (!isOpenedBrace(next, SIMPLE))
        unexpectedToken(next, "(");
    CallExprToken call = processCall(current, nextToken(iterator), iterator);
    for (ExprStmtToken param : call.getParameters()) {
        List<Token> tokens = param.getTokens();
        Token last = tokens.get(tokens.size() - 1);
        Token newToken = null;
        if (last instanceof DynamicAccessExprToken) {
            newToken = new DynamicAccessIssetExprToken((DynamicAccessExprToken) last);
            if (analyzer.getClazz() != null && !"__isset".equals(analyzer.getFunction().getFulledName())) {
                ((DynamicAccessIssetExprToken) newToken).setWithMagic(false);
            }
        } else if (last instanceof VariableExprToken || last instanceof GetVarExprToken) {
        // nop
        } else if (last instanceof StaticAccessExprToken && ((StaticAccessExprToken) last).isGetStaticField()) {
            newToken = new StaticAccessIssetExprToken((StaticAccessExprToken) last);
        } else if (last instanceof ArrayGetExprToken) {
            ArrayGetIssetExprToken el = new ArrayGetIssetExprToken(last.getMeta());
            el.setParameters(((ArrayGetExprToken) last).getParameters());
            newToken = el;
        } else
            unexpectedToken(param.getSingle());
        if (newToken != null) {
            tokens.set(tokens.size() - 1, newToken);
            param.updateAsmExpr(analyzer.getEnvironment(), analyzer.getContext());
        }
    }
    IssetExprToken result = (IssetExprToken) current;
    result.setParameters(call.getParameters());
    return result;
}

Example 74 with Token

use of org.develnext.jphp.core.tokenizer.token.Token in project jphp by jphp-compiler.

the class SimpleExprGenerator method processArrayToken.

protected Token processArrayToken(Token previous, Token current, ListIterator<Token> iterator) {
    BraceExprToken.Kind braceKind = ARRAY;
    Separator separator = Separator.ARRAY;
    if (isOpenedBrace(current, BLOCK)) {
        braceKind = BLOCK;
        separator = Separator.ARRAY_BLOCK;
    }
    VariableExprToken var = null;
    if (previous instanceof VariableExprToken) {
        if (analyzer.getFunction() != null) {
            analyzer.getFunction().variable(var = (VariableExprToken) previous).setArrayAccess(true);
        }
    }
    Token next = nextToken(iterator);
    if (isClosedBrace(next, braceKind)) {
        // !!! allow [] anywhere
        return new ArrayPushExprToken(TokenMeta.of(current, next));
    /*} else
                unexpectedToken(tk);*/
    } else
        iterator.previous();
    ExprStmtToken param;
    List<ExprStmtToken> parameters = new ArrayList<>();
    boolean lastPush = false;
    do {
        Token token = nextToken(iterator);
        if (isClosedBrace(token, ARRAY)) {
            iterator.previous();
            if (iterator.hasPrevious()) {
                iterator.previous();
            }
            lastPush = true;
            break;
        } else {
            param = analyzer.generator(SimpleExprGenerator.class).getNextExpression(token, iterator, separator, braceKind);
        }
        if (param != null) {
            parameters.add(param);
            if (iterator.hasNext()) {
                iterator.next();
                if (iterator.hasNext()) {
                    Token tmp = nextToken(iterator);
                    if (isOpenedBrace(tmp, ARRAY)) {
                        braceKind = ARRAY;
                        separator = Separator.ARRAY;
                        continue;
                    } else if (isOpenedBrace(tmp, BLOCK)) {
                        braceKind = BLOCK;
                        separator = Separator.ARRAY_BLOCK;
                        continue;
                    }
                    iterator.previous();
                }
                break;
            }
        }
    } while (param != null);
    // nextToken(iterator); // skip ]
    ArrayGetExprToken result;
    result = new ArrayGetExprToken(current.getMeta());
    result.setParameters(parameters);
    if (isRef) {
        if (previous instanceof StaticAccessExprToken && !((StaticAccessExprToken) previous).isGetStaticField()) {
            unexpectedToken(current);
        }
        result = new ArrayGetRefExprToken(result);
        ((ArrayGetRefExprToken) result).setShortcut(true);
        if (var != null && analyzer.getFunction() != null) {
            analyzer.getFunction().variable(var).setMutable(true);
        }
    } else if (iterator.hasNext()) {
        next = iterator.next();
        if (next instanceof AssignableOperatorToken || lastPush) {
            if (previous instanceof StaticAccessExprToken && !((StaticAccessExprToken) previous).isGetStaticField()) {
                unexpectedToken(next);
            }
            result = new ArrayGetRefExprToken(result);
            if (var != null && analyzer.getFunction() != null) {
                analyzer.getFunction().variable(var).setMutable(true);
            }
        }
        iterator.previous();
    }
    return result;
}

Example 75 with Token

use of org.develnext.jphp.core.tokenizer.token.Token in project jphp by jphp-compiler.

the class SimpleExprGenerator method getToken.

@SuppressWarnings("unchecked")
public ExprStmtToken getToken(Token current, ListIterator<Token> iterator, Separator separator, BraceExprToken.Kind closedBraceKind, Callback<Boolean, Token> breakCallback) {
    isRef = false;
    List<Token> tokens = new ArrayList<>();
    Token previous = null;
    Token next = iterator.hasNext() ? iterator.next() : null;
    if (next != null)
        iterator.previous();
    int braceOpened = 0;
    boolean needBreak = false;
    do {
        if (breakCallback != null && current != null && breakCallback.call(current)) {
            break;
        }
        if (isOpenedBrace(current, SIMPLE)) {
            boolean isFunc = false;
            if (previous instanceof NameToken && "array".equalsIgnoreCase(previous.getMeta().getWord())) {
                iterator.previous();
                tokens.set(tokens.size() - 1, current = processNewArray(previous, iterator));
            } else {
                if (previous instanceof NameToken || previous instanceof VariableExprToken || previous instanceof ClosureStmtToken || previous instanceof ArrayGetExprToken || previous instanceof CallExprToken)
                    isFunc = true;
                else if (previous instanceof StaticAccessExprToken) {
                    // !((StaticAccessExprToken)previous).isGetStaticField(); TODO check it!
                    isFunc = true;
                } else if (previous instanceof DynamicAccessExprToken) {
                    isFunc = true;
                } else if (isClosedBrace(previous, SIMPLE)) {
                    isFunc = true;
                }
                if (isFunc) {
                    CallExprToken call = processCall(previous, current, iterator);
                    if (call.getName() != null) {
                        current = call;
                        tokens.set(tokens.size() - 1, call);
                    } else {
                        tokens.add(current = new CallOperatorToken(call));
                    }
                } else {
                    if (needBreak)
                        unexpectedToken(current);
                    braceOpened += 1;
                    tokens.add(current);
                }
            }
        } else if (braceOpened > 0 && isClosedBrace(current, SIMPLE)) {
            braceOpened -= 1;
            tokens.add(current);
            if (isOpenedBrace(previous, SIMPLE))
                unexpectedToken(current);
        } else if (isOpenedBrace(current, ARRAY) || isOpenedBrace(current, BLOCK)) {
            if (isTokenClass(previous, NameToken.class, VariableExprToken.class, GetVarExprToken.class, CallExprToken.class, ArrayGetExprToken.class, DynamicAccessExprToken.class, StringExprToken.class, StringBuilderExprToken.class, CallOperatorToken.class, ArrayPushExprToken.class, YieldExprToken.class) || (previous instanceof StaticAccessExprToken) || isClosedBrace(previous, SIMPLE)) {
                // array
                current = processArrayToken(previous, current, iterator);
                if (previous instanceof DynamicAccessExprToken && current instanceof ArrayGetRefExprToken) {
                    tokens.set(tokens.size() - 1, new DynamicAccessGetRefExprToken((DynamicAccessExprToken) previous));
                }
                tokens.add(current);
            } else if (previous instanceof OperatorExprToken || previous == null || isOpenedBrace(previous, SIMPLE) || isOpenedBrace(previous, BLOCK)) {
                int currIndex = iterator.nextIndex() - 1;
                ArrayExprToken arrToken = (ArrayExprToken) processNewArray(current, iterator);
                boolean nextAssign = nextTokenAndPrev(iterator) instanceof AssignExprToken;
                if (nextAssign) {
                    int doneIndex = iterator.nextIndex();
                    int prevCount = doneIndex - currIndex;
                    for (int i = 0; i < prevCount; i++) {
                        current = iterator.previous();
                    }
                    iterator.next();
                    tokens.add(processList(current, iterator, null, closedBraceKind, braceOpened));
                } else {
                    if (arrToken.isListSyntax()) {
                        unexpectedToken(nextToken(iterator), "=");
                    }
                    tokens.add(current = arrToken);
                }
            } else {
                unexpectedToken(current);
            }
        } else if (braceOpened == 0 && isClosedBrace(current, ARRAY)) {
            if (separator == Separator.ARRAY)
                break;
            if (closedBraceKind == ARRAY || closedBraceKind == BraceExprToken.Kind.ANY) {
                // if (tokens.isEmpty())
                iterator.previous();
                break;
            }
            unexpectedToken(current);
        } else if (separator == Separator.ARRAY_BLOCK && braceOpened == 0 && isClosedBrace(current, BLOCK)) {
            break;
        } else if (current instanceof StaticExprToken && nextTokenAndPrev(iterator) instanceof FunctionStmtToken) {
            current = nextToken(iterator);
            current = processClosure(current, next, iterator, true);
            tokens.add(current);
        } else if (current instanceof FunctionStmtToken) {
            current = processClosure(current, next, iterator, false);
            tokens.add(current);
        } else if (current instanceof StaticExprToken && nextTokenAndPrev(iterator) instanceof LambdaStmtToken) {
            current = nextToken(iterator);
            current = processLambda(current, iterator, separator, closedBraceKind, true);
            tokens.add(current);
        } else if (current instanceof LambdaStmtToken) {
            current = processLambda(current, iterator, separator, closedBraceKind, false);
            tokens.add(current);
        } else if (current instanceof ListExprToken && isOpenedBrace(next, SIMPLE)) {
            current = processList(current, iterator, null, closedBraceKind, braceOpened);
            tokens.add(current);
        } else if (current instanceof DieExprToken) {
            processDie(current, next, iterator);
            tokens.add(current);
        } else if (current instanceof EmptyExprToken) {
            processEmpty(current, iterator);
            tokens.add(current);
        } else if (current instanceof IssetExprToken) {
            processIsset(previous, current, iterator);
            tokens.add(current);
        } else if (current instanceof UnsetExprToken) {
            if (isOpenedBrace(previous, SIMPLE) && isClosedBrace(next, SIMPLE)) {
                current = new UnsetCastExprToken(current.getMeta());
                tokens.set(tokens.size() - 1, current);
                iterator.next();
                braceOpened--;
            } else {
                if (previous != null)
                    unexpectedToken(current);
                processUnset(previous, current, iterator);
                tokens.add(current);
                needBreak = true;
            }
        } else if (current instanceof CommaToken) {
            if (separator == Separator.COMMA || separator == Separator.COMMA_OR_SEMICOLON) {
                if (tokens.isEmpty())
                    unexpectedToken(current);
                break;
            } else {
                unexpectedToken(current);
            }
        } else if (current instanceof AsStmtToken) {
            if (separator == Separator.AS)
                break;
            unexpectedToken(current);
        } else if (isClosedBrace(current, closedBraceKind)) {
            iterator.previous();
            break;
        } else if (current instanceof BreakToken) {
            break;
        } else if (current instanceof ColonToken) {
            if (separator == Separator.COLON) {
                /*if (tokens.isEmpty()) see: issues/93
                        unexpectedToken(current);*/
                break;
            }
            unexpectedToken(current);
        } else if (current instanceof SemicolonToken) {
            // TODO refactor!
            if (separator == Separator.SEMICOLON || separator == Separator.COMMA_OR_SEMICOLON) {
                /*if (tokens.isEmpty()) see: issues/94
                        unexpectedToken(current);*/
                break;
            }
            if (separator == Separator.COMMA || closedBraceKind != null || tokens.isEmpty())
                unexpectedToken(current);
            break;
        } else if (current instanceof BraceExprToken) {
            if (closedBraceKind == BraceExprToken.Kind.ANY && isClosedBrace(current)) {
                iterator.previous();
                break;
            }
            unexpectedToken(current);
        } else if (current instanceof ArrayExprToken) {
            if (needBreak)
                unexpectedToken(current);
            tokens.add(current = processNewArray(current, iterator));
        } else if (current instanceof ExprToken) {
            if (needBreak)
                unexpectedToken(current);
            CastExprToken cast = null;
            if (current instanceof NameToken && isOpenedBrace(previous, SIMPLE) && isClosedBrace(next, SIMPLE)) {
                cast = CastExprToken.valueOf(((NameToken) current).getName(), current.getMeta());
                if (cast != null) {
                    current = cast;
                    iterator.next();
                    braceOpened--;
                    tokens.set(tokens.size() - 1, current);
                }
            }
            if (cast == null) {
                Token token = processSimpleToken(current, previous, next, iterator, closedBraceKind, braceOpened, separator);
                if (token != null)
                    current = token;
                tokens.add(current);
            }
        } else
            unexpectedToken(current);
        previous = current;
        if (iterator.hasNext()) {
            current = nextToken(iterator);
            next = iterator.hasNext() ? iterator.next() : null;
            if (next != null)
                iterator.previous();
        } else
            current = null;
        if (current == null)
            nextToken(iterator);
    } while (current != null);
    if (tokens.isEmpty())
        return null;
    if (braceOpened != 0)
        unexpectedToken(iterator.previous());
    ExprStmtToken exprStmtToken = new ExprStmtToken(analyzer.getEnvironment(), analyzer.getContext(), tokens);
    return exprStmtToken;
}