Examples with Token org.develnext.jphp.core.tokenizer.token.Token used on opensource projects

Example 16 with Token

use of org.develnext.jphp.core.tokenizer.token.Token in project jphp by jphp-compiler.

the class UseGenerator method parseBody.

public void parseBody(NamespaceUseStmtToken use, Token current, ListIterator<Token> iterator, FulledNameToken prefix, NamespaceUseStmtToken.UseType prefixUseType) {
    boolean first = true;
    NamespaceUseStmtToken.UseType useType = prefixUseType;
    Environment environment = this.analyzer.getEnvironment();
    PackageManager packageManager = null;
    if (environment != null) {
        packageManager = environment.getPackageManager();
    }
    do {
        Token next = nextToken(iterator);
        if (next instanceof FunctionStmtToken) {
            if ((!first && prefix == null) || (prefixUseType != CLASS)) {
                unexpectedToken(next);
            }
            useType = FUNCTION;
            next = nextToken(iterator);
        } else if (next instanceof ConstStmtToken) {
            if ((!first && prefix == null) || (prefixUseType != CLASS)) {
                unexpectedToken(next);
            }
            useType = CONSTANT;
            next = nextToken(iterator);
        }
        use.setUseType(useType);
        if (prefix != null && next instanceof FulledNameToken && next.getMeta().getWord().startsWith("\\")) {
            unexpectedToken(new BackslashExprToken(TokenMeta.of("\\", next)), "identifier or function or const", false);
        }
        FulledNameToken name = analyzer.generator(NameGenerator.class).getToken(next, iterator);
        if (name == null) {
            unexpectedToken(iterator.previous());
            return;
        }
        if (prefix == null) {
            use.setName(name);
        } else {
            ArrayList<NameToken> nameTokens = new ArrayList<>(prefix.getNames());
            nameTokens.addAll(name.getNames());
            use.setName(new FulledNameToken(name.getMeta(), nameTokens));
        }
        Token token = nextToken(iterator);
        if (token instanceof AsStmtToken) {
            token = nextToken(iterator);
            if (token instanceof NameToken) {
                use.setAs((NameToken) token);
                token = nextToken(iterator);
            } else
                unexpectedToken(token);
        } else if (isOpenedBrace(token, BraceExprToken.Kind.BLOCK)) {
            if (prefix == null) {
                parseBody(use, current, iterator, name, useType);
                return;
            }
        } else if (token instanceof BackslashExprToken) {
            next = nextToken(iterator);
            if (isOpenedBrace(next, BraceExprToken.Kind.BLOCK) && prefix == null) {
                parseBody(use, current, iterator, name, useType);
                return;
            }
        }
        NamespaceStmtToken namespace = analyzer.getNamespace();
        if (analyzer.getEnvironment() != null && analyzer.getEnvironment().scope.getLangMode() == LangMode.MODERN) {
            if (packageManager != null && use.isPackageImport()) {
                Package aPackage = packageManager.tryFind(use.getName().toName(), use.toTraceInfo(analyzer.getContext()));
                if (aPackage != null) {
                    for (String cls : aPackage.getClasses()) {
                        FulledNameToken nameToken = FulledNameToken.valueOf(StringUtils.split(cls, Information.NAMESPACE_SEP_CHAR));
                        NamespaceUseStmtToken useStmtToken = new NamespaceUseStmtToken(TokenMeta.of(cls, use));
                        useStmtToken.setName(nameToken);
                        useStmtToken.setUseType(NamespaceUseStmtToken.UseType.CLASS);
                        namespace.getUses().add(useStmtToken);
                    }
                    for (String s : aPackage.getFunctions()) {
                        FulledNameToken nameToken = FulledNameToken.valueOf(StringUtils.split(s, Information.NAMESPACE_SEP_CHAR));
                        NamespaceUseStmtToken useStmtToken = new NamespaceUseStmtToken(TokenMeta.of(s, use));
                        useStmtToken.setName(nameToken);
                        useStmtToken.setUseType(NamespaceUseStmtToken.UseType.FUNCTION);
                        namespace.getUses().add(useStmtToken);
                    }
                    for (String s : aPackage.getConstants()) {
                        FulledNameToken nameToken = FulledNameToken.valueOf(StringUtils.split(s, Information.NAMESPACE_SEP_CHAR));
                        NamespaceUseStmtToken useStmtToken = new NamespaceUseStmtToken(TokenMeta.of(s, use));
                        useStmtToken.setName(nameToken);
                        useStmtToken.setUseType(NamespaceUseStmtToken.UseType.CONSTANT);
                        namespace.getUses().add(useStmtToken);
                    }
                } else {
                    namespace.getUses().add(use);
                }
            } else {
                namespace.getUses().add(use);
            }
        } else {
            namespace.getUses().add(use);
        }
        if (token instanceof CommaToken) {
            use = new NamespaceUseStmtToken(current.getMeta());
        } else if (!(token instanceof SemicolonToken)) {
            if (prefix != null && isClosedBrace(token, BraceExprToken.Kind.BLOCK)) {
                nextAndExpected(iterator, SemicolonToken.class);
                break;
            }
            unexpectedToken(token);
        } else
            break;
        first = false;
    } while (true);
}

Example 17 with Token

use of org.develnext.jphp.core.tokenizer.token.Token in project jphp by jphp-compiler.

the class SimpleExprGenerator method processNewArray.

protected Token processNewArray(Token current, ListIterator<Token> iterator) {
    ArrayExprToken result = new ArrayExprToken(current.getMeta());
    List<ExprStmtToken> parameters = new ArrayList<ExprStmtToken>();
    Token next;
    BraceExprToken.Kind braceKind;
    if (isOpenedBrace(current, ARRAY)) {
        next = current;
        braceKind = ARRAY;
        result.setShortSyntax(true);
    } else {
        next = nextToken(iterator);
        if (!isOpenedBrace(next, SIMPLE)) {
            unexpectedToken(next, "(");
        }
        braceKind = SIMPLE;
        result.setShortSyntax(false);
    }
    do {
        SimpleExprGenerator generator = analyzer.generator(SimpleExprGenerator.class);
        generator.setCanStartByReference(true);
        Token nextToken = nextToken(iterator);
        if (nextToken instanceof CommaToken && result.isShortSyntax()) {
            result.setListSyntax(true);
            parameters.add(null);
            continue;
        }
        if (nextToken instanceof ArgumentUnpackExprToken) {
        }
        ExprStmtToken argument = generator.getToken(nextToken, iterator, Separator.COMMA, braceKind);
        if (argument == null)
            break;
        parameters.add(argument);
    } while (true);
    // skip )
    nextToken(iterator);
    result.setParameters(parameters);
    return result;
}

Example 18 with Token

use of org.develnext.jphp.core.tokenizer.token.Token in project jphp by jphp-compiler.

the class SimpleExprGenerator method processNew.

protected Token processNew(Token current, BraceExprToken.Kind closedBrace, int braceOpened, ListIterator<Token> iterator) {
    NewExprToken result = (NewExprToken) current;
    Token next = nextToken(iterator);
    if (!isTokenClass(next, StaticExprToken.class, ParentExprToken.class, SelfExprToken.class, ClassStmtToken.class)) {
        next = makeSensitive(next);
    }
    boolean skipArgs = false;
    if (next instanceof ClassStmtToken) {
        ((ClassStmtToken) next).setAnonymous(true);
        ClassStmtToken token = analyzer.generator(ClassGenerator.class).getToken(next, iterator);
        FulledNameToken name = new FulledNameToken(TokenMeta.of(token.getFulledName(), token));
        result.setName(name);
        result.setParameters(token.getParameters());
        analyzer.registerClass(token);
        skipArgs = true;
    } else if (next instanceof NameToken) {
        FulledNameToken nameToken = analyzer.getRealName((NameToken) next);
        result.setName(nameToken);
        iterator.previous();
        iterator.previous();
        if (iterator.hasPrevious()) {
            Token previous = iterator.previous();
            if (previous instanceof AssignExprToken) {
                if (iterator.hasPrevious()) {
                    previous = iterator.previous();
                    if (previous instanceof VariableExprToken) {
                        analyzer.getScope().typeInfoOf(previous).addType(nameToken.getName());
                    }
                    iterator.next();
                }
            }
            iterator.next();
        }
        iterator.next();
        iterator.next();
    } else if (next instanceof VariableExprToken) {
        result.setName(processNewExpr(next, closedBrace, braceOpened, iterator, true));
    } else if (next instanceof StaticExprToken) {
        Scope scope = analyzer.getScope();
        scope.setStaticExists(true);
        result.setName((StaticExprToken) next);
    } else if (next instanceof SelfExprToken) {
        if (analyzer.getClazz() == null) {
            result.setName((SelfExprToken) next);
        } else {
            if (analyzer.getClazz().isTrait()) {
                result.setName((SelfExprToken) next);
            } else {
                result.setName(new FulledNameToken(next.getMeta(), new ArrayList<Token>() {

                    {
                        if (analyzer.getClazz().getNamespace().getName() != null)
                            addAll(analyzer.getClazz().getNamespace().getName().getNames());
                        add(analyzer.getClazz().getName());
                    }
                }));
            }
        }
    } else
        unexpectedToken(next);
    if (!skipArgs) {
        next = nextToken(iterator);
        if (isOpenedBrace(next, SIMPLE)) {
            ExprStmtToken param;
            List<ExprStmtToken> parameters = new ArrayList<ExprStmtToken>();
            do {
                param = analyzer.generator(SimpleExprGenerator.class).getToken(nextToken(iterator), iterator, true, SIMPLE);
                if (param != null)
                    parameters.add(param);
            } while (param != null);
            nextToken(iterator);
            result.setParameters(parameters);
        } else {
            result.setParameters(new ArrayList<ExprStmtToken>());
            iterator.previous();
        }
    }
    if (analyzer.getFunction() != null) {
        analyzer.getFunction().setCallsExist(true);
    }
    return result;
}

Example 19 with Token

use of org.develnext.jphp.core.tokenizer.token.Token in project jphp by jphp-compiler.

the class SimpleExprGenerator method processNewExpr.

protected ExprStmtToken processNewExpr(Token next, BraceExprToken.Kind closedBrace, int braceOpened, ListIterator<Token> iterator, boolean first) {
    List<Token> tmp = new ArrayList<Token>();
    if (first) {
        if (next instanceof VariableExprToken) {
            analyzer.getScope().addVariable((VariableExprToken) next);
            if (analyzer.getFunction() != null) {
                analyzer.getFunction().setVarsExists(true);
                analyzer.getFunction().variable((VariableExprToken) next).setUsed(true);
            }
        }
        tmp.add(next);
    }
    Token previous = next;
    Token token = nextToken(iterator);
    if (isOpenedBrace(token, ARRAY) || isOpenedBrace(token, BLOCK)) {
        tmp.add(processArrayToken(previous, token, iterator));
        if (iterator.hasNext()) {
            if (nextTokenAndPrev(iterator) instanceof DynamicAccessExprToken)
                tmp.addAll(processNewExpr(next, closedBrace, braceOpened, iterator, false).getTokens());
        }
    } else if (token instanceof DynamicAccessExprToken) {
        next = null;
        if (iterator.hasNext()) {
            next = iterator.next();
            iterator.previous();
        }
        tmp.add(processDynamicAccess(token, next, iterator, closedBrace, braceOpened));
        if (iterator.hasNext()) {
            token = nextTokenAndPrev(iterator);
            if (isOpenedBrace(token, ARRAY) || isOpenedBrace(token, BLOCK) || token instanceof DynamicAccessExprToken) {
                tmp.addAll(processNewExpr(next, closedBrace, braceOpened, iterator, false).getTokens());
            }
        }
    } else
        iterator.previous();
    if (!first) {
        return new ExprStmtToken(null, null, tmp);
    }
    return new ExprStmtToken(analyzer.getEnvironment(), analyzer.getContext(), tmp);
}

Example 20 with Token

use of org.develnext.jphp.core.tokenizer.token.Token in project jphp by jphp-compiler.

the class SimpleExprGenerator method processSimpleToken.

protected Token processSimpleToken(Token current, Token previous, Token next, ListIterator<Token> iterator, BraceExprToken.Kind closedBraceKind, int braceOpened, Separator separator) {
    if (current instanceof DynamicAccessExprToken) {
        return processDynamicAccess(current, next, iterator, closedBraceKind, braceOpened);
    }
    if (current instanceof OperatorExprToken) {
        isRef = false;
        if (current instanceof InstanceofExprToken && next instanceof NameToken) {
            if (previous instanceof VariableExprToken) {
                analyzer.getScope().typeInfoOf(previous).addType(analyzer.getRealName((NameToken) next, NamespaceUseStmtToken.UseType.CLASS).getName());
            }
        }
    }
    if (current instanceof NameToken && next instanceof StringExprToken) {
        // binary string
        if (((NameToken) current).getName().equalsIgnoreCase("b")) {
            ((StringExprToken) next).setBinary(true);
            iterator.next();
            return processString((StringExprToken) next);
        }
    }
    if (current instanceof YieldExprToken) {
        return processYield(current, next, iterator, separator, closedBraceKind);
    }
    if (current instanceof ImportExprToken) {
        return processImport(current, next, iterator, closedBraceKind, braceOpened);
    }
    if (current instanceof PrintNameToken) {
        return processPrint(current, next, iterator, closedBraceKind, braceOpened);
    }
    if (current instanceof NewExprToken) {
        return processNew(current, closedBraceKind, braceOpened, iterator);
    }
    if (current instanceof DollarExprToken) {
        return processVarVar(current, next, iterator);
    }
    if (current instanceof VariableExprToken) {
        VariableExprToken variable = (VariableExprToken) current;
        analyzer.getScope().addVariable(variable);
        FunctionStmtToken function = analyzer.getFunction();
        if (function != null) {
            function.setVarsExists(true);
            function.variable((VariableExprToken) current).setUsed(true);
        }
    }
    // Если переменная меняется, значит она нестабильна и не может быть заменена на костантное значение.
    if ((current instanceof AssignOperatorExprToken || current instanceof IncExprToken || current instanceof DecExprToken) && previous instanceof VariableExprToken) {
        if (analyzer.getFunction() != null) {
            analyzer.getFunction().variable((VariableExprToken) previous).setUnstable(true);
        }
    }
    if (current instanceof AssignExprToken && previous instanceof VariableExprToken && next instanceof VariableExprToken) {
        ExpressionInfo info = analyzer.getScope().typeInfoOf(next);
        analyzer.getScope().typeInfoOf(previous).addTypes(info.getTypes());
    }
    if (current instanceof ValueIfElseToken) {
        return processValueIfElse((ValueIfElseToken) current, next, iterator, closedBraceKind, braceOpened, separator);
    }
    // &
    if (current instanceof AmpersandRefToken) {
        /*if (previous == null)
                unexpectedToken(current);*/
        isRef = true;
        if (next instanceof VariableExprToken)
            if (analyzer.getFunction() != null) {
                analyzer.getFunction().variable((VariableExprToken) next).setReference(true).setMutable(true);
            }
        if (previous instanceof AssignExprToken || previous instanceof KeyValueExprToken || (canStartByReference && previous == null)) {
            if (previous instanceof AssignExprToken)
                ((AssignExprToken) previous).setAsReference(true);
            iterator.previous();
            // =
            Token token = iterator.previous();
            if (iterator.hasPrevious()) {
                token = iterator.previous();
                if (token instanceof VariableExprToken && analyzer.getFunction() != null) {
                    analyzer.getFunction().variable((VariableExprToken) token).setReference(true).setMutable(true);
                // analyzer.getFunction().getUnstableLocal().add((VariableExprToken)token); TODO: check is needed?
                }
                iterator.next();
            }
            iterator.next();
            iterator.next();
            if (!(next instanceof ValueExprToken))
                unexpectedToken(token);
        } else {
            return new AndExprToken(current.getMeta());
        }
        return current;
    }
    // &$var, &$obj->prop['x'], &class::$prop, &$arr['x'], &call()->x;
    if (previous instanceof AmpersandRefToken) {
        if (current instanceof VariableExprToken)
            if (analyzer.getFunction() != null)
                analyzer.getFunction().variable((VariableExprToken) current).setReference(true);
    }
    if ((current instanceof MinusExprToken || current instanceof PlusExprToken) && (next instanceof IntegerExprToken || next instanceof DoubleExprToken)) {
        if (!(previous instanceof ValueExprToken || previous instanceof ArrayGetExprToken || previous instanceof DynamicAccessExprToken || isClosedBrace(previous, SIMPLE))) {
            iterator.next();
            // if it minus
            if (current instanceof MinusExprToken) {
                if (next instanceof IntegerExprToken) {
                    return new IntegerExprToken(TokenMeta.of(current, next));
                } else if (next instanceof DoubleExprToken) {
                    return new DoubleExprToken(TokenMeta.of(current, next));
                }
            }
            // if it plus nothing
            return next;
        }
    }
    if (current instanceof MinusExprToken) {
        if (!(previous instanceof ValueExprToken || previous instanceof ArrayGetExprToken || previous instanceof DynamicAccessExprToken || isClosedBrace(previous))) {
            return new UnarMinusExprToken(current.getMeta());
        }
    }
    if (current instanceof LogicOperatorExprToken) {
        if (next == null)
            unexpectedToken(current);
        final LogicOperatorExprToken logic = (LogicOperatorExprToken) current;
        ExprStmtToken result = analyzer.generator(SimpleExprGenerator.class).getNextExpression(nextToken(iterator), iterator, separator, braceOpened > 0 ? SIMPLE : closedBraceKind);
        logic.setRightValue(result);
        return logic;
    }
    if (next instanceof StaticAccessExprToken) {
        return processStaticAccess(next, current, iterator);
    }
    if (current instanceof StaticAccessExprToken) {
        return processStaticAccess(current, null, iterator);
    }
    if (current instanceof StringExprToken) {
        return processString((StringExprToken) current);
    }
    if (current instanceof NameToken) {
        if (previous instanceof InstanceofExprToken) {
            return analyzer.getRealName((NameToken) current, NamespaceUseStmtToken.UseType.CLASS);
        } else {
            return analyzer.getRealName((NameToken) current, NamespaceUseStmtToken.UseType.CONSTANT);
        }
    }
    if (current instanceof MacroToken) {
        return null;
    }
    if (current instanceof OperatorExprToken) {
        return null;
    }
    if (current.isNamedToken()) {
        return makeSensitive(current);
    }
    return null;
}