Examples with Token org.develnext.jphp.core.tokenizer.token.Token used on opensource projects

Example 41 with Token

use of org.develnext.jphp.core.tokenizer.token.Token in project jphp by jphp-compiler.

the class FunctionGenerator method getToken.

@SuppressWarnings("unchecked")
public FunctionStmtToken getToken(Token current, ListIterator<Token> iterator, boolean closureAllowed) {
    if (current instanceof FunctionStmtToken) {
        CommentToken commentToken = null;
        iterator.previous();
        if (iterator.hasPrevious()) {
            int cnt = 0;
            while (iterator.hasPrevious()) {
                cnt++;
                Token previous = iterator.previous();
                if (previous.isNamedToken())
                    continue;
                if (previous instanceof CommentToken && ((CommentToken) previous).getKind() == CommentToken.Kind.DOCTYPE) {
                    commentToken = ((CommentToken) previous);
                }
                break;
            }
            for (int i = 0; i < cnt; i++) {
                iterator.next();
            }
        }
        iterator.next();
        FunctionStmtToken result = (FunctionStmtToken) current;
        result.setStatic(analyzer.getFunction() == null);
        Class<? extends Token>[] excludes = new Class[] { EchoStmtToken.class, ImportExprToken.class };
        if (analyzer.getClazz() != null) {
            excludes = new Class[0];
        }
        Token next = nextTokenSensitive(iterator, excludes);
        if (next instanceof AmpersandRefToken) {
            result.setReturnReference(true);
            next = nextTokenSensitive(iterator, excludes);
        }
        if (next instanceof NameToken) {
            /*if (analyzer.getFunction() != null)
                    unexpectedToken(current);*/
            analyzer.addScope(true);
            FunctionStmtToken oldFunction = analyzer.getFunction();
            analyzer.setFunction(result);
            BraceExprToken brace = nextAndExpected(iterator, BraceExprToken.class);
            if (!brace.isSimpleOpened())
                unexpectedToken(brace, "(");
            result.setNamespace(analyzer.getNamespace());
            result.setName((NameToken) next);
            result.setDocComment(commentToken);
            processArguments(result, iterator, true);
            processBody(result, iterator);
            result.setTypeInfo(analyzer.getScope().getTypeInfo());
            result.setLabels(analyzer.getScope().getLabels());
            result.setLocal(analyzer.removeScope().getVariables());
            Token previous = iterator.previous();
            result.getMeta().setEndLine(previous.getMeta().getStartLine());
            result.getMeta().setEndPosition(previous.getMeta().getStartPosition());
            iterator.next();
            analyzer.setFunction(oldFunction);
            return result;
        } else if (next instanceof BraceExprToken) {
            // xClosure
            if (((BraceExprToken) next).isSimpleOpened()) {
                if (closureAllowed) {
                    analyzer.pushClosure(result);
                    analyzer.addScope(true);
                    processArguments(result, iterator, true);
                    processUses(result, iterator);
                    processBody(result, iterator);
                    // boolean thisExists = result.isThisExists();
                    result.setTypeInfo(analyzer.getScope().getTypeInfo());
                    result.setLabels(analyzer.getScope().getLabels());
                    result.setStaticExists(analyzer.getScope().isStaticExists());
                    result.setLocal(analyzer.removeScope().getVariables());
                    // result.setThisExists(thisExists);
                    analyzer.popClosure();
                    FunctionStmtToken prevClosure = analyzer.peekClosure();
                    if (prevClosure != null) {
                        if (result.isThisExists()) {
                            analyzer.getScope().addVariable(FunctionStmtToken.thisVariable);
                        // prevClosure.variable(FunctionStmtToken.thisVariable).setUsed(true);
                        // prevClosure.setThisExists(true);
                        }
                    }
                    List<VariableExprToken> uses = new ArrayList<VariableExprToken>();
                    for (ArgumentStmtToken argument : result.getUses()) {
                        if (argument.isReference()) {
                            if (analyzer.getFunction() != null) {
                                analyzer.getFunction().variable(argument.getName()).setReference(true);
                            }
                        }
                        if (analyzer.getFunction() != null) {
                            analyzer.getFunction().variable(argument.getName()).setUsed(true);
                        }
                        uses.add(argument.getName());
                    }
                    analyzer.getScope().addVariables(uses);
                    Token previous = iterator.previous();
                    result.getMeta().setEndLine(previous.getMeta().getStartLine());
                    result.getMeta().setEndPosition(previous.getMeta().getStartPosition());
                    iterator.next();
                    return result;
                }
                iterator.previous();
                return null;
            }
        }
        unexpectedToken(next);
    }
    return null;
}

Example 42 with Token

use of org.develnext.jphp.core.tokenizer.token.Token in project jphp by jphp-compiler.

the class LambdaGenerator method processBody.

private void processBody(FunctionStmtToken func, ListIterator<Token> iterator, Separator separator, BraceExprToken.Kind braceKind) {
    Token token = nextToken(iterator);
    if (isOpenedBrace(token, BraceExprToken.Kind.BLOCK)) {
        prevToken(iterator);
        analyzer.generator(FunctionGenerator.class).processBody(func, iterator);
    } else {
        ExprStmtToken expression = analyzer.generator(SimpleExprGenerator.class).getNextExpression(token, iterator, separator, braceKind);
        ReturnStmtToken returnStmtToken = new ReturnStmtToken(expression.getMeta());
        returnStmtToken.setValue(expression);
        BodyStmtToken body = new BodyStmtToken(returnStmtToken.getMeta());
        body.setInstructions(Arrays.asList(new ExprStmtToken(analyzer.getEnvironment(), analyzer.getContext(), returnStmtToken)));
        func.setBody(body);
        token = prevToken(iterator);
        if (!isBreak(token)) {
            nextToken(iterator);
        }
    }
}

Example 43 with Token

use of org.develnext.jphp.core.tokenizer.token.Token in project jphp by jphp-compiler.

the class LambdaGenerator method processArguments.

private void processArguments(FunctionStmtToken func, ListIterator<Token> iterator) {
    Token token = nextToken(iterator);
    if (isOpenedBrace(token, BraceExprToken.Kind.SIMPLE)) {
        FunctionGenerator funcGenerator = analyzer.generator(FunctionGenerator.class);
        funcGenerator.processArguments(func, iterator, true);
    } else {
        prevToken(iterator);
        func.setArguments(new ArrayList<>());
    }
}

Example 44 with Token

use of org.develnext.jphp.core.tokenizer.token.Token in project jphp by jphp-compiler.

the class LambdaGenerator method getToken.

public LambdaStmtToken getToken(Token current, ListIterator<Token> iterator, Separator separator, BraceExprToken.Kind braceKind) {
    if (current instanceof LambdaStmtToken) {
        LambdaStmtToken lambda = (LambdaStmtToken) current;
        iterator.previous();
        FunctionStmtToken func = new FunctionStmtToken(current.getMeta());
        func.setUses(new ArrayList<>());
        func.setAutoUses(true);
        analyzer.pushClosure(func);
        analyzer.addScope(true);
        nextToken(iterator);
        Token tk = nextToken(iterator);
        if (tk instanceof AmpersandRefToken) {
            func.setReturnReference(true);
        } else {
            prevToken(iterator);
        }
        processArguments(func, iterator);
        nextAndExpected(iterator, KeyValueExprToken.class);
        processBody(func, iterator, separator, braceKind);
        func.setTypeInfo(analyzer.getScope().getTypeInfo());
        func.setLabels(analyzer.getScope().getLabels());
        func.setStaticExists(analyzer.getScope().isStaticExists());
        func.setLocal(analyzer.removeScope().getVariables());
        analyzer.popClosure();
        FunctionStmtToken prevClosure = analyzer.peekClosure();
        if (prevClosure != null) {
            if (func.isThisExists()) {
                analyzer.getScope().addVariable(FunctionStmtToken.thisVariable);
            }
        }
        Set<VariableExprToken> variables = analyzer.getScope().getVariables();
        if (analyzer.getFunction() != null) {
            for (ArgumentStmtToken token : analyzer.getFunction().getArguments()) {
                variables.add(token.getName());
            }
        }
        FunctionStmtToken peekClosure = analyzer.peekClosure();
        if (peekClosure != null && peekClosure.getUses() != null) {
            for (ArgumentStmtToken token : peekClosure.getUses()) {
                variables.add(token.getName());
            }
        }
        Set<VariableExprToken> local = func.getLocal();
        Set<VariableExprToken> arguments = new HashSet<>();
        List<ArgumentStmtToken> uses = func.getUses();
        for (ArgumentStmtToken argument : func.getArguments()) {
            arguments.add(argument.getName());
        }
        for (VariableExprToken l : local) {
            if (!arguments.contains(l) && !l.equals(FunctionStmtToken.thisVariable)) {
                CompileScope compileScope = this.analyzer.getEnvironment().getScope();
                if (compileScope.superGlobals.contains(l.getName())) {
                    continue;
                }
                ArgumentStmtToken use = new ArgumentStmtToken(l.getMeta());
                use.setName(l);
                use.setReference(false);
                if (analyzer.getFunction() != null) {
                    analyzer.getFunction().variable(l).setUsed(true);
                }
                uses.add(use);
                if (peekClosure != null && peekClosure.isAutoUses() && peekClosure.getUses() != null) {
                    peekClosure.getUses().add(use);
                }
            }
        }
        func.setUses(uses);
        Token previous = iterator.previous();
        func.getMeta().setEndLine(previous.getMeta().getStartLine());
        func.getMeta().setEndPosition(previous.getMeta().getStartPosition());
        iterator.next();
        lambda.setFunction(func);
        lambda.setParentFunction(peekClosure);
        return lambda;
    }
    return null;
}

Example 45 with Token

use of org.develnext.jphp.core.tokenizer.token.Token in project jphp by jphp-compiler.

the class TokenizerTest method testScalarTokens.

@Test
public void testScalarTokens() throws IOException {
    Token token;
    Tokenizer tokenizer = new Tokenizer(new Context("10 3.3 'foo' '' \"xyz\" 0xCC 0b0011 true false"));
    token = tokenizer.nextToken();
    assertTrue(token instanceof IntegerExprToken);
    assertEquals(10L, ((IntegerExprToken) token).getValue());
    token = tokenizer.nextToken();
    assertTrue(token instanceof DoubleExprToken);
    assertEquals(3.3, ((DoubleExprToken) token).getValue(), 0.01);
    token = tokenizer.nextToken();
    assertTrue(token instanceof StringExprToken);
    assertEquals(StringExprToken.Quote.SINGLE, ((StringExprToken) token).getQuote());
    assertEquals("foo", ((StringExprToken) token).getValue());
    token = tokenizer.nextToken();
    assertTrue(token instanceof StringExprToken);
    assertEquals(StringExprToken.Quote.SINGLE, ((StringExprToken) token).getQuote());
    assertEquals("", ((StringExprToken) token).getValue());
    token = tokenizer.nextToken();
    assertTrue(token instanceof StringExprToken);
    assertEquals(DOUBLE, ((StringExprToken) token).getQuote());
    assertEquals("xyz", ((StringExprToken) token).getValue());
    token = tokenizer.nextToken();
    assertTrue(token instanceof IntegerExprToken);
    assertEquals(new BigInteger("CC", 16).longValue(), ((IntegerExprToken) token).getValue());
    token = tokenizer.nextToken();
    assertTrue(token instanceof IntegerExprToken);
    assertEquals(new BigInteger("0011", 2).longValue(), ((IntegerExprToken) token).getValue());
    token = tokenizer.nextToken();
    assertTrue(token instanceof BooleanExprToken);
    assertEquals(true, ((BooleanExprToken) token).getValue());
    token = tokenizer.nextToken();
    assertTrue(token instanceof BooleanExprToken);
    assertEquals(false, ((BooleanExprToken) token).getValue());
    token = tokenizer.nextToken();
    assertNull(token);
}