Examples with SemicolonToken org.develnext.jphp.core.tokenizer.token.SemicolonToken used on opensource projects

Example 6 with SemicolonToken

use of org.develnext.jphp.core.tokenizer.token.SemicolonToken in project jphp by jphp-compiler.

the class TokenizerTest method testComplex.

@Test
public void testComplex() throws IOException {
    Tokenizer tokenizer = new Tokenizer(new Context("0==10==='30';"));
    assertTrue(tokenizer.nextToken() instanceof IntegerExprToken);
    assertTrue(tokenizer.nextToken() instanceof EqualExprToken);
    assertTrue(tokenizer.nextToken() instanceof IntegerExprToken);
    assertTrue(tokenizer.nextToken() instanceof IdenticalExprToken);
    assertTrue(tokenizer.nextToken() instanceof StringExprToken);
    assertTrue(tokenizer.nextToken() instanceof SemicolonToken);
    tokenizer = new Tokenizer(new Context("F =; 20;"));
    Token token;
    assertTrue((token = tokenizer.nextToken()) instanceof NameToken);
    assertEquals("F", ((NameToken) token).getName());
    assertTrue(tokenizer.nextToken() instanceof AssignExprToken);
    assertTrue(tokenizer.nextToken() instanceof SemicolonToken);
    assertTrue(tokenizer.nextToken() instanceof IntegerExprToken);
    assertTrue(tokenizer.nextToken() instanceof SemicolonToken);
    tokenizer = new Tokenizer(new Context("123foobar MAX_64Bit"));
    token = tokenizer.nextToken();
    assertTrue(token instanceof IntegerExprToken);
    token = tokenizer.nextToken();
    assertTrue(token instanceof NameToken);
    assertEquals("foobar", token.getWord());
    assertTrue(tokenizer.nextToken() instanceof NameToken);
    assertNull(tokenizer.nextToken());
    assertNull(tokenizer.nextToken());
}

Example 7 with SemicolonToken

use of org.develnext.jphp.core.tokenizer.token.SemicolonToken in project jphp by jphp-compiler.

the class TokenizerTest method testBraces.

@Test
public void testBraces() throws IOException {
    Tokenizer tokenizer = new Tokenizer(new Context(" :: ->foobar('a', 1, 3.0);"));
    assertTrue(tokenizer.nextToken() instanceof StaticAccessExprToken);
    assertTrue(tokenizer.nextToken() instanceof DynamicAccessExprToken);
    assertTrue(tokenizer.nextToken() instanceof NameToken);
    assertTrue(tokenizer.nextToken() instanceof BraceExprToken);
    assertTrue(tokenizer.nextToken() instanceof StringExprToken);
    assertTrue(tokenizer.nextToken() instanceof CommaToken);
    assertTrue(tokenizer.nextToken() instanceof IntegerExprToken);
    assertTrue(tokenizer.nextToken() instanceof CommaToken);
    assertTrue(tokenizer.nextToken() instanceof DoubleExprToken);
    assertTrue(tokenizer.nextToken() instanceof BraceExprToken);
    assertTrue(tokenizer.nextToken() instanceof SemicolonToken);
}

Example 8 with SemicolonToken

use of org.develnext.jphp.core.tokenizer.token.SemicolonToken in project jphp by jphp-compiler.

the class SimpleExprGenerator method getToken.

@SuppressWarnings("unchecked")
public ExprStmtToken getToken(Token current, ListIterator<Token> iterator, Separator separator, BraceExprToken.Kind closedBraceKind, Callback<Boolean, Token> breakCallback) {
    isRef = false;
    List<Token> tokens = new ArrayList<Token>();
    Token previous = null;
    Token next = iterator.hasNext() ? iterator.next() : null;
    if (next != null)
        iterator.previous();
    int braceOpened = 0;
    boolean needBreak = false;
    do {
        if (breakCallback != null && current != null && breakCallback.call(current)) {
            break;
        }
        if (isOpenedBrace(current, BraceExprToken.Kind.SIMPLE)) {
            boolean isFunc = false;
            if (previous instanceof NameToken && previous.getMeta().getWord().equalsIgnoreCase("array")) {
                iterator.previous();
                tokens.set(tokens.size() - 1, current = processNewArray(previous, iterator));
            } else {
                if (previous instanceof NameToken || previous instanceof VariableExprToken || previous instanceof ClosureStmtToken || previous instanceof ArrayGetExprToken || previous instanceof CallExprToken)
                    isFunc = true;
                else if (previous instanceof StaticAccessExprToken) {
                    // !((StaticAccessExprToken)previous).isGetStaticField(); TODO check it!
                    isFunc = true;
                } else if (previous instanceof DynamicAccessExprToken) {
                    isFunc = true;
                }
                if (isFunc) {
                    CallExprToken call = processCall(previous, current, iterator);
                    if (call.getName() != null) {
                        current = call;
                        tokens.set(tokens.size() - 1, call);
                    } else {
                        tokens.add(current = new CallOperatorToken(call));
                    }
                } else {
                    if (needBreak)
                        unexpectedToken(current);
                    braceOpened += 1;
                    tokens.add(current);
                }
            }
        } else if (braceOpened > 0 && isClosedBrace(current, BraceExprToken.Kind.SIMPLE)) {
            braceOpened -= 1;
            tokens.add(current);
            if (isOpenedBrace(previous, BraceExprToken.Kind.SIMPLE))
                unexpectedToken(current);
        } else if (isOpenedBrace(current, ARRAY) || isOpenedBrace(current, BLOCK)) {
            if (isTokenClass(previous, NameToken.class, VariableExprToken.class, GetVarExprToken.class, CallExprToken.class, ArrayGetExprToken.class, DynamicAccessExprToken.class, StringExprToken.class, StringBuilderExprToken.class, CallOperatorToken.class, ArrayPushExprToken.class) || (previous instanceof StaticAccessExprToken && ((StaticAccessExprToken) previous).isGetStaticField())) {
                // array
                current = processArrayToken(previous, current, iterator);
                if (previous instanceof DynamicAccessExprToken && current instanceof ArrayGetRefExprToken) {
                    tokens.set(tokens.size() - 1, new DynamicAccessGetRefExprToken((DynamicAccessExprToken) previous));
                }
                tokens.add(current);
            } else if (previous instanceof OperatorExprToken || previous == null || isOpenedBrace(previous, BraceExprToken.Kind.SIMPLE) || isOpenedBrace(previous, BLOCK)) {
                tokens.add(current = processNewArray(current, iterator));
            } else
                unexpectedToken(current);
        } else if (braceOpened == 0 && isClosedBrace(current, ARRAY)) {
            if (separator == Separator.ARRAY)
                break;
            if (closedBraceKind == ARRAY || closedBraceKind == BraceExprToken.Kind.ANY) {
                //if (tokens.isEmpty())
                iterator.previous();
                break;
            }
            unexpectedToken(current);
        } else if (separator == Separator.ARRAY_BLOCK && braceOpened == 0 && isClosedBrace(current, BLOCK)) {
            break;
        } else if (current instanceof FunctionStmtToken) {
            current = processClosure(current, next, iterator);
            tokens.add(current);
        } else if (current instanceof ListExprToken && isOpenedBrace(next, BraceExprToken.Kind.SIMPLE)) {
            current = processList(current, iterator, null, closedBraceKind, braceOpened);
            tokens.add(current);
        } else if (current instanceof DieExprToken) {
            processDie(current, next, iterator);
            tokens.add(current);
        } else if (current instanceof EmptyExprToken) {
            processEmpty(current, iterator);
            tokens.add(current);
        } else if (current instanceof IssetExprToken) {
            processIsset(previous, current, iterator);
            tokens.add(current);
        } else if (current instanceof UnsetExprToken) {
            if (isOpenedBrace(previous, BraceExprToken.Kind.SIMPLE) && isClosedBrace(next, BraceExprToken.Kind.SIMPLE)) {
                current = new UnsetCastExprToken(current.getMeta());
                tokens.set(tokens.size() - 1, current);
                iterator.next();
                braceOpened--;
            } else {
                if (previous != null)
                    unexpectedToken(current);
                processUnset(previous, current, iterator);
                tokens.add(current);
                needBreak = true;
            }
        } else if (current instanceof CommaToken) {
            if (separator == Separator.COMMA || separator == Separator.COMMA_OR_SEMICOLON) {
                if (tokens.isEmpty())
                    unexpectedToken(current);
                break;
            } else {
                unexpectedToken(current);
            }
        } else if (current instanceof AsStmtToken) {
            if (separator == Separator.AS)
                break;
            unexpectedToken(current);
        } else if (isClosedBrace(current, closedBraceKind)) {
            iterator.previous();
            break;
        } else if (current instanceof BreakToken) {
            break;
        } else if (current instanceof ColonToken) {
            if (separator == Separator.COLON) {
                /*if (tokens.isEmpty()) see: issues/93
                        unexpectedToken(current);*/
                break;
            }
            unexpectedToken(current);
        } else if (current instanceof SemicolonToken) {
            // TODO refactor!
            if (separator == Separator.SEMICOLON || separator == Separator.COMMA_OR_SEMICOLON) {
                /*if (tokens.isEmpty()) see: issues/94
                        unexpectedToken(current);*/
                break;
            }
            if (separator == Separator.COMMA || closedBraceKind != null || tokens.isEmpty())
                unexpectedToken(current);
            break;
        } else if (current instanceof BraceExprToken) {
            if (closedBraceKind == BraceExprToken.Kind.ANY && isClosedBrace(current)) {
                iterator.previous();
                break;
            }
            unexpectedToken(current);
        } else if (current instanceof ArrayExprToken) {
            if (needBreak)
                unexpectedToken(current);
            tokens.add(current = processNewArray(current, iterator));
        } else if (current instanceof ExprToken) {
            if (needBreak)
                unexpectedToken(current);
            CastExprToken cast = null;
            if (current instanceof NameToken && isOpenedBrace(previous, BraceExprToken.Kind.SIMPLE) && isClosedBrace(next, BraceExprToken.Kind.SIMPLE)) {
                cast = CastExprToken.valueOf(((NameToken) current).getName(), current.getMeta());
                if (cast != null) {
                    current = cast;
                    iterator.next();
                    braceOpened--;
                    tokens.set(tokens.size() - 1, current);
                }
            }
            if (cast == null) {
                Token token = processSimpleToken(current, previous, next, iterator, closedBraceKind, braceOpened, separator);
                if (token != null)
                    current = token;
                tokens.add(current);
            }
        } else
            unexpectedToken(current);
        previous = current;
        if (iterator.hasNext()) {
            current = nextToken(iterator);
            next = iterator.hasNext() ? iterator.next() : null;
            if (next != null)
                iterator.previous();
        } else
            current = null;
        if (current == null)
            nextToken(iterator);
    } while (current != null);
    if (tokens.isEmpty())
        return null;
    if (braceOpened != 0)
        unexpectedToken(iterator.previous());
    ExprStmtToken exprStmtToken = new ExprStmtToken(analyzer.getEnvironment(), analyzer.getContext(), tokens);
    return exprStmtToken;
}

Example 9 with SemicolonToken

use of org.develnext.jphp.core.tokenizer.token.SemicolonToken in project jphp by jphp-compiler.

the class NamespaceGenerator method processBody.

protected void processBody(NamespaceStmtToken namespace, ListIterator<Token> iterator) {
    Token next = nextToken(iterator);
    if (next instanceof SemicolonToken) {
        List<Token> tree = analyzer.process(iterator);
        namespace.setTree(tree);
        namespace.setTokenRegistered(true);
    } else if (isOpenedBrace(next, BraceExprToken.Kind.BLOCK)) {
        boolean done = false;
        List<Token> tree = new ArrayList<Token>();
        while (iterator.hasNext()) {
            Token item = analyzer.processNext(iterator);
            if (isClosedBrace(item, BraceExprToken.Kind.BLOCK)) {
                done = true;
                break;
            }
            tree.add(item);
        }
        if (!done)
            nextToken(iterator);
        namespace.setTree(tree);
    } else
        unexpectedToken(next);
}

Example 10 with SemicolonToken

use of org.develnext.jphp.core.tokenizer.token.SemicolonToken in project jphp by jphp-compiler.

the class ClassGenerator method processProperty.

protected List<ClassVarStmtToken> processProperty(ClassStmtToken clazz, VariableExprToken current, List<Token> modifiers, ListIterator<Token> iterator) {
    Token next = current;
    Token prev = null;
    Set<VariableExprToken> variables = new LinkedHashSet<VariableExprToken>();
    List<ExprStmtToken> initValues = new ArrayList<>();
    ExprStmtToken initValue = null;
    List<ClassVarStmtToken> result = new ArrayList<ClassVarStmtToken>();
    Modifier modifier = Modifier.PUBLIC;
    boolean isStatic = false;
    for (Token token : modifiers) {
        if (token instanceof PrivateStmtToken)
            modifier = Modifier.PRIVATE;
        else if (token instanceof ProtectedStmtToken)
            modifier = Modifier.PROTECTED;
        else if (token instanceof StaticExprToken)
            isStatic = true;
    }
    do {
        if (next instanceof VariableExprToken) {
            if (!variables.add((VariableExprToken) next)) {
                throw new ParseException(Messages.ERR_IDENTIFIER_X_ALREADY_USED.fetch(next.getWord()), next.toTraceInfo(analyzer.getContext()));
            }
            initValues.add(null);
        } else if (next instanceof CommaToken) {
            if (!(prev instanceof VariableExprToken))
                unexpectedToken(next);
        } else if (next instanceof AssignExprToken) {
            if (!(prev instanceof VariableExprToken))
                unexpectedToken(next);
            initValue = analyzer.generator(SimpleExprGenerator.class).getToken(nextToken(iterator), iterator, Separator.COMMA_OR_SEMICOLON, null);
            initValues.set(initValues.size() - 1, initValue);
            if (iterator.hasPrevious() && isBreak(iterator.previous())) {
                iterator.next();
                break;
            }
            if (iterator.hasNext()) {
                iterator.next();
            }
        //break;
        } else if (next instanceof SemicolonToken) {
            if (!(prev instanceof VariableExprToken))
                unexpectedToken(next);
            break;
        }
        prev = next;
        next = nextToken(iterator);
    } while (true);
    int i = 0;
    for (VariableExprToken variable : variables) {
        ClassVarStmtToken classVar = new ClassVarStmtToken(variable.getMeta());
        classVar.setModifier(modifier);
        classVar.setStatic(isStatic);
        classVar.setValue(initValues.get(i));
        classVar.setVariable(variable);
        classVar.setClazz(clazz);
        result.add(classVar);
        i++;
    }
    return result;
}