Examples with Token org.develnext.jphp.core.tokenizer.token.Token used on opensource projects

Example 51 with Token

use of org.develnext.jphp.core.tokenizer.token.Token in project jphp by jphp-compiler.

the class Tokenizer method tryNextToken.

protected Token tryNextToken() {
    int len = 0;
    char ch;
    if (currentPosition + 1 < codeLength) {
        ch = code.charAt(currentPosition + 1);
        if (TokenizeGrammarUtils.isDelimiter(ch) && !TokenizeGrammarUtils.isSpace(ch)) {
            len = 1;
            if (currentPosition + 2 < codeLength) {
                ch = code.charAt(currentPosition + 2);
                if (TokenizeGrammarUtils.isDelimiter(ch) && !TokenizeGrammarUtils.isSpace(ch)) {
                    len = 2;
                }
            }
        }
    }
    if (len == 0)
        return null;
    String word = getWord(currentPosition, currentPosition + len + 1);
    Class<? extends Token> tokenClass = tokenFinder.find(word);
    if (len == 2 && tokenClass == null) {
        len -= 1;
        word = getWord(currentPosition, currentPosition + len + 1);
        tokenClass = tokenFinder.find(word);
    }
    if (tokenClass != null) {
        int startPosition = currentPosition;
        currentPosition += len + 1;
        Token token = buildToken(tokenClass, buildMeta(startPosition, currentLine));
        currentPosition -= 1;
        return token;
    }
    return null;
}

Example 52 with Token

use of org.develnext.jphp.core.tokenizer.token.Token in project jphp by jphp-compiler.

the class ExprStmtToken method setTokens.

public void setTokens(List<Token> tokens) {
    int cnt = 0;
    variadic = false;
    for (Token el : tokens) {
        if (el == null)
            continue;
        cnt++;
        if (cnt == 1 && el instanceof ArgumentUnpackExprToken) {
            variadic = true;
        }
        if (!(el instanceof StmtToken)) {
            isStmtList = false;
            break;
        }
        if (el instanceof GotoStmtToken || el instanceof LabelStmtToken) {
            constantly = false;
        }
    }
    this.tokens = tokens;
}

Example 53 with Token

use of org.develnext.jphp.core.tokenizer.token.Token in project jphp by jphp-compiler.

the class SourceTokenizer method fetchAll.

@Signature
public List<SourceToken> fetchAll(Environment env) {
    Token token;
    List<SourceToken> result = new ArrayList<>();
    while ((token = tokenizer.nextToken()) != null) {
        result.add(new SourceToken(env, token));
    }
    return result;
}

Example 54 with Token

use of org.develnext.jphp.core.tokenizer.token.Token in project jphp by jphp-compiler.

the class SimpleExprGenerator method processList.

protected ListExprToken processList(Token current, ListIterator<Token> iterator, List<Integer> indexes, BraceExprToken.Kind closedBraceKind, int braceOpened) {
    ListExprToken result = (ListExprToken) current;
    Token next = nextToken(iterator);
    if (!isOpenedBrace(next, BraceExprToken.Kind.SIMPLE))
        unexpectedToken(next, "(");
    int i = 0;
    while (true) {
        next = nextToken(iterator);
        /*if (next instanceof VariableExprToken){
                if (prev != null && !(prev instanceof CommaToken))
                    unexpectedToken(next);

                analyzer.getLocalScope().add((VariableExprToken)next);
                result.addVariable(((VariableExprToken) next), i, indexes);
            } else*/
        if (next instanceof ListExprToken) {
            List<Integer> indexes_ = new ArrayList<Integer>();
            if (indexes != null)
                indexes_.addAll(indexes);
            indexes_.add(i);
            ListExprToken tmp = processList(next, iterator, indexes_, null, -1);
            result.addList(tmp);
            if (nextTokenAndPrev(iterator) instanceof CommaToken)
                iterator.next();
            i++;
        } else if (isClosedBrace(next, BraceExprToken.Kind.SIMPLE)) {
            break;
        } else if (next instanceof CommaToken) {
            i++;
        } else {
            SimpleExprGenerator generator = analyzer.generator(SimpleExprGenerator.class);
            ExprStmtToken var = generator.getToken(next, iterator, Separator.COMMA, BraceExprToken.Kind.SIMPLE);
            Token single = var.getLast();
            if (!(single instanceof VariableExprToken || single instanceof ArrayGetExprToken || single instanceof DynamicAccessExprToken || single instanceof ArrayPushExprToken || (single instanceof StaticAccessExprToken && ((StaticAccessExprToken) single).isGetStaticField()))) {
                unexpectedToken(single);
            }
            if (single instanceof ArrayGetExprToken) {
                single = new ArrayGetRefExprToken((ArrayGetExprToken) single);
                var.getTokens().set(var.getTokens().size() - 1, single);
                var.updateAsmExpr(analyzer.getEnvironment(), analyzer.getContext());
            }
            result.addVariable(var, i, indexes);
            i++;
        }
    }
    if (braceOpened != -1) {
        next = nextToken(iterator);
        if (!(next instanceof AssignExprToken))
            unexpectedToken(next, "=");
        ExprStmtToken value = analyzer.generator(SimpleExprGenerator.class).getNextExpression(nextToken(iterator), iterator, BraceExprToken.Kind.ANY);
        result.setValue(value);
    }
    return result;
}

Example 55 with Token

use of org.develnext.jphp.core.tokenizer.token.Token in project jphp by jphp-compiler.

the class ClassGenerator method processProperty.

protected List<ClassVarStmtToken> processProperty(ClassStmtToken clazz, VariableExprToken current, List<Token> modifiers, ListIterator<Token> iterator) {
    Token next = current;
    Token prev = null;
    Set<VariableExprToken> variables = new LinkedHashSet<VariableExprToken>();
    List<ExprStmtToken> initValues = new ArrayList<>();
    ExprStmtToken initValue = null;
    List<ClassVarStmtToken> result = new ArrayList<ClassVarStmtToken>();
    Modifier modifier = Modifier.PUBLIC;
    boolean isStatic = false;
    for (Token token : modifiers) {
        if (token instanceof PrivateStmtToken)
            modifier = Modifier.PRIVATE;
        else if (token instanceof ProtectedStmtToken)
            modifier = Modifier.PROTECTED;
        else if (token instanceof StaticExprToken)
            isStatic = true;
    }
    do {
        if (next instanceof VariableExprToken) {
            if (!variables.add((VariableExprToken) next)) {
                throw new ParseException(Messages.ERR_IDENTIFIER_X_ALREADY_USED.fetch(next.getWord()), next.toTraceInfo(analyzer.getContext()));
            }
            initValues.add(null);
        } else if (next instanceof CommaToken) {
            if (!(prev instanceof VariableExprToken))
                unexpectedToken(next);
        } else if (next instanceof AssignExprToken) {
            if (!(prev instanceof VariableExprToken))
                unexpectedToken(next);
            initValue = analyzer.generator(SimpleExprGenerator.class).getToken(nextToken(iterator), iterator, Separator.COMMA_OR_SEMICOLON, null);
            initValues.set(initValues.size() - 1, initValue);
            if (iterator.hasPrevious() && isBreak(iterator.previous())) {
                iterator.next();
                break;
            }
            if (iterator.hasNext()) {
                iterator.next();
            }
        // break;
        } else if (next instanceof SemicolonToken) {
            if (!(prev instanceof VariableExprToken))
                unexpectedToken(next);
            break;
        }
        prev = next;
        next = nextToken(iterator);
    } while (true);
    int i = 0;
    for (VariableExprToken variable : variables) {
        ClassVarStmtToken classVar = new ClassVarStmtToken(variable.getMeta());
        classVar.setModifier(modifier);
        classVar.setStatic(isStatic);
        classVar.setValue(initValues.get(i));
        classVar.setVariable(variable);
        classVar.setClazz(clazz);
        result.add(classVar);
        i++;
    }
    return result;
}