Examples with IToken dyvilx.tools.parsing.token.IToken used on opensource projects

Example 11 with IToken

use of dyvilx.tools.parsing.token.IToken in project Dyvil by Dyvil.

the class SourceFileParser method parse.

@Override
public void parse(IParserManager pm, IToken token) {
    final int type = token.type();
    switch(this.mode) {
        case SEPARATOR:
            if (this.unit.classCount() > 0) {
                // any classes -> only allow classes from here
                this.mode = CLASS;
            } else if (this.unit.importCount() > 0 || this.unit.typeAliasCount() > 0 || this.unit.operatorCount() > 0 || this.unit.getHeaderDeclaration() != null) {
                // any imports, type aliases, operators or header declarations -> don't allow any package declarations
                this.mode = IMPORT;
            } else {
                // nothing defined yet -> allow a package declaration
                this.mode = PACKAGE;
            }
            if (!checkEnd(pm, type)) {
                pm.report(token, "header.separator");
                pm.reparse();
            }
            return;
        case PACKAGE:
            if (type == DyvilKeywords.PACKAGE) {
                PackageDeclaration pack = new PackageDeclaration(token.raw());
                this.unit.setPackageDeclaration(pack);
                pm.pushParser(new PackageParser(pack));
                this.mode = SEPARATOR;
                return;
            }
        // Fallthrough
        case IMPORT:
            switch(type) {
                case DyvilKeywords.IMPORT:
                    pm.pushParser(new ImportParser(this.importConsumer(token)));
                    this.mode = SEPARATOR;
                    return;
                case DyvilKeywords.USING:
                    pm.pushParser(new ImportParser(this.importConsumer(token), KindedImport.USING_DECLARATION));
                    this.mode = SEPARATOR;
                    return;
                case DyvilKeywords.OPERATOR:
                    pm.pushParser(new OperatorParser(this.unit, Operator.INFIX), true);
                    this.mode = SEPARATOR;
                    return;
                case DyvilKeywords.PREFIX:
                case DyvilKeywords.POSTFIX:
                case DyvilKeywords.INFIX:
                    if (// 
                    token.next().type() == DyvilKeywords.OPERATOR || token.next().type() == DyvilKeywords.POSTFIX && token.next().next().type() == DyvilKeywords.OPERATOR) {
                        pm.pushParser(new OperatorParser(this.unit), true);
                        this.mode = SEPARATOR;
                        return;
                    }
                    // parse as modifier (in 'case CLASS' via fallthrough)
                    break;
                case DyvilKeywords.TYPE:
                    pm.pushParser(new TypeAliasParser(this.unit, new TypeAlias()));
                    this.mode = SEPARATOR;
                    return;
                case DyvilKeywords.HEADER:
                    final IToken next = token.next();
                    if (!Tokens.isIdentifier(next.type())) {
                        this.attributes = new AttributeList();
                        pm.report(next, "header.declaration.identifier");
                        return;
                    }
                    pm.skip();
                    if (this.unit.getHeaderDeclaration() != null) {
                        this.attributes = new AttributeList();
                        pm.report(token, "header.declaration.duplicate");
                        this.mode = SEPARATOR;
                        return;
                    }
                    final HeaderDeclaration declaration = new HeaderDeclaration(this.unit, next.raw(), next.nameValue(), this.attributes);
                    this.unit.setHeaderDeclaration(declaration);
                    // reset
                    this.attributes = new AttributeList();
                    this.mode = SEPARATOR;
                    return;
            }
        // Fallthrough
        case CLASS:
            if (this.parseAttribute(pm, token)) {
                return;
            }
            final int classType;
            if ((classType = ModifierParser.parseClassTypeModifier(token, pm)) >= 0) {
                if ((this.flags & NO_CLASSES) != 0) {
                    pm.report(token, "header.class");
                }
                this.attributes.addFlag(classType);
                pm.pushParser(new ClassDeclarationParser(this.unit, this.attributes));
                // reset
                this.attributes = new AttributeList();
                this.mode = SEPARATOR;
                return;
            }
    }
    if (!checkEnd(pm, type)) {
        pm.report(Markers.syntaxError(token, "header.element.invalid", token.toString()));
    }
}

Example 12 with IToken

use of dyvilx.tools.parsing.token.IToken in project Dyvil by Dyvil.

the class StatementListParser method parse.

@Override
public void parse(IParserManager pm, IToken token) {
    final int type = token.type();
    if (type == BaseSymbols.CLOSE_CURLY_BRACKET) {
        this.end(pm);
        return;
    }
    if (type == Tokens.EOF) {
        this.end(pm);
        pm.report(token, "statement_list.close_brace");
        return;
    }
    switch(this.mode) {
        case OPEN_BRACKET:
            {
                final IToken next = token.next();
                final IToken lambdaArrow = this.findLambdaArrow(next);
                if (lambdaArrow != null) {
                    this.lambdaExpr = new LambdaExpr(lambdaArrow.raw());
                    this.lambdaExpr.setValue(this.statementList = new StatementList(token));
                    if (next == lambdaArrow) {
                        // { ->
                        // { =>
                        this.mode = LAMBDA_TYPE_ARROW;
                        return;
                    }
                    if (next.type() == BaseSymbols.OPEN_PARENTHESIS) {
                        // { ( ... ) =>
                        // { ( ... ) ->
                        pm.skip();
                        pm.pushParser(new ParameterListParser(this.lambdaExpr));
                        this.mode = LAMBDA_PARAMETERS_END;
                        return;
                    }
                    // { ... ->
                    // { ... =>
                    pm.pushParser(new ParameterListParser(this.lambdaExpr).withFlags(LAMBDA_ARROW_END));
                    this.mode = LAMBDA_TYPE_ARROW;
                    return;
                }
                // { ...
                this.statementList = this.closure ? new Closure(token) : new StatementList(token);
                this.mode = EXPRESSION;
                if (type != BaseSymbols.OPEN_CURLY_BRACKET) {
                    pm.report(token, "statement_list.open_brace");
                    pm.reparse();
                }
                return;
            }
        case LAMBDA_PARAMETERS_END:
            this.mode = LAMBDA_TYPE_ARROW;
            if (type != BaseSymbols.CLOSE_PARENTHESIS) {
                pm.report(token, "statement_list.lambda.close_paren");
            }
            return;
        case LAMBDA_TYPE_ARROW:
            if (type == DyvilSymbols.ARROW_RIGHT) {
                pm.pushParser(LambdaOrTupleParser.returnTypeParser(this.lambdaExpr));
                this.mode = LAMBDA_RETURN_ARROW;
                return;
            }
        // Fallthrough
        case LAMBDA_RETURN_ARROW:
            if (type != DyvilSymbols.DOUBLE_ARROW_RIGHT) {
                pm.report(token, "statement_list.lambda.arrow");
                return;
            }
            this.mode = EXPRESSION;
            return;
        case EXPRESSION:
            switch(type) {
                case BaseSymbols.SEMICOLON:
                case BaseSymbols.COMMA:
                    return;
                case DyvilKeywords.LABEL:
                    this.mode = LABEL_NAME;
                    return;
            }
            this.mode = SEPARATOR;
            final MemberParser<IVariable> parser = new MemberParser<>(this).withFlags(MemberParser.NO_FIELD_PROPERTIES);
            if (this.tryParserManager.tryParse(pm, parser, token, EXIT_ON_ROOT)) {
                return;
            }
            pm.pushParser(new ExpressionParser(this));
            return;
        case LABEL_NAME:
            if (Tokens.isIdentifier(type)) {
                this.label = token.nameValue();
                this.mode = LABEL_END;
                return;
            }
            this.mode = EXPRESSION;
            if (type != BaseSymbols.COLON) {
                pm.reparse();
            }
            pm.report(token, "statement_list.label.name");
            return;
        case LABEL_END:
            switch(type) {
                case BaseSymbols.COLON:
                case BaseSymbols.SEMICOLON:
                    this.mode = EXPRESSION;
                    return;
            }
            this.mode = EXPRESSION;
            pm.reparse();
            pm.report(SourcePosition.between(token, token.next()), "statement_list.label.separator");
            return;
        case SEPARATOR:
            this.mode = EXPRESSION;
            switch(type) {
                case BaseSymbols.SEMICOLON:
                case BaseSymbols.COMMA:
                    return;
            }
            pm.report(token, "statement_list.semicolon");
    }
}

Example 13 with IToken

use of dyvilx.tools.parsing.token.IToken in project Dyvil by Dyvil.

the class ExpressionParser method parse.

@Override
public void parse(IParserManager pm, IToken token) {
    final int type = token.type();
    switch(type) {
        case Tokens.EOF:
        case BaseSymbols.SEMICOLON:
        case BaseSymbols.COMMA:
        case Tokens.STRING_PART:
        case Tokens.STRING_END:
            this.end(pm, true);
            return;
    }
    switch(this.mode) {
        case END:
            this.end(pm, true);
            return;
        case VALUE:
            if ((type & Tokens.IDENTIFIER) != 0) {
                // IDENTIFIER
                this.parseInfixAccess(pm, token, token.nameValue());
                return;
            }
            if (this.parseValue(pm, token, type)) {
                // STATEMENT
                return;
            }
        // Fallthrough
        case ACCESS:
            switch(type) {
                case BaseSymbols.DOT:
                    this.mode = DOT_ACCESS;
                    return;
                case DyvilSymbols.ARROW_RIGHT:
                case DyvilSymbols.DOUBLE_ARROW_RIGHT:
                    if (!this.hasFlag(IGNORE_LAMBDA)) {
                        break;
                    }
                // Fallthrough
                case DyvilKeywords.ELSE:
                case DyvilKeywords.CATCH:
                case DyvilKeywords.FINALLY:
                case DyvilKeywords.WHILE:
                    this.end(pm, true);
                    return;
                case DyvilKeywords.AS:
                    {
                        // EXPRESSION as
                        final IToken next = token.next();
                        final boolean optional;
                        if (next.type() == Tokens.SYMBOL_IDENTIFIER && next.nameValue() == Names.qmark && token.isNeighboring(next)) {
                            // EXPRESSION as?
                            optional = true;
                            pm.skip();
                        } else {
                            optional = false;
                        }
                        final CastOperator castOperator = new CastOperator(token.raw(), this.value, optional);
                        pm.pushParser(new TypeParser(castOperator).withFlags(TypeParser.IGNORE_OPERATOR));
                        this.value = castOperator;
                        return;
                    }
                case DyvilKeywords.IS:
                    {
                        // EXPRESSION is
                        final InstanceOfOperator instanceOfOperator = new InstanceOfOperator(token.raw(), this.value);
                        pm.pushParser(new TypeParser(instanceOfOperator).withFlags(TypeParser.IGNORE_OPERATOR));
                        this.value = instanceOfOperator;
                        return;
                    }
                case DyvilKeywords.MATCH:
                    // EXPRESSION match
                    // Parse a match expression
                    // e.g. int1 match { ... }, this match { ... }
                    MatchExpr me = new MatchExpr(token.raw(), this.value);
                    pm.pushParser(new MatchExpressionParser(me));
                    this.value = me;
                    return;
                case BaseSymbols.OPEN_SQUARE_BRACKET:
                    // EXPRESSION [
                    // Parse a subscript access
                    // e.g. this[1], array[0]
                    final SubscriptAccess subscriptAccess = new SubscriptAccess(token, this.value);
                    ArgumentListParser.parseArguments(pm, token.next(), subscriptAccess);
                    this.value = subscriptAccess;
                    this.mode = SUBSCRIPT_END;
                    return;
                case BaseSymbols.OPEN_PARENTHESIS:
                    // EXPRESSION (
                    // Parse an apply call
                    // e.g. 1("a"), this("stuff"), "myString"(2)
                    final ApplyAccess applyAccess = new ApplyAccess(this.value.getPosition(), this.value);
                    ArgumentListParser.parseArguments(pm, token.next(), applyAccess);
                    this.value = applyAccess;
                    this.mode = PARAMETERS_END;
                    return;
                case BaseSymbols.COLON:
                    if (this.hasFlag(IGNORE_COLON)) {
                        this.end(pm, true);
                        return;
                    }
                    this.parseInfixAccess(pm, token, Names.colon, true);
                    return;
                case BaseSymbols.EQUALS:
                    this.parseInfixAccess(pm, token, Names.eq, true);
                    return;
            }
            if (isExpressionEnd(type)) {
                // ... ]
                // Close bracket, end expression
                this.end(pm, true);
                return;
            }
            if (isSymbolic(type)) {
                // EXPRESSION IDENTIFIER
                // EXPRESSION SYMBOL
                this.parseInfixAccess(pm, token);
                return;
            }
            if (this.value != null) {
                // EXPRESSION EXPRESSION -> EXPRESSION ( EXPRESSION )
                // Juxtaposition
                this.parseApply(pm, token);
                return;
            }
            pm.report(Markers.syntaxError(token, "expression.invalid", token.toString()));
            return;
        case DOT_ACCESS:
            if (type == BaseSymbols.OPEN_CURLY_BRACKET) {
                // EXPRESSION . {
                final BraceAccessExpr braceAccessExpr = new BraceAccessExpr(token.raw(), this.value);
                pm.pushParser(new StatementListParser(braceAccessExpr::setStatement), true);
                this.value = braceAccessExpr;
                this.mode = ACCESS;
                return;
            }
            if (Tokens.isIdentifier(type)) {
                if (this.value == null) {
                    // .IDENTIFIER
                    this.value = new EnumValue(token.raw(), token.nameValue());
                    this.mode = ACCESS;
                    return;
                }
                // EXPRESSION . IDENTIFIER
                this.parseInfixAccess(pm, token, token.nameValue());
                return;
            }
            pm.report(Markers.syntaxError(token, "expression.access.dot.invalid"));
            if (BaseSymbols.isTerminator(type)) {
                pm.popParser(true);
                return;
            }
            this.mode = ACCESS;
            pm.reparse();
            return;
        case PARAMETERS_END:
            // ... ( ... )
            // ^
            this.mode = ACCESS;
            this.value.expandPosition(token);
            if (type != BaseSymbols.CLOSE_PARENTHESIS) {
                pm.reparse();
                pm.report(token, "method.call.close_paren");
            }
            return;
        case SUBSCRIPT_END:
            // ... [ ... ]
            // ^
            this.mode = ACCESS;
            this.value.expandPosition(token);
            if (type != BaseSymbols.CLOSE_SQUARE_BRACKET) {
                pm.reparse();
                pm.report(token, "method.subscript.close_bracket");
            }
            return;
        case TYPE_ARGUMENTS_END:
            if (!TypeParser.isGenericEnd(token, type)) {
                pm.reparse();
                pm.report(token, "method.call.generic.close_angle");
            }
            pm.splitJump(token, 1);
            final IToken next = token.next();
            if (next.type() == BaseSymbols.OPEN_PARENTHESIS) {
                pm.skip();
                ArgumentListParser.parseArguments(pm, next.next(), (ICall) this.value);
                this.mode = PARAMETERS_END;
                return;
            }
            this.mode = ACCESS;
            return;
    }
    throw new Error("unreachable");
}

Example 14 with IToken

use of dyvilx.tools.parsing.token.IToken in project Dyvil by Dyvil.

the class ExpressionParser method parseInfixAccess.

private void parseInfixAccess(IParserManager pm, IToken token, Name name, boolean forceInfix) {
    final int type = token.type();
    final IToken next = token.next();
    final int nextType = next.type();
    if (isSymbolic(type)) {
        // Identifier is an operator
        final IToken prev = token.prev();
        final boolean leftNeighbor = prev.isNeighboring(token);
        final boolean rightNeighbor = token.isNeighboring(next);
        if (// prefix
        this.value == null) {
            if (// only true iff this.value == null
            forceInfix) {
                pm.report(SourcePosition.before(token), "expression.infix.before");
            }
            // OPERATOR EXPRESSION
            // token    next
            this.mode = ACCESS;
            if (this.isOperatorEnd(nextType)) {
                this.value = new FieldAccess(token.raw(), null, name);
                return;
            }
            final PrefixCall call = new PrefixCall(token.raw(), name);
            this.value = call;
            this.parseApply(pm, next, call);
            return;
        } else if (!forceInfix && !leftNeighbor && rightNeighbor) {
            // Revert to Juxtaposition
            this.parseApply(pm, token);
            return;
        }
        if (this.isOperatorEnd(nextType) || !forceInfix && leftNeighbor && !rightNeighbor) {
            if (// only true iff this.isOperatorEnd(nextType)
            forceInfix) {
                pm.report(SourcePosition.after(token), "expression.infix.after");
            }
            // EXPRESSION_OPERATOR EXPRESSION
            // EXPRESSION OPERATOR EOF
            // token    next
            this.value = new PostfixCall(token.raw(), this.value, name);
            this.mode = ACCESS;
            return;
        }
        if (this.hasFlag(IGNORE_OPERATOR)) {
            this.valueConsumer.setValue(this.value);
            pm.popParser(true);
            return;
        }
        // EXPRESSION OPERATOR EXPRESSION
        // token    next
        final InfixCallChain chain;
        if (this.value.valueTag() == IValue.OPERATOR_CHAIN) {
            chain = (InfixCallChain) this.value;
        } else {
            chain = new InfixCallChain();
            chain.addOperand(this.value);
            this.value = chain;
        }
        chain.addOperator(name, token.raw());
        pm.pushParser(new ExpressionParser(chain::addOperand).withFlags(this.flags | IGNORE_OPERATOR));
        return;
    }
    switch(nextType) {
        case BaseSymbols.OPEN_PARENTHESIS:
            {
                // IDENTIFIER (
                final MethodCall call = new MethodCall(token.raw(), this.value, name);
                ArgumentListParser.parseArguments(pm, next.next(), call);
                this.value = call;
                this.mode = PARAMETERS_END;
                pm.skip();
                return;
            }
        case BaseSymbols.OPEN_SQUARE_BRACKET:
            {
                // IDENTIFIER [
                final FieldAccess fieldAccess = new FieldAccess(token.raw(), this.value, name);
                final SubscriptAccess subscriptAccess = new SubscriptAccess(next.raw(), fieldAccess);
                ArgumentListParser.parseArguments(pm, next.next(), subscriptAccess);
                this.value = subscriptAccess;
                this.mode = SUBSCRIPT_END;
                pm.skip();
                return;
            }
        case DyvilSymbols.ARROW_RIGHT:
        case DyvilSymbols.DOUBLE_ARROW_RIGHT:
            if (this.hasFlag(IGNORE_LAMBDA)) {
                break;
            }
            // IDENTIFIER =>   ...
            // token      next
            // Lambda Expression with one untyped parameter
            pm.pushParser(new LambdaOrTupleParser(this, LambdaOrTupleParser.SINGLE_PARAMETER), true);
            this.mode = END;
            return;
    }
    if (isGenericCall(next, nextType)) {
        final MethodCall call = new MethodCall(token.raw(), this.value, name, ArgumentList.EMPTY);
        this.value = call;
        pm.splitJump(next, 1);
        pm.pushParser(new TypeListParser(call.getGenericData().getTypes(), true));
        this.mode = TYPE_ARGUMENTS_END;
        return;
    }
    if (this.isFieldAccess(token, next, nextType)) {
        this.value = new FieldAccess(token.raw(), this.value, name);
        this.mode = ACCESS;
        return;
    }
    // IDENTIFIER EXPRESSION
    // token      next
    // Parse a single-argument call
    // e.g. println "abc"
    // println -1
    // println i
    final MethodCall call = new MethodCall(token.raw(), this.value, name, ArgumentList.empty());
    this.value = call;
    this.mode = ACCESS;
    this.parseApply(pm, token.next(), call);
}

Example 15 with IToken

use of dyvilx.tools.parsing.token.IToken in project Dyvil by Dyvil.

the class LambdaOrTupleParser method parse.

@Override
public void parse(IParserManager pm, IToken token) {
    final int type = token.type();
    switch(this.mode) {
        case OPEN_PARENTHESIS:
            // ( ...
            // ^
            /*
			 * The new version of this parser tries to find the matching closing parenthesis instead of try-parsing the
			 * expression. If it finds that parenthesis token and the next token is a lambda arrow, we can assume that
			 * the expression is a lambda expression. Thus, we directly push a Parameter List Parser that may also
			 * produce syntax errors.
			 */
            final IToken closeParen = BracketMatcher.findMatch(token);
            if (closeParen != null) {
                final IToken next = closeParen.next();
                final int nextType = next.type();
                if (nextType == DyvilSymbols.ARROW_RIGHT || nextType == DyvilSymbols.DOUBLE_ARROW_RIGHT) {
                    // (     ... )          =>
                    // (     ... )          ->
                    // token     closeParen next
                    final LambdaExpr lambdaExpr = new LambdaExpr(next);
                    pm.pushParser(new ParameterListParser(lambdaExpr));
                    this.value = lambdaExpr;
                    this.mode = PARAMETERS_END;
                    return;
                }
            }
        // Fallthrough
        case TUPLE:
            // ( ... )
            final TupleLikeExpr tupleExpr = new TupleLikeExpr(token);
            pm.pushParser(new ArgumentListParser(tupleExpr));
            this.value = tupleExpr;
            this.mode = TUPLE_END;
            return;
        case TUPLE_END:
            this.value.expandPosition(token);
            this.consumer.setValue(this.value);
            pm.popParser();
            if (type != BaseSymbols.CLOSE_PARENTHESIS) {
                pm.reparse();
                pm.report(token, "tuple.close_paren");
            }
            return;
        case PARAMETERS_END:
            this.mode = TYPE_ARROW;
            if (type != BaseSymbols.CLOSE_PARENTHESIS) {
                pm.reparse();
                pm.report(token, "lambda.close_paren");
            }
            return;
        case SINGLE_PARAMETER:
            if (Tokens.isIdentifier(type)) {
                final LambdaExpr lambdaExpr = new LambdaExpr(token.next());
                final IParameter parameter = lambdaExpr.createParameter(token.raw(), token.nameValue(), Types.UNKNOWN, new AttributeList());
                lambdaExpr.getParameters().add(parameter);
                this.value = lambdaExpr;
                this.mode = TYPE_ARROW;
                return;
            }
        // Fallthrough
        case TYPE_ARROW:
            if (this.value == null) {
                this.value = new LambdaExpr(token.raw());
            }
            if (type == DyvilSymbols.ARROW_RIGHT) {
                pm.pushParser(returnTypeParser((LambdaExpr) this.value));
                this.mode = RETURN_ARROW;
                return;
            }
        // Fallthrough
        case RETURN_ARROW:
            pm.pushParser(new ExpressionParser(((LambdaExpr) this.value)));
            this.mode = END;
            if (type != DyvilSymbols.DOUBLE_ARROW_RIGHT) {
                pm.reparse();
                pm.report(token, "lambda.arrow");
            }
            return;
        case END:
            pm.popParser(true);
            this.consumer.setValue(this.value);
    }
}