Examples with IToken dyvilx.tools.parsing.token.IToken used on opensource projects

Example 6 with IToken

use of dyvilx.tools.parsing.token.IToken in project Dyvil by Dyvil.

the class ExpressionParser method isGenericCall.

public static boolean isGenericCall(IToken token, int tokenType) {
    if (!TypeParser.isGenericStart(token, tokenType)) {
        // IDENTIFIER
        return false;
    }
    // IDENTIFIER <
    final IToken endToken = BracketMatcher.findMatch(token, true);
    if (endToken == null) {
        // IDENTIFIER < ...
        return false;
    }
    final IToken endTokenNext = endToken.next();
    final int endTokenNextType = endTokenNext.type();
    if (!TypeParser.isGenericEnd2(endToken)) {
        // Return true iff the end token and the next token are NOT separated by whitespace
        return endToken.isNeighboring(endTokenNext);
    }
    if (isExpressionEnd(endTokenNextType)) {
        // IDENTIFIER < ... SYMBOL> END
        return true;
    }
    switch(endTokenNextType) {
        case Tokens.SYMBOL:
        case Tokens.SYMBOL_IDENTIFIER:
            // The end token is followed by another symbol token, but they are separated by whitespace
            // IDENTIFIER < ... > SYMBOL
            // IDENTIFIER < ... SYMBOL> SYMBOL
            // Return true iff the symbol token is either followed by a token that ends an expression
            // or separated from the next token via whitespace.
            final IToken endTokenNextNext = endTokenNext.next();
            return isExpressionEnd(endTokenNextNext.type()) || !endTokenNext.isNeighboring(endTokenNextNext);
        case BaseSymbols.OPEN_CURLY_BRACKET:
            // IDENTIFIER < ... > {
            return true;
        case BaseSymbols.OPEN_PARENTHESIS:
        case BaseSymbols.OPEN_SQUARE_BRACKET:
            // IDENTIFIER < ... >[
            return endToken.isNeighboring(endTokenNext);
    }
    return false;
}

Example 7 with IToken

use of dyvilx.tools.parsing.token.IToken in project Dyvil by Dyvil.

the class ImportParser method parse.

@Override
public void parse(IParserManager pm, IToken token) {
    final int type = token.type();
    switch(this.mode) {
        case IMPORT:
            switch(type) {
                case BaseSymbols.OPEN_CURLY_BRACKET:
                    {
                        final MultiImport multiImport = new MultiImport(token);
                        multiImport.setParent(this.theImport);
                        this.theImport = multiImport;
                        if (token.next().type() == BaseSymbols.CLOSE_CURLY_BRACKET) {
                            // Fast-path; import ... { }
                            this.mode = END;
                            pm.skip();
                            return;
                        }
                        pm.pushParser(new ImportListParser(multiImport));
                        this.mode = MULTI_IMPORT_END;
                        return;
                    }
                case DyvilSymbols.UNDERSCORE:
                    {
                        final WildcardImport wildcardImport = new WildcardImport(token.raw());
                        wildcardImport.setParent(this.theImport);
                        this.theImport = wildcardImport;
                        this.mode = END;
                        return;
                    }
                case Tokens.IDENTIFIER:
                case Tokens.SPECIAL_IDENTIFIER:
                case Tokens.SYMBOL_IDENTIFIER:
                case Tokens.LETTER_IDENTIFIER:
                    {
                        final SingleImport singleImport = new SingleImport(token.raw(), token.nameValue());
                        singleImport.setParent(this.theImport);
                        this.theImport = singleImport;
                        this.mode = DOT_ALIAS;
                        return;
                    }
            }
            final int mask = KindedImport.parseMask(type);
            if (mask != 0) {
                this.masks |= mask;
                return;
            }
            pm.report(token, "import.identifier");
            if (BaseSymbols.isTerminator(type)) {
                pm.popParser(type != Tokens.EOF);
            }
            return;
        case DOT_ALIAS:
            switch(type) {
                case BaseSymbols.DOT:
                    this.mode = IMPORT;
                    return;
                case Tokens.SYMBOL_IDENTIFIER:
                    if (token.nameValue().unqualified.equals(".*")) {
                        // Handle Java-style wildcard imports gracefully
                        pm.report(Markers.syntaxWarning(token, "import.wildcard.java"));
                        final WildcardImport wildcardImport = new WildcardImport(token.raw());
                        wildcardImport.setParent(this.theImport);
                        this.theImport = wildcardImport;
                        this.mode = END;
                        return;
                    }
                    // create an error
                    break;
                case DyvilSymbols.DOUBLE_ARROW_RIGHT:
                case DyvilKeywords.AS:
                    this.mode = END;
                    final IToken next = token.next();
                    if (Tokens.isIdentifier(next.type())) {
                        this.theImport.setAlias(next.nameValue());
                        pm.skip();
                        return;
                    }
                    pm.report(next, "import.alias.identifier");
                    return;
                case BaseSymbols.COMMA:
                case BaseSymbols.SEMICOLON:
                case BaseSymbols.CLOSE_CURLY_BRACKET:
                case BaseSymbols.CLOSE_PARENTHESIS:
                    this.end();
                    pm.popParser(true);
                    return;
                case Tokens.EOF:
                    this.end();
                    pm.popParser();
                    return;
            }
            pm.report(token, "import.dot");
            return;
        case MULTI_IMPORT_END:
            this.theImport.expandPosition(token);
            this.end();
            pm.popParser();
            if (type != BaseSymbols.CLOSE_CURLY_BRACKET) {
                pm.reparse();
                pm.report(token, "import.multi.close_brace");
            }
            return;
        case END:
            this.end();
            pm.popParser(type != Tokens.EOF);
    }
}

Example 8 with IToken

use of dyvilx.tools.parsing.token.IToken in project Dyvil by Dyvil.

the class ParameterListParser method parse.

@Override
public void parse(IParserManager pm, IToken token) {
    final int type = token.type();
    switch(this.mode) {
        case DECLARATOR:
            switch(type) {
                case BaseSymbols.SEMICOLON:
                    if (token.isInferred()) {
                        return;
                    }
                    break;
                case DyvilKeywords.LET:
                    this.attributes.addFlag(Modifiers.FINAL);
                // Fallthrough
                case DyvilKeywords.VAR:
                    this.mode = NAME;
                    return;
                case DyvilKeywords.THIS:
                    if (token.next().type() != BaseSymbols.COLON) {
                        pm.report(token, "parameter.identifier");
                        this.mode = SEPARATOR;
                        return;
                    }
                    // this : TYPE
                    this.mode = TYPE_ASCRIPTION;
                    // the colon
                    pm.skip();
                    pm.pushParser(new TypeParser(t -> this.setThisType(t, token, pm)));
                    return;
                case DyvilSymbols.AT:
                    final Annotation annotation = new CodeAnnotation(token.raw());
                    this.attributes.add(annotation);
                    pm.pushParser(new AnnotationParser(annotation));
                    return;
            }
            final Modifier modifier;
            if ((modifier = ModifierParser.parseModifier(token, pm)) != null) {
                this.attributes.add(modifier);
                return;
            }
            if (BaseSymbols.isCloseBracket(type)) {
                pm.popParser(true);
                return;
            }
        // Fallthrough
        case NAME:
            final Name name;
            if (Tokens.isIdentifier(type)) {
                name = token.nameValue();
            } else if (type == BaseSymbols.UNDERSCORE) {
                name = null;
            } else if (Tokens.isKeyword(type)) {
                name = Name.fromRaw(token.stringValue());
            } else {
                if (BaseSymbols.isCloseBracket(type)) {
                    pm.popParser(true);
                }
                if (type == Tokens.EOF) {
                    pm.popParser();
                }
                this.mode = SEPARATOR;
                pm.report(token, "parameter.identifier");
                return;
            }
            this.parameter = this.consumer.createParameter(token.raw(), name, this.type, this.attributes);
            this.mode = INTERNAL_NAME;
            return;
        case INTERNAL_NAME:
            this.mode = VARARGS_AFTER_NAME;
            // overwrite the internal name if necessary
            if (Tokens.isIdentifier(type)) {
                // IDENTIFIER IDENTIFIER : TYPE
                this.parameter.setName(token.nameValue());
                return;
            } else if (type == BaseSymbols.UNDERSCORE) {
                // IDENTIFIER _ : TYPE
                this.parameter.setName(null);
                return;
            }
        // Fallthrough
        case VARARGS_AFTER_NAME:
            if (type == DyvilSymbols.ELLIPSIS) {
                this.flags |= VARARGS;
                this.mode = TYPE_ASCRIPTION;
                return;
            }
        // Fallthrough
        case TYPE_ASCRIPTION:
        case VARARGS_AFTER_POST_TYPE:
            // continue with the remaining cases (DEFAULT_VALUE, PROPERTY, ...).
            if (this.mode == VARARGS_AFTER_POST_TYPE) {
                // case (2)
                if (type == DyvilSymbols.ELLIPSIS) {
                    this.setTypeVarargs();
                    this.mode = DEFAULT_VALUE;
                    return;
                }
            } else /* case (1) */
            if (type == BaseSymbols.COLON) {
                this.mode = VARARGS_AFTER_POST_TYPE;
                final TypeParser parser = new TypeParser(this);
                if (this.hasFlag(LAMBDA_ARROW_END)) {
                    parser.withFlags(TypeParser.IGNORE_LAMBDA);
                }
                pm.pushParser(parser);
                return;
            }
        // Fallthrough
        case DEFAULT_VALUE:
            if (type == BaseSymbols.EQUALS) {
                this.mode = PROPERTY;
                pm.pushParser(new ExpressionParser(this.parameter));
                return;
            }
        // Fallthrough
        case PROPERTY:
            if (type == BaseSymbols.OPEN_CURLY_BRACKET && this.hasFlag(ALLOW_PROPERTIES)) {
                final IProperty property = this.parameter.createProperty();
                pm.pushParser(new PropertyBodyParser(property), true);
                this.mode = SEPARATOR;
                return;
            }
        // Fallthrough
        case SEPARATOR:
            this.mode = DECLARATOR;
            if (this.parameter != null) {
                if (this.hasFlag(VARARGS)) {
                    this.parameter.setVarargs();
                }
                this.parameter.setType(this.type);
                this.consumer.getParameters().add(this.parameter);
            }
            this.reset();
            switch(type) {
                case DyvilSymbols.ARROW_RIGHT:
                case DyvilSymbols.DOUBLE_ARROW_RIGHT:
                    if (!this.hasFlag(LAMBDA_ARROW_END)) {
                        // produce a syntax error
                        break;
                    }
                // Fallthrough
                case BaseSymbols.CLOSE_PARENTHESIS:
                case BaseSymbols.CLOSE_CURLY_BRACKET:
                case BaseSymbols.CLOSE_SQUARE_BRACKET:
                    pm.reparse();
                // Fallthrough
                case Tokens.EOF:
                    pm.popParser();
                    return;
                case BaseSymbols.COMMA:
                case BaseSymbols.SEMICOLON:
                    return;
            }
            pm.report(token, "parameter.separator");
    }
}

Example 9 with IToken

use of dyvilx.tools.parsing.token.IToken in project Dyvil by Dyvil.

the class TypeParser method parse.

@Override
public void parse(IParserManager pm, IToken token) {
    final int type = token.type();
    switch(this.mode) {
        case NAME:
            {
                if ((this.flags & NAMED_ONLY) == 0) {
                    switch(type) {
                        case DyvilSymbols.AT:
                            Annotation a = new CodeAnnotation(token.raw());
                            pm.pushParser(new AnnotationParser(a));
                            this.type = new AnnotatedType(a);
                            this.mode = ANNOTATION_END;
                            return;
                        case BaseSymbols.OPEN_PARENTHESIS:
                            {
                                final TypeList arguments;
                                if (this.parentType != null) {
                                    final LambdaType lambdaType = new LambdaType();
                                    lambdaType.setExtension(true);
                                    this.type = lambdaType;
                                    arguments = lambdaType.getArguments();
                                    arguments.add(this.parentType);
                                } else {
                                    final TupleType tupleType = new TupleType();
                                    this.type = tupleType;
                                    arguments = tupleType.getArguments();
                                }
                                pm.pushParser(new TypeListParser(arguments));
                                this.mode = TUPLE_END;
                                return;
                            }
                        case BaseSymbols.OPEN_SQUARE_BRACKET:
                            {
                                final ArrayType arrayType = new ArrayType();
                                switch(token.next().type()) {
                                    case DyvilKeywords.FINAL:
                                        arrayType.setMutability(Mutability.IMMUTABLE);
                                        pm.skip();
                                        break;
                                    case DyvilKeywords.VAR:
                                        arrayType.setMutability(Mutability.MUTABLE);
                                        pm.skip();
                                        break;
                                }
                                this.mode = ARRAY_COLON;
                                this.type = arrayType;
                                pm.pushParser(new TypeParser(arrayType::setElementType));
                                return;
                            }
                        case DyvilSymbols.ARROW_RIGHT:
                            {
                                if ((this.flags & IGNORE_LAMBDA) != 0) {
                                    pm.popParser(true);
                                    return;
                                }
                                final LambdaType lambdaType = new LambdaType(token.raw());
                                final TypeList arguments = lambdaType.getArguments();
                                if (this.parentType != null) {
                                    arguments.add(this.parentType);
                                }
                                pm.pushParser(this.subParser(arguments));
                                this.type = lambdaType;
                                this.mode = LAMBDA_END;
                                return;
                            }
                        case DyvilKeywords.NULL:
                            this.type = Types.NULL;
                            this.mode = END;
                            return;
                        case DyvilSymbols.UNDERSCORE:
                            this.type = new WildcardType(token.raw(), Variance.COVARIANT);
                            this.mode = END;
                            return;
                        case Tokens.SYMBOL_IDENTIFIER:
                            final Name name = token.nameValue();
                            final int closeAngleIndex;
                            if ((this.flags & CLOSE_ANGLE) == 0 || (closeAngleIndex = name.unqualified.indexOf('>')) < 0) {
                                // SYMBOL_IDENTIFIER type
                                final PrefixType prefixType = new PrefixType(token.raw(), name);
                                pm.pushParser(this.subParser(prefixType.getArguments()).withFlags(IGNORE_OPERATOR | IGNORE_LAMBDA));
                                this.type = prefixType;
                                this.mode = END;
                                return;
                            }
                            if (closeAngleIndex == 0) {
                                // Token starts with a >
                                // Handles Type< > gracefully
                                pm.popParser(true);
                                return;
                            }
                            // strip the trailing > and reparse the first part of the token
                            // Handles Type<_> gracefully
                            pm.splitReparse(token, closeAngleIndex);
                            return;
                    }
                }
                if (!Tokens.isIdentifier(type)) {
                    if (isTerminator(type)) {
                        pm.popParser(true);
                        return;
                    }
                    pm.report(Markers.syntaxError(token, "type.invalid", token.toString()));
                    return;
                }
                final Name name = token.nameValue();
                final IToken next = token.next();
                if (isGenericStart(next, next.type())) {
                    this.type = new NamedGenericType(token.raw(), this.parentType, name);
                    this.mode = GENERICS;
                    return;
                }
                this.type = new NamedType(token.raw(), name, this.parentType);
                this.mode = END;
                return;
            }
        case TUPLE_END:
            {
                if (type != BaseSymbols.CLOSE_PARENTHESIS) {
                    pm.reparse();
                    pm.report(token, "type.tuple.close_paren");
                }
                final IToken nextToken = token.next();
                if (nextToken.type() == DyvilSymbols.ARROW_RIGHT) {
                    final LambdaType lambdaType;
                    if (this.type instanceof LambdaType) {
                        lambdaType = (LambdaType) this.type;
                    } else {
                        lambdaType = new LambdaType(nextToken.raw(), ((TupleType) this.type).getArguments());
                        this.type = lambdaType;
                    }
                    lambdaType.setPosition(nextToken);
                    this.mode = LAMBDA_END;
                    pm.skip();
                    pm.pushParser(this.subParser(lambdaType.getArguments()));
                    return;
                }
                if (this.parentType != null) {
                    pm.report(nextToken, "type.tuple.lambda_arrow");
                }
                this.type.expandPosition(token);
                this.mode = END;
                return;
            }
        case LAMBDA_END:
            this.type.expandPosition(token.prev());
            this.consumer.setType(this.type);
            pm.popParser(true);
            return;
        case ARRAY_COLON:
            if (type == BaseSymbols.COLON) {
                final MapType mapType = new MapType(this.type.getMutability(), ((ArrayType) this.type).getElementType());
                this.type = mapType;
                this.mode = ARRAY_END;
                pm.pushParser(new TypeParser(mapType.getArguments()));
                return;
            }
        // Fallthrough
        case ARRAY_END:
            this.type.expandPosition(token);
            this.mode = END;
            if (type != BaseSymbols.CLOSE_SQUARE_BRACKET) {
                pm.reparse();
                pm.report(token, "type.array.close_bracket");
            }
            return;
        case GENERICS:
            if (isGenericStart(token, type)) {
                pm.splitJump(token, 1);
                pm.pushParser(new TypeListParser(((GenericType) this.type).getArguments(), true));
                this.mode = GENERICS_END;
                return;
            }
            return;
        case ANNOTATION_END:
            this.mode = END;
            pm.pushParser(this.subParser((ITyped) this.type), true);
            return;
        case GENERICS_END:
            this.mode = END;
            if (isGenericEnd(token, type)) {
                pm.splitJump(token, 1);
                return;
            }
            pm.report(token, "type.generic.close_angle");
        // Fallthrough
        case END:
            {
                switch(type) {
                    case BaseSymbols.DOT:
                        pm.pushParser(new TypeParser(this, this.type, this.flags));
                        return;
                    case BaseSymbols.OPEN_SQUARE_BRACKET:
                        {
                            final IToken next = token.next();
                            if (next.type() == BaseSymbols.CLOSE_SQUARE_BRACKET) {
                                this.type = new ArrayType(this.type);
                                pm.report(Markers.syntaxWarning(token.to(next), "type.array.java"));
                                pm.skip();
                                return;
                            }
                            break;
                        }
                    case Tokens.SYMBOL_IDENTIFIER:
                        {
                            if ((this.flags & NAMED_ONLY) != 0) {
                                break;
                            }
                            if ((this.flags & CLOSE_ANGLE) != 0) {
                                final String string = token.stringValue();
                                int index = string.indexOf('>');
                                if (index == 0) {
                                    // ... >
                                    pm.splitJump(token, 1);
                                    break;
                                } else if (index > 0) {
                                    // ... SYMBOL>
                                    pm.splitJump(token, index);
                                    this.type = new PostfixType(token.raw(), Name.fromUnqualified(string.substring(0, index)), this.type);
                                    return;
                                }
                            }
                            final IToken next = token.next();
                            final boolean leftNeighbor = token.prev().isNeighboring(token);
                            final boolean rightNeighbor = token.isNeighboring(next);
                            if (isTerminator(next.type()) || leftNeighbor && !rightNeighbor) {
                                // type_OPERATOR
                                this.type = new PostfixType(token.raw(), token.nameValue(), this.type);
                                // move stays END
                                return;
                            }
                            if (leftNeighbor != rightNeighbor || (this.flags & IGNORE_OPERATOR) != 0) {
                                // type end
                                break;
                            }
                            // Parse part of an infix operator
                            // type SYMBOL type
                            // type_SYMBOL_type
                            final InfixTypeChain chain;
                            if (this.type.typeTag() == IType.INFIX_CHAIN) {
                                chain = (InfixTypeChain) this.type;
                            } else {
                                chain = new InfixTypeChain();
                                chain.addOperand(this.type);
                                this.type = chain;
                            }
                            chain.addOperator(token.nameValue(), token.raw());
                            pm.pushParser(this.subParser(chain::addOperand).withFlags(IGNORE_OPERATOR));
                            return;
                        }
                    case DyvilSymbols.ARROW_RIGHT:
                        // all these flags have to be unset
                        if (this.parentType == null && (this.flags & (NAMED_ONLY | IGNORE_OPERATOR | IGNORE_LAMBDA)) == 0) {
                            final LambdaType lambdaType = new LambdaType(token.raw(), this.type);
                            this.type = lambdaType;
                            this.mode = LAMBDA_END;
                            pm.pushParser(this.subParser(lambdaType.getArguments()));
                            return;
                        }
                        break;
                }
                if (this.type != null) {
                    this.consumer.setType(this.type);
                }
                pm.popParser(true);
            }
    }
}

Example 10 with IToken

use of dyvilx.tools.parsing.token.IToken in project Dyvil by Dyvil.

the class SemicolonInference method inferSemicolons.

public static void inferSemicolons(IToken first) {
    if (first == null) {
        return;
    }
    IToken prev = first;
    IToken next = first.next();
    while (next.type() != Tokens.EOF) {
        next.setPrev(prev);
        if (inferSemicolon(prev, next)) {
            final IToken semicolon = new InferredSemicolon(prev.endLine(), prev.endColumn());
            semicolon.setNext(next);
            semicolon.setPrev(prev);
            next.setPrev(semicolon);
            prev.setNext(semicolon);
        }
        prev = next;
        next = next.next();
    }
}