Examples with IToken dyvilx.tools.parsing.token.IToken used on opensource projects

Example 1 with IToken

use of dyvilx.tools.parsing.token.IToken in project Dyvil by Dyvil.

the class Colorizer method colorize.

public static String colorize(String text, REPLContext context) {
    final TextSource source = new TextSource(text);
    final TokenList tokens = new DyvilLexer(new MarkerList(Markers.INSTANCE), DyvilSymbols.INSTANCE).tokenize(text);
    // Split into lines
    final int lineCount = source.lineCount();
    final StringBuilder[] lines = new StringBuilder[lineCount];
    for (int i = 0; i < lineCount; i++) {
        lines[i] = new StringBuilder(source.line(i + 1));
    }
    // iterate, starting from the last token
    for (IToken token = tokens.last(); token != null && token.type() != Tokens.EOF; token = token.prev()) {
        final String color = tokenColor(token, context);
        if (color != null) {
            // insert ANSI color codes before and after the token
            final StringBuilder line = lines[token.startLine() - 1];
            line.insert(token.endColumn(), Console.ANSI_RESET);
            line.insert(token.startColumn(), color);
        }
    }
    // Merge back together
    final StringBuilder first = lines[0];
    for (int i = 1; i < lineCount; i++) {
        first.append('\n').append(lines[i]);
    }
    return first.toString();
}

Example 2 with IToken

use of dyvilx.tools.parsing.token.IToken in project Dyvil by Dyvil.

the class ParserManager method parse.

public void parse(Parser parser) {
    this.parser = parser;
    IToken token = null;
    while (!this.hasStopped) {
        if (this.reparse) {
            this.reparse = false;
        } else {
            token = this.tokens.next();
            if (token.type() == Tokens.EOF) {
                break;
            }
        }
        if (this.skip > 0) {
            this.skip--;
            continue;
        }
        if (this.parser == null) {
            this.reportUnparsed(token);
            continue;
        }
        this.tryParse(token, this.parser);
    }
    this.parseRemaining(token);
}

Example 3 with IToken

use of dyvilx.tools.parsing.token.IToken in project Dyvil by Dyvil.

the class TryParserManager method setNextAndReset.

private void setNextAndReset(IToken token) {
    this.reset();
    this.tokens.setNext(token);
    if (this.splitTokens == null || this.splitTokens.isEmpty()) {
        return;
    }
    // Restore all tokens that have been split
    for (IToken splitToken : this.splitTokens) {
        // The original tokens prev and next fields have not been updated by the split method
        splitToken.prev().setNext(splitToken);
        splitToken.next().setPrev(splitToken);
    }
    this.splitTokens.clear();
}

Example 4 with IToken

use of dyvilx.tools.parsing.token.IToken in project Dyvil by Dyvil.

the class TryParserManager method parse.

public boolean parse(Parser parser, MarkerList markers, int flags) {
    this.parser = parser;
    this.hasSyntaxErrors = false;
    this.markers = new MarkerList(markers.getI18n());
    this.reportErrors = (flags & REPORT_ERRORS) != 0;
    IToken token = null;
    while (true) {
        if (this.reparse) {
            this.reparse = false;
        } else {
            token = this.tokens.next();
            if (token.type() == Tokens.EOF) {
                break;
            }
        }
        if (this.skip > 0) {
            this.skip--;
            continue;
        }
        if (this.parser == null) {
            if ((flags & EXIT_ON_ROOT) != 0) {
                return this.success(markers);
            }
            this.reportUnparsed(token);
            continue;
        }
        if (!this.reportErrors && this.parser.reportErrors()) {
            if (this.hasSyntaxErrors) {
                return this.success(markers);
            }
            this.reportErrors = true;
        }
        try {
            this.parser.parse(this, token);
        } catch (Exception ex) {
            this.reportError(token, ex);
            return this.success(markers);
        }
        if (this.hasSyntaxErrors && !this.reportErrors) {
            return this.success(markers);
        }
    }
    this.parseRemaining(token);
    this.reparse = false;
    return this.success(markers);
}

Example 5 with IToken

use of dyvilx.tools.parsing.token.IToken in project Dyvil by Dyvil.

the class ExpressionParser method parseValue.

private boolean parseValue(IParserManager pm, IToken token, int type) {
    switch(type) {
        case Tokens.STRING:
        case Tokens.VERBATIM_STRING:
            this.value = new StringValue(token.raw(), token.stringValue());
            this.mode = ACCESS;
            return true;
        case Tokens.STRING_START:
            {
                final StringInterpolationExpr stringInterpolation = new StringInterpolationExpr(token);
                this.value = stringInterpolation;
                this.mode = ACCESS;
                pm.pushParser(new StingInterpolationParser(stringInterpolation), true);
                return true;
            }
        case Tokens.SINGLE_QUOTED_STRING:
            this.value = new CharValue(token.raw(), token.stringValue());
            this.mode = ACCESS;
            return true;
        case Tokens.VERBATIM_CHAR:
            this.value = new CharValue(token.raw(), token.stringValue(), true);
            this.mode = ACCESS;
            return true;
        case Tokens.INT:
            this.value = new IntValue(token.raw(), token.intValue());
            this.mode = ACCESS;
            return true;
        case Tokens.LONG:
            this.value = new LongValue(token.raw(), token.longValue());
            this.mode = ACCESS;
            return true;
        case Tokens.FLOAT:
            this.value = new FloatValue(token.raw(), token.floatValue());
            this.mode = ACCESS;
            return true;
        case Tokens.DOUBLE:
            this.value = new DoubleValue(token.raw(), token.doubleValue());
            this.mode = ACCESS;
            return true;
        case DyvilSymbols.UNDERSCORE:
            // _ ...
            this.value = new WildcardValue(token.raw());
            this.mode = ACCESS;
            return true;
        case BaseSymbols.OPEN_PARENTHESIS:
            {
                // ( ...
                final IToken next = token.next();
                if (next.type() != BaseSymbols.CLOSE_PARENTHESIS) {
                    pm.pushParser(new LambdaOrTupleParser(this, this.hasFlag(IGNORE_LAMBDA)), true);
                    this.mode = ACCESS;
                    return true;
                }
                if (!this.hasFlag(IGNORE_LAMBDA)) {
                    final IToken next2 = next.next();
                    final int next2Type = next2.type();
                    if (next2Type == DyvilSymbols.ARROW_RIGHT || next2Type == DyvilSymbols.DOUBLE_ARROW_RIGHT) {
                        // () =>
                        // () ->
                        pm.skip();
                        pm.pushParser(new LambdaOrTupleParser(this, LambdaOrTupleParser.TYPE_ARROW));
                        this.mode = END;
                        return true;
                    }
                }
                // ()
                this.value = new VoidValue(token.to(token.next()));
                pm.skip();
                this.mode = ACCESS;
                return true;
            }
        case BaseSymbols.OPEN_SQUARE_BRACKET:
            // [ ...
            this.mode = ACCESS;
            pm.pushParser(new ArrayLiteralParser(this), true);
            return true;
        case BaseSymbols.OPEN_CURLY_BRACKET:
            // { ...
            this.mode = ACCESS;
            pm.pushParser(new StatementListParser(this), true);
            return true;
        case DyvilSymbols.AT:
            // @ ...
            Annotation a = new CodeAnnotation(token.raw());
            pm.pushParser(new AnnotationParser(a));
            this.value = new AnnotationExpr(a);
            this.mode = END;
            return true;
        case DyvilSymbols.ARROW_RIGHT:
        case DyvilSymbols.DOUBLE_ARROW_RIGHT:
            {
                if (this.hasFlag(IGNORE_LAMBDA)) {
                    pm.popParser(true);
                    return true;
                }
                // => ...
                // -> ...
                pm.pushParser(new LambdaOrTupleParser(this, LambdaOrTupleParser.TYPE_ARROW), true);
                return true;
            }
        case DyvilKeywords.NULL:
            this.value = new NullValue(token.raw());
            this.mode = ACCESS;
            return true;
        case DyvilKeywords.TRUE:
            this.value = new BooleanValue(token.raw(), true);
            this.mode = ACCESS;
            return true;
        case DyvilKeywords.FALSE:
            this.value = new BooleanValue(token.raw(), false);
            this.mode = ACCESS;
            return true;
        case DyvilKeywords.INIT:
            // init ...
            this.mode = ACCESS;
            pm.pushParser(new ThisSuperParser(this), true);
            return true;
        case DyvilKeywords.THIS:
            // this ...
            this.mode = ACCESS;
            pm.pushParser(new ThisSuperParser(this), true);
            return true;
        case DyvilKeywords.SUPER:
            // super ...
            this.mode = ACCESS;
            pm.pushParser(new ThisSuperParser(this), true);
            return true;
        case DyvilKeywords.CLASS:
            // class ...
            this.mode = ACCESS;
            pm.pushParser(new TypeClassParser(this, token, true));
            return true;
        case DyvilKeywords.TYPE:
            // type ...
            this.mode = ACCESS;
            pm.pushParser(new TypeClassParser(this, token, false));
            return true;
        case DyvilKeywords.NEW:
            // new ...
            this.mode = ACCESS;
            final int flags = this.hasFlag(IGNORE_CLOSURE) ? ConstructorCallParser.IGNORE_ANON_CLASS : 0;
            pm.pushParser(new ConstructorCallParser(this).withFlags(flags), true);
            return true;
        case DyvilKeywords.RETURN:
            {
                // return ...
                ReturnStatement returnStatement = new ReturnStatement(token.raw());
                this.value = returnStatement;
                pm.pushParser(new ExpressionParser(returnStatement));
                this.mode = END;
                return true;
            }
        case DyvilKeywords.IF:
            {
                // if ...
                pm.pushParser(new IfStatementParser(this), true);
                this.mode = END;
                return true;
            }
        case DyvilKeywords.ELSE:
            {
                if (!(this.parent instanceof IfStatementParser) && !(this.parent instanceof ExpressionParser)) {
                    pm.report(token, "expression.else");
                    return true;
                }
                this.end(pm, true);
                return true;
            }
        case DyvilKeywords.MATCH:
            {
                // match ...
                final MatchExpr matchExpr = new MatchExpr(token.raw());
                this.value = matchExpr;
                pm.pushParser(new MatchExpressionParser(matchExpr));
                this.mode = END;
                return true;
            }
        case DyvilKeywords.WHILE:
            {
                if (// repeat parent
                this.parent instanceof RepeatStatementParser || // repeat grandparent
                this.parent instanceof ExpressionParser && this.parent.getParent() instanceof RepeatStatementParser) {
                    this.end(pm, true);
                    return true;
                }
                final WhileStatement whileStatement = new WhileStatement(token.raw());
                this.value = whileStatement;
                pm.pushParser(new WhileStatementParser(whileStatement));
                this.mode = END;
                return true;
            }
        case DyvilKeywords.REPEAT:
            {
                // repeat ...
                final RepeatStatement repeatStatement = new RepeatStatement(token.raw());
                this.value = repeatStatement;
                pm.pushParser(new RepeatStatementParser(repeatStatement));
                this.mode = END;
                return true;
            }
        case DyvilKeywords.FOR:
            {
                pm.pushParser(new ForStatementParser(this.valueConsumer, token.raw()));
                this.mode = END;
                return true;
            }
        case DyvilKeywords.BREAK:
            {
                final BreakStatement breakStatement = new BreakStatement(token);
                this.value = breakStatement;
                final IToken next = token.next();
                if (Tokens.isIdentifier(next.type())) {
                    breakStatement.setName(next.nameValue());
                    pm.skip();
                }
                this.mode = END;
                return true;
            }
        case DyvilKeywords.CONTINUE:
            {
                final ContinueStatement continueStatement = new ContinueStatement(token);
                this.value = continueStatement;
                final IToken next = token.next();
                if (Tokens.isIdentifier(next.type())) {
                    continueStatement.setName(next.nameValue());
                    pm.skip();
                }
                this.mode = END;
                return true;
            }
        case DyvilKeywords.GOTO:
            {
                GoToStatement statement = new GoToStatement(token);
                this.value = statement;
                final IToken next = token.next();
                if (Tokens.isIdentifier(next.type())) {
                    statement.setName(next.nameValue());
                    pm.skip();
                }
                this.mode = END;
                return true;
            }
        case DyvilKeywords.TRY:
            {
                // try ...
                final TryStatement tryStatement = new TryStatement(token.raw());
                this.value = tryStatement;
                pm.pushParser(new TryStatementParser(tryStatement));
                this.mode = END;
                return true;
            }
        case DyvilKeywords.CATCH:
            {
                if (!(this.parent instanceof TryStatementParser) && !(this.parent instanceof ExpressionParser)) {
                    pm.report(token, "expression.catch");
                    return true;
                }
                this.end(pm, true);
                return true;
            }
        case DyvilKeywords.FINALLY:
            {
                if (!(this.parent instanceof TryStatementParser) && !(this.parent instanceof ExpressionParser)) {
                    pm.report(token, "expression.finally");
                    return true;
                }
                this.end(pm, true);
                return true;
            }
        case DyvilKeywords.THROW:
            {
                final ThrowStatement throwStatement = new ThrowStatement(token.raw());
                this.value = throwStatement;
                pm.pushParser(new ExpressionParser(throwStatement));
                this.mode = END;
                return true;
            }
        case DyvilKeywords.SYNCHRONIZED:
            {
                final SyncStatement syncStatement = new SyncStatement(token.raw());
                this.value = syncStatement;
                pm.pushParser(new SyncStatementParser(syncStatement));
                this.mode = END;
                return true;
            }
    }
    return false;
}