Examples with TokenList dyvilx.tools.parsing.TokenList used on opensource projects

Example 1 with TokenList

use of dyvilx.tools.parsing.TokenList in project Dyvil by Dyvil.

the class Colorizer method colorize.

public static String colorize(String text, REPLContext context) {
    final TextSource source = new TextSource(text);
    final TokenList tokens = new DyvilLexer(new MarkerList(Markers.INSTANCE), DyvilSymbols.INSTANCE).tokenize(text);
    // Split into lines
    final int lineCount = source.lineCount();
    final StringBuilder[] lines = new StringBuilder[lineCount];
    for (int i = 0; i < lineCount; i++) {
        lines[i] = new StringBuilder(source.line(i + 1));
    }
    // iterate, starting from the last token
    for (IToken token = tokens.last(); token != null && token.type() != Tokens.EOF; token = token.prev()) {
        final String color = tokenColor(token, context);
        if (color != null) {
            // insert ANSI color codes before and after the token
            final StringBuilder line = lines[token.startLine() - 1];
            line.insert(token.endColumn(), Console.ANSI_RESET);
            line.insert(token.startColumn(), color);
        }
    }
    // Merge back together
    final StringBuilder first = lines[0];
    for (int i = 1; i < lineCount; i++) {
        first.append('\n').append(lines[i]);
    }
    return first.toString();
}

Example 2 with TokenList

use of dyvilx.tools.parsing.TokenList in project Dyvil by Dyvil.

the class DyvilREPL method evaluate.

public void evaluate(String code) {
    this.context.startEvaluation(code);
    final MarkerList markers = this.context.getMarkers();
    final TokenList tokens = new DyvilLexer(markers, DyvilSymbols.INSTANCE).tokenize(code);
    SemicolonInference.inferSemicolons(tokens.first());
    this.parser.reset();
    new ParserManager(DyvilSymbols.INSTANCE, tokens.iterator(), markers).parse(this.parser);
    this.context.endEvaluation();
}

Example 3 with TokenList

use of dyvilx.tools.parsing.TokenList in project Dyvil by Dyvil.

the class CompleteCommand method execute.

@Override
public void execute(DyvilREPL repl, String argument) {
    final IContext context = repl.getContext().getContext();
    if (argument == null) {
        // REPL Variables
        this.printREPLMembers(repl, "");
        return;
    }
    final int dotIndex = argument.lastIndexOf('.');
    if (dotIndex <= 0) {
        // REPL Variable Completions
        this.printREPLMembers(repl, Qualifier.qualify(argument));
        return;
    }
    final String expression = argument.substring(0, dotIndex);
    final String memberStart = Qualifier.qualify(argument.substring(dotIndex + 1));
    final MarkerList markers = new MarkerList(Markers.INSTANCE);
    final TokenList tokens = new DyvilLexer(markers, DyvilSymbols.INSTANCE).tokenize(expression);
    new ParserManager(DyvilSymbols.INSTANCE, tokens.iterator(), markers).parse(new ExpressionParser((IValue value) -> {
        value.resolveTypes(markers, context);
        value = value.resolve(markers, context);
        value.checkTypes(markers, context);
        if (!markers.isEmpty()) {
            // Print Errors, if any
            final boolean colors = repl.getCompiler().config.useAnsiColors();
            final StringBuilder buffer = new StringBuilder();
            markers.log(new LineSource(expression), buffer, colors);
            repl.getOutput().println(buffer);
        }
        try {
            this.printCompletions(repl, memberStart, value);
        } catch (Throwable throwable) {
            throwable.printStackTrace(repl.getErrorOutput());
        }
    }));
}

Example 4 with TokenList

use of dyvilx.tools.parsing.TokenList in project Dyvil by Dyvil.

the class Lexer method init.

protected void init(String code, int cursor, int line, int column) {
    this.tokens = new TokenList();
    this.code = code;
    this.length = code.length();
    this.cursor = cursor;
    this.line = line;
    this.column = column;
    this.clearBuffer();
}

Example 5 with TokenList

use of dyvilx.tools.parsing.TokenList in project Dyvil by Dyvil.

the class Lexer method useSubLexer.

protected void useSubLexer(Lexer sublexer) {
    final TokenList tokens = sublexer.tokenize(this.code, this.cursor, this.line, this.column);
    this.tokens.addAll(tokens);
    this.cursor = sublexer.getCursor();
    this.line = sublexer.getLine();
    this.column = sublexer.getColumn();
}