Examples with Token com.laytonsmith.core.constructs.Token used on opensource projects

Example 6 with Token

use of com.laytonsmith.core.constructs.Token in project CommandHelper by EngineHub.

the class NewMethodScriptCompiler method preprocess.

public static List<NewScript> preprocess(TokenStream tokenStream, Environment compilerEnvironment) throws ConfigCompileException {
    List<NewScript> scripts = new ArrayList<NewScript>();
    // We need to split the command definition and the pure mscript parts. First,
    // we split on newlines, those are each going to be our alias definitions
    List<List<Token>> commands = new ArrayList<List<Token>>();
    List<Token> working = new ArrayList<Token>();
    for (int i = 0; i < tokenStream.size(); i++) {
        Token t = tokenStream.get(i);
        if (t.type == Token.TType.NEWLINE) {
            commands.add(working);
            working = new ArrayList<Token>();
            continue;
        }
        working.add(t);
    }
    // Now they are split into individual aliases
    for (List<Token> stream : commands) {
        // We need to make constructs from the left, and compile the right
        // Compiling the right can be simply passed off to the compile
        // function, but we need to parse the left ourselves
        // We *should* only have (bare) strings, numbers, brackets on the left
        List<Token> left = new ArrayList<Token>();
        TokenStream right = new TokenStream(new ArrayList<Token>(), tokenStream.getFileOptions());
        boolean inLeft = true;
        boolean hasLabel = false;
        for (Token t : stream) {
            if (t.type == Token.TType.ALIAS_END) {
                inLeft = false;
                continue;
            }
            if (t.type == TType.LABEL) {
                hasLabel = true;
            }
            if (inLeft) {
                left.add(t);
            } else {
                right.add(t);
            }
        }
        ParseTree cright = compile(right, compilerEnvironment);
        List<Construct> cleft = new ArrayList<Construct>();
        boolean atFinalVar = false;
        boolean atOptionalVars = false;
        boolean pastLabel = false;
        String label = "";
        try {
            for (int i = 0; i < left.size(); i++) {
                Token t = left.get(i);
                if (hasLabel && !pastLabel) {
                    if (t.type == TType.LABEL) {
                        pastLabel = true;
                        continue;
                    }
                    label += t.val();
                    continue;
                }
                if (atFinalVar) {
                    throw new ConfigCompileException("The final var must be the last declaration in the alias", t.getTarget());
                }
                if (t.type == TType.LSQUARE_BRACKET) {
                    Token tname = left.get(i + 1);
                    atOptionalVars = true;
                    if (tname.val().equals("$")) {
                        atFinalVar = true;
                    }
                    if (tname.type != TType.VARIABLE && tname.type != TType.FINAL_VAR) {
                        throw new ConfigCompileException("Expecting a variable, but found " + tname.val(), tname.getTarget());
                    }
                    i++;
                    Token next = left.get(i + 1);
                    if (next.type != TType.OPT_VAR_ASSIGN && next.type != TType.RSQUARE_BRACKET) {
                        throw new ConfigCompileException("Expecting either a variable assignment or right square bracket, but found " + next.val(), next.getTarget());
                    }
                    i++;
                    String defaultVal = "";
                    if (next.type == TType.OPT_VAR_ASSIGN) {
                        // We have an assignment here
                        Token val = left.get(i + 1);
                        i++;
                        defaultVal = val.val();
                        next = left.get(i + 1);
                    }
                    if (next.type != TType.RSQUARE_BRACKET) {
                        throw new ConfigCompileException("Expecting a right square bracket, but found " + next.val() + " instead. (Did you forget to quote a multi word string?)", next.getTarget());
                    }
                    i++;
                    Variable v = new Variable(tname.val(), defaultVal, true, (tname.val().equals("$")), tname.getTarget());
                    cleft.add(v);
                    continue;
                }
                if (t.type == TType.VARIABLE || t.type == TType.FINAL_VAR) {
                    // Required variable
                    if (atOptionalVars) {
                        throw new ConfigCompileException("Only optional variables may come after the first optional variable", t.getTarget());
                    }
                    if (t.val().equals("$")) {
                        atFinalVar = true;
                    }
                    Variable v = new Variable(t.val(), "", false, t.val().equals("$"), t.getTarget());
                    cleft.add(v);
                    continue;
                }
                cleft.add(tokenToConstruct(t));
            }
        } catch (IndexOutOfBoundsException e) {
            throw new ConfigCompileException("Expecting more tokens, but reached end of alias signature before tokens were resolved.", left.get(0).getTarget());
        }
        if (!cleft.isEmpty()) {
            link(cright, compilerEnvironment);
            scripts.add(new NewScript(cleft, cright, label));
        }
    }
    return scripts;
}

Example 7 with Token

use of com.laytonsmith.core.constructs.Token in project CommandHelper by EngineHub.

the class MethodScriptCompilerTest method testLex.

/**
 * Test of lex method, of class MethodScriptCompiler.
 */
@Test
public void testLex() throws Exception {
    String config = "/cmd = msg('string')";
    List<Token> e = new ArrayList<>();
    // This is the decomposed version of the above config
    e.add(new Token(Token.TType.COMMAND, "/cmd", Target.UNKNOWN));
    e.add(new Token(Token.TType.WHITESPACE, " ", Target.UNKNOWN));
    e.add(new Token(Token.TType.ALIAS_END, "=", Target.UNKNOWN));
    e.add(new Token(Token.TType.WHITESPACE, " ", Target.UNKNOWN));
    e.add(new Token(Token.TType.FUNC_NAME, "msg", Target.UNKNOWN));
    e.add(new Token(Token.TType.FUNC_START, "(", Target.UNKNOWN));
    e.add(new Token(Token.TType.STRING, "string", Target.UNKNOWN));
    e.add(new Token(Token.TType.FUNC_END, ")", Target.UNKNOWN));
    e.add(new Token(Token.TType.NEWLINE, "\n", Target.UNKNOWN));
    List<Token> result = MethodScriptCompiler.lex(config, null, false);
    assertEquals(e, result);
    String[] badConfigs = { // Bad escape sequences
    "'\\q'", "'\\m'" };
    for (String c : badConfigs) {
        try {
            MethodScriptCompiler.lex(c, null, false);
            // Shouldn't get here
            fail(c + " should not have lexed, but did.");
        } catch (ConfigCompileException ex) {
        // Success!
        }
    }
}

Example 8 with Token

use of com.laytonsmith.core.constructs.Token in project CommandHelper by EngineHub.

the class SimpleSyntaxHighlighter method main.

public static void main(String[] args) throws Exception {
    String script = "<!\nstrict: on;\n>";
    List<Token> ts = MethodScriptCompiler.lex(script, null, true, true);
    for (Token t : ts) {
        System.out.println(t.type);
        System.out.println(t.value);
        System.out.println(t.target);
        System.out.println("----------------");
    }
    StreamUtils.GetSystemOut().println(SimpleSyntaxHighlighter.Highlight(null, script, true));
}

Example 9 with Token

use of com.laytonsmith.core.constructs.Token in project CommandHelper by EngineHub.

the class SimpleSyntaxHighlighter method highlight.

private String highlight() throws ConfigCompileException {
    TokenStream tokens = MethodScriptCompiler.lex(code, null, inPureMscript, true);
    // take out the last token, which is always a newline
    tokens.remove(tokens.size() - 1);
    // if the first token is a newline, also take that out.
    if (tokens.get(0).type == TType.NEWLINE) {
        tokens.remove(0);
    }
    String newlineString = "<div><span style=\"font-style: italic; " + getColor(ElementTypes.LINE_NUMBER) + "\">" + "%0" + Integer.toString(tokens.get(tokens.size() - 1).line_num - 1).length() + "d</span>&nbsp;&nbsp;&nbsp;";
    StringBuilder out = new StringBuilder();
    AtomicInteger lineNum = new AtomicInteger(1);
    out.append(String.format(newlineString, lineNum.get()));
    for (Token t : tokens) {
        if (null != t.type) {
            switch(t.type) {
                case SMART_COMMENT:
                case COMMENT:
                    for (String line : t.val().split("\n")) {
                        out.append(getOpenSpan(t.type == TType.SMART_COMMENT ? ElementTypes.SMART_COMMENT : ElementTypes.COMMENT)).append(escapeLit(line)).append(getCloseSpan());
                        // this is not a bug.
                        if (t != tokens.get(tokens.size() - 1)) {
                            out.append("</div>").append(String.format(newlineString, lineNum.addAndGet(1)));
                        }
                    }
                    break;
                case SMART_STRING:
                    out.append(getOpenSpan(ElementTypes.DOUBLE_STRING)).append("&quot;");
                    out.append(processDoubleString(t.toOutputString()));
                    out.append("&quot;").append(getCloseSpan());
                    break;
                case VARIABLE:
                    out.append(getOpenSpan(ElementTypes.DVAR));
                    out.append(escapeLit(t.val()));
                    out.append(getCloseSpan());
                    break;
                case FUNC_NAME:
                    if (t.val().equals("__autoconcat__")) {
                        // case, we just want to get rid of this token, so we skip it.
                        break;
                    }
                    out.append(getOpenSpan(ElementTypes.FUNCTION, "font-style: italic"));
                    out.append("{{function|").append(escapeLit(t.val())).append("}}");
                    out.append(getCloseSpan());
                    break;
                case KEYWORD:
                    out.append(getOpenSpan(ElementTypes.KEYWORD));
                    out.append("{{keyword|").append(escapeLit(t.val())).append("}}");
                    out.append(getCloseSpan());
                    break;
                case STRING:
                    out.append(getOpenSpan(ElementTypes.SINGLE_STRING));
                    out.append("&apos;").append(escapeLit(t.toOutputString())).append("&apos;");
                    out.append(getCloseSpan());
                    break;
                case IVARIABLE:
                    out.append(getOpenSpan(ElementTypes.VAR));
                    out.append(escapeLit(t.val()));
                    out.append(getCloseSpan());
                    break;
                case LIT:
                    if (NativeTypeList.getNativeTypeList().contains(t.val())) {
                        out.append(getOpenSpan(ElementTypes.OBJECT_TYPE));
                        out.append("{{object|").append(t.val()).append("}}");
                        out.append(getCloseSpan());
                    } else {
                        out.append(escapeLit(t.val()));
                    }
                    break;
                case COMMAND:
                    out.append(getOpenSpan(ElementTypes.COMMAND));
                    out.append(t.val());
                    out.append(getCloseSpan());
                    break;
                case NEWLINE:
                    out.append("</div>").append(String.format(newlineString, lineNum.addAndGet(1)));
                    break;
                case WHITESPACE:
                    out.append(escapeLit(t.val()));
                    break;
                case FILE_OPTIONS_START:
                    out.append(getOpenSpan(ElementTypes.FILE_OPTIONS_BLOCK));
                    out.append(escapeLit(t.val()));
                    out.append(getCloseSpan());
                    break;
                case FILE_OPTIONS_END:
                    out.append(getOpenSpan(ElementTypes.FILE_OPTIONS_BLOCK));
                    out.append(escapeLit(t.val()));
                    out.append(getCloseSpan());
                    break;
                case FILE_OPTIONS_STRING:
                    out.append(processFileOptionsString(newlineString, lineNum, t.val()));
                    break;
                default:
                    out.append(escapeLit(t.val()));
                    break;
            }
        }
    }
    out.append("</div>");
    String totalOutput = "<div style=\"font-family: 'Consolas','DejaVu Sans','Lucida Console',monospace; " + (classes == null ? "" : "background-color: #" + getRGB(classes.get(ElementTypes.BACKGROUND_COLOR)) + ";" + " border-color: #" + getRGB(classes.get(ElementTypes.BORDER_COLOR)) + "; ") + " border-style: solid; border-width: 1px 0px 1px 0px; margin: 1em 2em;" + " padding: 12px 2px 1em 1em;\" class=\"methodscript_code\">" + out.toString() + "</div>";
    return totalOutput;
}

Example 10 with Token

use of com.laytonsmith.core.constructs.Token in project CommandHelper by EngineHub.

the class MethodScriptCompiler method preprocess.

/**
 * This function breaks the token stream into parts, separating the aliases/MethodScript from the command triggers
 *
 * @param tokenStream
 * @return
 * @throws ConfigCompileException
 */
public static List<Script> preprocess(TokenStream tokenStream) throws ConfigCompileException {
    if (tokenStream == null || tokenStream.isEmpty()) {
        return new ArrayList<>();
    }
    // Remove leading newlines.
    while (!tokenStream.isEmpty() && tokenStream.getFirst().type == TType.NEWLINE) {
        // Remove leading newlines.
        tokenStream.removeFirst();
    }
    // Return an empty list if there were only newlines.
    if (tokenStream.isEmpty()) {
        return new ArrayList<>();
    }
    // Remove whitespaces and duplicate newlines.
    {
        ListIterator<Token> it = tokenStream.listIterator(0);
        Token token = it.next();
        outerLoop: while (true) {
            switch(token.type) {
                case WHITESPACE:
                    {
                        // Remove whitespaces.
                        it.remove();
                        if (!it.hasNext()) {
                            break outerLoop;
                        }
                        token = it.next();
                        continue outerLoop;
                    }
                case NEWLINE:
                    {
                        while (true) {
                            if (!it.hasNext()) {
                                break outerLoop;
                            } else if ((token = it.next()).type == TType.NEWLINE) {
                                // Remove duplicate newlines.
                                it.remove();
                            } else {
                                continue outerLoop;
                            }
                        }
                    }
                default:
                    {
                        if (!it.hasNext()) {
                            break outerLoop;
                        }
                        token = it.next();
                        continue outerLoop;
                    }
            }
        }
    }
    // Handle multiline constructs.
    // Take out newlines between the '= >>>' and '<<<' tokens (also removing the '>>>' and '<<<' tokens).
    // Also remove comments and also remove newlines that are behind a '\'.
    boolean inside_multiline = false;
    ListIterator<Token> it = tokenStream.listIterator(0);
    Token token = null;
    while (it.hasNext()) {
        token = it.next();
        switch(token.type) {
            case ALIAS_END:
                {
                    // "=".
                    if (it.hasNext()) {
                        if (it.next().type == TType.MULTILINE_START) {
                            // "= >>>".
                            inside_multiline = true;
                            // Remove multiline start (>>>).
                            it.remove();
                            // Select 'token' <---.
                            it.previous();
                            // Select 'token' -->.
                            it.next();
                        } else {
                            // Select 'next' <---.
                            it.previous();
                        }
                    }
                    continue;
                }
            case MULTILINE_END:
                {
                    // Handle multiline end token (<<<) without start.
                    if (!inside_multiline) {
                        throw new ConfigCompileException("Found multiline end symbol, and no multiline start found", token.target);
                    }
                    inside_multiline = false;
                    // Remove multiline end (<<<).
                    it.remove();
                    continue;
                }
            case MULTILINE_START:
                {
                    // Handle multiline start token (>>>) while already in multiline.
                    if (inside_multiline) {
                        throw new ConfigCompileException("Did not expect a multiline start symbol here," + " are you missing a multiline end symbol above this line?", token.target);
                    }
                    // Handle multiline start token (>>>) without alias end (=) in front.
                    // Select 'token' <--.
                    it.previous();
                    if (!it.hasPrevious() || it.previous().type != TType.ALIAS_END) {
                        throw new ConfigCompileException("Multiline symbol must follow the alias_end (=) symbol", token.target);
                    }
                    // Select 'prev' -->.
                    it.next();
                    // Select 'token' -->.
                    it.next();
                    continue;
                }
            case NEWLINE:
                {
                    // Skip newlines that are inside a multiline construct.
                    if (inside_multiline) {
                        // Remove newline.
                        it.remove();
                    }
                    continue;
                }
            // Remove comments.
            case COMMENT:
            case SMART_COMMENT:
                {
                    // Remove comment.
                    it.remove();
                    continue;
                }
            default:
                {
                    // Remove newlines that are behind a '\'.
                    if (token.type != TType.STRING && token.val().equals("\\") && it.hasNext()) {
                        if (it.next().type == TType.NEWLINE) {
                            // Remove newline.
                            it.remove();
                            // Select 'token' <--.
                            it.previous();
                            // Select 'token' -->.
                            it.next();
                        } else {
                            // Select 'next' <--.
                            it.previous();
                        }
                    }
                }
        }
    }
    assert token != null;
    // Handle missing multiline end token.
    if (inside_multiline) {
        throw new ConfigCompileException("Expecting a multiline end symbol, but your last multiline alias appears to be missing one.", token.target);
    }
    // Now that we have all lines minified, we should be able to split on newlines
    // and easily find the left and right sides.
    List<Token> left = new ArrayList<>();
    List<Token> right = new ArrayList<>();
    List<Script> scripts = new ArrayList<>();
    tokenLoop: for (it = tokenStream.listIterator(0); it.hasNext(); ) {
        Token t = it.next();
        // Add all tokens until ALIAS_END (=) or end of stream.
        while (t.type != TType.ALIAS_END) {
            if (!it.hasNext()) {
                // End of stream.
                break tokenLoop;
            }
            left.add(t);
            t = it.next();
        }
        // Add all tokens until NEWLINE (\n).
        while (t.type != TType.NEWLINE) {
            // All files end with a newline, so end of stream should be impossible here.
            assert it.hasNext();
            right.add(t);
            t = it.next();
        }
        // Create a new script for the obtained left and right if end of stream has not been reached.
        if (t.type == TType.NEWLINE) {
            // Check for spurious symbols, which indicate an issue with the script, but ignore any whitespace.
            for (int j = left.size() - 1; j >= 0; j--) {
                if (left.get(j).type == TType.NEWLINE) {
                    if (j > 0 && left.get(j - 1).type != TType.WHITESPACE) {
                        throw new ConfigCompileException("Unexpected token: " + left.get(j - 1).val(), left.get(j - 1).getTarget());
                    }
                }
            }
            // Create a new script from the command descriptor (left) and code (right) and add it to the list.
            Script s = new Script(left, right, null, tokenStream.getFileOptions());
            scripts.add(s);
            // Create new left and right array for the next script.
            left = new ArrayList<>();
            right = new ArrayList<>();
        }
    }
    // Return the scripts.
    return scripts;
}