Examples with TokenStream com.laytonsmith.core.compiler.TokenStream used on opensource projects

Example 1 with TokenStream

use of com.laytonsmith.core.compiler.TokenStream in project CommandHelper by EngineHub.

the class Script method compileRight.

public void compileRight() throws ConfigCompileException, ConfigCompileGroupException {
    List<Token> temp = new ArrayList<>();
    right = new ArrayList<>();
    for (Token t : fullRight) {
        if (t.type == TType.SEPERATOR) {
            right.add(temp);
            temp = new ArrayList<>();
        } else {
            if (t.type == TType.WHITESPACE) {
                // Whitespace is ignored on the right side
                continue;
            }
            temp.add(t);
        }
    }
    right.add(temp);
    cright = new ArrayList<>();
    for (List<Token> l : right) {
        cright.add(MethodScriptCompiler.compile(new TokenStream(l, fileOptions)));
    }
}

Example 2 with TokenStream

use of com.laytonsmith.core.compiler.TokenStream in project CommandHelper by EngineHub.

the class MethodScriptCompiler method lex.

/**
 * Lexes the script, and turns it into a token stream. This looks through the script character by character.
 *
 * @param script The script to lex
 * @param file The file this script came from, or potentially null if the code is from a dynamic source
 * @param inPureMScript If the script is in pure MethodScript, this should be true. Pure MethodScript is defined as
 * code that doesn't have command alias wrappers.
 * @param saveAllTokens If this script is planning to be compiled, then this value should always be false, however,
 * if the calling code needs all tokens for informational purposes (and doesn't plan on actually compiling the code)
 * then this can be true. If true, all tokens are saved, including comments and (some) whitespace. Given this lexing
 * stream, the exact source code could be re-constructed.
 *
 * A note on whitespace: The whitespace tokens are not guaranteed to be accurate, however, the column information
 * is. If you have two tokens t1 and t2, each with a value of length 1, where the columns are 1 and 5, then that
 * means there are 4 spaces between the two.
 * @return A stream of tokens
 * @throws ConfigCompileException If compilation fails due to bad syntax
 */
public static TokenStream lex(String script, File file, boolean inPureMScript, boolean saveAllTokens) throws ConfigCompileException {
    if (script.isEmpty()) {
        return new TokenStream(new LinkedList<>(), "");
    }
    if ((int) script.charAt(0) == 65279) {
        // Remove the UTF-8 Byte Order Mark, if present.
        script = script.substring(1);
    }
    final StringBuilder fileOptions = new StringBuilder();
    script = script.replaceAll("\r\n", "\n");
    script = script + "\n";
    final Set<String> keywords = KeywordList.getKeywordNames();
    final TokenStream token_list = new TokenStream();
    // Set our state variables.
    boolean state_in_quote = false;
    int quoteLineNumberStart = 1;
    boolean in_smart_quote = false;
    int smartQuoteLineNumberStart = 1;
    boolean in_comment = false;
    int commentLineNumberStart = 1;
    boolean comment_is_block = false;
    boolean in_opt_var = false;
    boolean inCommand = (!inPureMScript);
    boolean inMultiline = false;
    boolean in_smart_comment = false;
    boolean in_file_options = false;
    int fileOptionsLineNumberStart = 1;
    StringBuilder buf = new StringBuilder();
    int line_num = 1;
    int column = 1;
    int lastColumn = 0;
    Target target = Target.UNKNOWN;
    // Lex the script character by character.
    for (int i = 0; i < script.length(); i++) {
        Character c = script.charAt(i);
        Character c2 = null;
        if (i < script.length() - 1) {
            c2 = script.charAt(i + 1);
        }
        column += i - lastColumn;
        lastColumn = i;
        if (c == '\n') {
            line_num++;
            column = 1;
            if (!inMultiline && !inPureMScript) {
                inCommand = true;
            }
        }
        if (buf.length() == 0) {
            target = new Target(line_num, file, column);
        }
        // If we are in file options, add the character to the buffer if it's not a file options end character.
        if (in_file_options) {
            // If support for more escaped characters would be desired in the future, it could be added here.
            switch(c) {
                case '\\':
                    {
                        if (c2 == '>') {
                            // "\>".
                            fileOptions.append('>');
                            i++;
                            continue;
                        }
                    }
                case '>':
                    {
                        if (saveAllTokens) {
                            token_list.add(new Token(TType.FILE_OPTIONS_STRING, fileOptions.toString(), target));
                            token_list.add(new Token(TType.FILE_OPTIONS_END, ">", target));
                        }
                        in_file_options = false;
                        continue;
                    }
            }
            fileOptions.append(c);
            continue;
        }
        // Comment handling. This is bypassed if we are in a string.
        if (!state_in_quote && !in_smart_quote) {
            switch(c) {
                // Block comments start (/* and /**) and Double slash line comment start (//).
                case '/':
                    {
                        if (!in_comment) {
                            if (c2 == '*') {
                                // "/*" or "/**".
                                buf.append("/*");
                                in_comment = true;
                                comment_is_block = true;
                                if (i < script.length() - 2 && script.charAt(i + 2) == '*') {
                                    // "/**".
                                    in_smart_comment = true;
                                    buf.append("*");
                                    i++;
                                }
                                commentLineNumberStart = line_num;
                                i++;
                                continue;
                            } else if (c2 == '/') {
                                // "//".
                                buf.append("//");
                                in_comment = true;
                                i++;
                                continue;
                            }
                        }
                        break;
                    }
                // Line comment start (#).
                case '#':
                    {
                        if (!in_comment) {
                            // "#".
                            buf.append("#");
                            in_comment = true;
                            continue;
                        }
                        break;
                    }
                // Block comment end (*/).
                case '*':
                    {
                        if (in_comment && comment_is_block && c2 == '/') {
                            // "*/".
                            if (saveAllTokens || in_smart_comment) {
                                buf.append("*/");
                                token_list.add(new Token(in_smart_comment ? TType.SMART_COMMENT : TType.COMMENT, buf.toString(), target));
                            }
                            buf = new StringBuilder();
                            target = new Target(line_num, file, column);
                            in_comment = false;
                            comment_is_block = false;
                            in_smart_comment = false;
                            i++;
                            continue;
                        }
                        break;
                    }
                // Line comment end (\n).
                case '\n':
                    {
                        if (in_comment && !comment_is_block) {
                            // "\n".
                            in_comment = false;
                            if (saveAllTokens) {
                                token_list.add(new Token(TType.COMMENT, buf.toString(), target));
                                token_list.add(new Token(TType.NEWLINE, "\n", new Target(line_num + 1, file, 0)));
                            }
                            buf = new StringBuilder();
                            target = new Target(line_num, file, column);
                            continue;
                        }
                        break;
                    }
            }
        }
        // If we are in a comment, add the character to the buffer.
        if (in_comment) {
            buf.append(c);
            continue;
        }
        // Handle non-comment non-quoted characters.
        if (!state_in_quote) {
            // (, ), ;, and whitespace.
            matched: {
                Token token;
                switch(c) {
                    case '+':
                        {
                            if (c2 == '=') {
                                // "+=".
                                token = new Token(TType.PLUS_ASSIGNMENT, "+=", target);
                                i++;
                            } else if (c2 == '+') {
                                // "++".
                                token = new Token(TType.INCREMENT, "++", target);
                                i++;
                            } else {
                                // "+".
                                token = new Token(TType.PLUS, "+", target);
                            }
                            break;
                        }
                    case '-':
                        {
                            if (c2 == '=') {
                                // "-=".
                                token = new Token(TType.MINUS_ASSIGNMENT, "-=", target);
                                i++;
                            } else if (c2 == '-') {
                                // "--".
                                token = new Token(TType.DECREMENT, "--", target);
                                i++;
                            } else if (c2 == '>') {
                                // "->".
                                token = new Token(TType.DEREFERENCE, "->", target);
                                i++;
                            } else {
                                // "-".
                                token = new Token(TType.MINUS, "-", target);
                            }
                            break;
                        }
                    case '*':
                        {
                            if (c2 == '=') {
                                // "*=".
                                token = new Token(TType.MULTIPLICATION_ASSIGNMENT, "*=", target);
                                i++;
                            } else if (c2 == '*') {
                                // "**".
                                token = new Token(TType.EXPONENTIAL, "**", target);
                                i++;
                            } else {
                                // "*".
                                token = new Token(TType.MULTIPLICATION, "*", target);
                            }
                            break;
                        }
                    case '/':
                        {
                            if (c2 == '=') {
                                // "/=".
                                token = new Token(TType.DIVISION_ASSIGNMENT, "/=", target);
                                i++;
                            } else {
                                // Protect against matching commands.
                                if (Character.isLetter(c2)) {
                                    // Pretend that division didn't match.
                                    break matched;
                                }
                                token = new Token(TType.DIVISION, "/", target);
                            }
                            break;
                        }
                    case '.':
                        {
                            if (c2 == '=') {
                                // ".=".
                                token = new Token(TType.CONCAT_ASSIGNMENT, ".=", target);
                                i++;
                            } else if (c2 == '.') {
                                // "..".
                                token = new Token(TType.SLICE, "..", target);
                                i++;
                            } else {
                                // ".".
                                token = new Token(TType.DOT, ".", target);
                            }
                            break;
                        }
                    case '%':
                        {
                            token = new Token(TType.MODULO, "%", target);
                            break;
                        }
                    case '>':
                        {
                            if (c2 == '=') {
                                // ">=".
                                token = new Token(TType.GTE, ">=", target);
                                i++;
                            } else if (c2 == '>' && i < script.length() - 2 && script.charAt(i + 2) == '>') {
                                // ">>>".
                                token = new Token(TType.MULTILINE_START, ">>>", target);
                                inMultiline = true;
                                i += 2;
                            } else {
                                // ">".
                                token = new Token(TType.GT, ">", target);
                            }
                            break;
                        }
                    case '<':
                        {
                            if (c2 == '!') {
                                // "<!".
                                if (buf.length() > 0) {
                                    token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
                                    buf = new StringBuilder();
                                    target = new Target(line_num, file, column);
                                }
                                if (saveAllTokens) {
                                    token_list.add(new Token(TType.FILE_OPTIONS_START, "<!", target));
                                }
                                in_file_options = true;
                                fileOptionsLineNumberStart = line_num;
                                i++;
                                continue;
                            } else if (c2 == '=') {
                                // "<=".
                                token = new Token(TType.LTE, "<=", target);
                                i++;
                            } else if (c2 == '<' && i < script.length() - 2 && script.charAt(i + 2) == '<') {
                                // "<<<".
                                token = new Token(TType.MULTILINE_END, "<<<", target);
                                inMultiline = false;
                                i += 2;
                            } else {
                                // "<".
                                token = new Token(TType.LT, "<", target);
                            }
                            break;
                        }
                    case '=':
                        {
                            if (c2 == '=') {
                                if (i < script.length() - 2 && script.charAt(i + 2) == '=') {
                                    // "===".
                                    token = new Token(TType.STRICT_EQUALS, "===", target);
                                    i += 2;
                                } else {
                                    // "==".
                                    token = new Token(TType.EQUALS, "==", target);
                                    i++;
                                }
                            } else {
                                // "=".
                                if (inCommand) {
                                    if (in_opt_var) {
                                        token = new Token(TType.OPT_VAR_ASSIGN, "=", target);
                                    } else {
                                        token = new Token(TType.ALIAS_END, "=", target);
                                        inCommand = false;
                                    }
                                } else {
                                    token = new Token(TType.ASSIGNMENT, "=", target);
                                }
                            }
                            break;
                        }
                    case '!':
                        {
                            if (c2 == '=') {
                                if (i < script.length() - 2 && script.charAt(i + 2) == '=') {
                                    // "!==".
                                    token = new Token(TType.STRICT_NOT_EQUALS, "!==", target);
                                    i += 2;
                                } else {
                                    // "!=".
                                    token = new Token(TType.NOT_EQUALS, "!=", target);
                                    i++;
                                }
                            } else {
                                // "!".
                                token = new Token(TType.LOGICAL_NOT, "!", target);
                            }
                            break;
                        }
                    case '&':
                        {
                            if (c2 == '&') {
                                if (i < script.length() - 2 && script.charAt(i + 2) == '&') {
                                    // "&&&".
                                    token = new Token(TType.DEFAULT_AND, "&&&", target);
                                    i += 2;
                                } else {
                                    // "&&".
                                    token = new Token(TType.LOGICAL_AND, "&&", target);
                                    i++;
                                }
                            } else {
                                // Pretend that bitwise AND didn't match.
                                break matched;
                            // token = new Token(TType.BIT_AND, "&", target);
                            }
                            break;
                        }
                    case '|':
                        {
                            if (c2 == '|') {
                                if (i < script.length() - 2 && script.charAt(i + 2) == '|') {
                                    // "|||".
                                    token = new Token(TType.DEFAULT_OR, "|||", target);
                                    i += 2;
                                } else {
                                    // "||".
                                    token = new Token(TType.LOGICAL_OR, "||", target);
                                    i++;
                                }
                            } else {
                                // Pretend that bitwise OR didn't match.
                                break matched;
                            // token = new Token(TType.BIT_OR, "|", target);
                            }
                            break;
                        }
                    // }
                    case ':':
                        {
                            if (c2 == ':') {
                                // "::".
                                token = new Token(TType.DEREFERENCE, "::", target);
                                i++;
                            } else {
                                // ":".
                                token = new Token(TType.LABEL, ":", target);
                            }
                            break;
                        }
                    case '{':
                        {
                            token = new Token(TType.LCURLY_BRACKET, "{", target);
                            break;
                        }
                    case '}':
                        {
                            token = new Token(TType.RCURLY_BRACKET, "}", target);
                            break;
                        }
                    case '[':
                        {
                            token = new Token(TType.LSQUARE_BRACKET, "[", target);
                            in_opt_var = true;
                            break;
                        }
                    case ']':
                        {
                            token = new Token(TType.RSQUARE_BRACKET, "]", target);
                            in_opt_var = false;
                            break;
                        }
                    case ',':
                        {
                            token = new Token(TType.COMMA, ",", target);
                            break;
                        }
                    case ';':
                        {
                            token = new Token(TType.SEMICOLON, ";", target);
                            break;
                        }
                    case '(':
                        {
                            token = new Token(TType.FUNC_START, "(", target);
                            // Handle the buffer or previous token, with the knowledge that a FUNC_START follows.
                            if (buf.length() > 0) {
                                if (saveAllTokens) {
                                    // true, because we know that won't be used by the compiler.
                                    if (KeywordList.getKeywordByName(buf.toString()) != null) {
                                        // It's a keyword.
                                        token_list.add(new Token(TType.KEYWORD, buf.toString(), target));
                                    } else {
                                        // It's not a keyword, but a normal function.
                                        token_list.add(new Token(TType.FUNC_NAME, buf.toString(), target));
                                    }
                                } else {
                                    token_list.add(new Token(TType.FUNC_NAME, buf.toString(), target));
                                }
                                buf = new StringBuilder();
                                target = new Target(line_num, file, column);
                            } else {
                                // function.
                                try {
                                    int count = 0;
                                    Iterator<Token> it = token_list.descendingIterator();
                                    Token t;
                                    while ((t = it.next()).type == TType.WHITESPACE) {
                                        count++;
                                    }
                                    if (t.type == TType.UNKNOWN) {
                                        t.type = TType.FUNC_NAME;
                                        // Go ahead and remove the whitespace here too, they break things.
                                        count--;
                                        for (int a = 0; a < count; a++) {
                                            token_list.removeLast();
                                        }
                                    } else {
                                        token_list.add(new Token(TType.FUNC_NAME, "__autoconcat__", target));
                                    }
                                } catch (NoSuchElementException e) {
                                    // This is the first element on the list, so, it's another autoconcat.
                                    token_list.add(new Token(TType.FUNC_NAME, "__autoconcat__", target));
                                }
                            }
                            break;
                        }
                    case ')':
                        {
                            token = new Token(TType.FUNC_END, ")", target);
                            break;
                        }
                    case ' ':
                        {
                            // Whitespace case #1.
                            token = new Token(TType.WHITESPACE, " ", target);
                            break;
                        }
                    case '\t':
                        {
                            // Whitespace case #2 (TAB).
                            token = new Token(TType.WHITESPACE, "\t", target);
                            break;
                        }
                    default:
                        {
                            // No match was found at this point, so continue matching below.
                            break matched;
                        }
                }
                // Add previous characters as UNKNOWN token.
                if (buf.length() > 0) {
                    token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
                    buf = new StringBuilder();
                    target = new Target(line_num, file, column);
                }
                // Add the new token to the token list.
                token_list.add(token);
                // Continue lexing.
                continue;
            }
        }
        // Handle non-comment characters that might start or stop a quoted string.
        switch(c) {
            case '\'':
                {
                    if (state_in_quote && !in_smart_quote) {
                        token_list.add(new Token(TType.STRING, buf.toString(), target));
                        buf = new StringBuilder();
                        target = new Target(line_num, file, column);
                        state_in_quote = false;
                        continue;
                    } else if (!state_in_quote) {
                        state_in_quote = true;
                        quoteLineNumberStart = line_num;
                        in_smart_quote = false;
                        if (buf.length() > 0) {
                            token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
                            buf = new StringBuilder();
                            target = new Target(line_num, file, column);
                        }
                        continue;
                    } else {
                        // We're in a smart quote.
                        buf.append("'");
                    }
                    break;
                }
            case '"':
                {
                    if (state_in_quote && in_smart_quote) {
                        token_list.add(new Token(TType.SMART_STRING, buf.toString(), target));
                        buf = new StringBuilder();
                        target = new Target(line_num, file, column);
                        state_in_quote = false;
                        in_smart_quote = false;
                        continue;
                    } else if (!state_in_quote) {
                        state_in_quote = true;
                        in_smart_quote = true;
                        smartQuoteLineNumberStart = line_num;
                        if (buf.length() > 0) {
                            token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
                            buf = new StringBuilder();
                            target = new Target(line_num, file, column);
                        }
                        continue;
                    } else {
                        // We're in normal quotes.
                        buf.append('"');
                    }
                    break;
                }
            case '\n':
                {
                    // Append a newline to the buffer if it's quoted.
                    if (state_in_quote) {
                        buf.append(c);
                    } else {
                        // Newline is not quoted. Move the buffer to an UNKNOWN token and add a NEWLINE token.
                        if (buf.length() > 0) {
                            token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
                            buf = new StringBuilder();
                            target = new Target(line_num, file, column);
                        }
                        token_list.add(new Token(TType.NEWLINE, "\n", target));
                    }
                    continue;
                }
            case '\\':
                {
                    // Handle backslash character outside of quotes.
                    if (!state_in_quote) {
                        token_list.add(new Token(TType.SEPERATOR, "\\", target));
                        break;
                    }
                    // Handle an escape sign in a quote.
                    switch(c2) {
                        case '\\':
                        case '\'':
                        case '"':
                            buf.append(c2);
                            break;
                        case 'n':
                            buf.append('\n');
                            break;
                        case 'r':
                            buf.append('\r');
                            break;
                        case 't':
                            buf.append('\t');
                            break;
                        case '0':
                            buf.append('\0');
                            break;
                        case 'f':
                            buf.append('\f');
                            // Form feed.
                            break;
                        case 'v':
                            buf.append('\u000B');
                            // Vertical TAB.
                            break;
                        case 'a':
                            buf.append('\u0007');
                            // Alarm.
                            break;
                        case 'b':
                            buf.append('\u0008');
                            // Backspace.
                            break;
                        case 'u':
                            {
                                // Grab the next 4 characters, and check to see if they are numbers.
                                if (i + 5 >= script.length()) {
                                    throw new ConfigCompileException("Unrecognized unicode escape sequence", target);
                                }
                                String unicode = script.substring(i + 2, i + 6);
                                int unicodeNum;
                                try {
                                    unicodeNum = Integer.parseInt(unicode, 16);
                                } catch (NumberFormatException e) {
                                    throw new ConfigCompileException("Unrecognized unicode escape sequence: \\u" + unicode, target);
                                }
                                buf.append(Character.toChars(unicodeNum));
                                i += 4;
                                break;
                            }
                        case 'U':
                            {
                                // Grab the next 8 characters and check to see if they are numbers.
                                if (i + 9 >= script.length()) {
                                    throw new ConfigCompileException("Unrecognized unicode escape sequence", target);
                                }
                                String unicode = script.substring(i + 2, i + 10);
                                int unicodeNum;
                                try {
                                    unicodeNum = Integer.parseInt(unicode, 16);
                                } catch (NumberFormatException e) {
                                    throw new ConfigCompileException("Unrecognized unicode escape sequence: \\u" + unicode, target);
                                }
                                buf.append(Character.toChars(unicodeNum));
                                i += 8;
                                break;
                            }
                        case '@':
                            {
                                if (!in_smart_quote) {
                                    throw new ConfigCompileException("The escape sequence \\@ is not" + " a recognized escape sequence in a non-smart string", target);
                                }
                                buf.append("\\@");
                                break;
                            }
                        default:
                            {
                                // Since we might expand this list later, don't let them use unescaped backslashes.
                                throw new ConfigCompileException("The escape sequence \\" + c2 + " is not a recognized escape sequence", target);
                            }
                    }
                    i++;
                    continue;
                }
            default:
                {
                    // Disallow Non-Breaking Space Characters.
                    if (!state_in_quote && c == '\u00A0') /*nbsp*/
                    {
                        throw new ConfigCompileException("NBSP character in script", target);
                    }
                    // Add the characters that didn't match anything to the buffer.
                    buf.append(c);
                    continue;
                }
        }
    }
    // Handle unended file options.
    if (in_file_options) {
        throw new ConfigCompileException("Unended file options. You started the the file options on line " + fileOptionsLineNumberStart, target);
    }
    // Handle unended string literals.
    if (state_in_quote) {
        if (in_smart_quote) {
            throw new ConfigCompileException("Unended string literal. You started the last double quote on line " + smartQuoteLineNumberStart, target);
        } else {
            throw new ConfigCompileException("Unended string literal. You started the last single quote on line " + quoteLineNumberStart, target);
        }
    }
    // Handle unended comment blocks. Since a newline is added to the end of the script, line comments are ended.
    if (in_comment || comment_is_block) {
        throw new ConfigCompileException("Unended block comment. You started the comment on line " + commentLineNumberStart, target);
    }
    // Look at the tokens and get meaning from them. Also, look for improper symbol locations
    // and go ahead and absorb unary +- into the token.
    ListIterator<Token> it = token_list.listIterator(0);
    while (it.hasNext()) {
        Token t = it.next();
        // Combine whitespace tokens into one.
        if (t.type == TType.WHITESPACE && it.hasNext()) {
            Token next;
            if ((next = it.next()).type == TType.WHITESPACE) {
                t.value += next.val();
                // Remove 'next'.
                it.remove();
            } else {
                // Select 'next' <--.
                it.previous();
            }
            // Select 't' <--.
            it.previous();
            // Select 't' -->.
            it.next();
        }
        // Convert "-" + number to -number if allowed.
        // Select 't' <--.
        it.previous();
        if (it.hasPrevious()) {
            // Select 'prev1' <--.
            Token prev1 = it.previous();
            if (it.hasPrevious()) {
                // Select 'prev2' <--.
                Token prev2 = it.previous();
                if (// Convert "± UNKNOWN".
                t.type == TType.UNKNOWN && prev1.type.isPlusMinus() && // Don't convert "number/string/var ± ...".
                !prev2.type.isIdentifier() && // Don't convert "func() ± ...".
                prev2.type != TType.FUNC_END && // Don't convert "± @var".
                !IVAR_PATTERN.matcher(t.val()).matches() && !VAR_PATTERN.matcher(t.val()).matches()) {
                    // Don't convert "± $var".
                    // It is a negative/positive number: Absorb the sign.
                    t.value = prev1.value + t.value;
                    // Select 'prev2' -->.
                    it.next();
                    // Select 'prev1' -->.
                    it.next();
                    // Remove 'prev1'.
                    it.remove();
                } else {
                    // Select 'prev2' -->.
                    it.next();
                    // Select 'prev1' -->.
                    it.next();
                }
            } else {
                // Select 'prev1' -->.
                it.next();
            }
        }
        // Select 't' -->.
        it.next();
        // Assign a type to all UNKNOWN tokens.
        if (t.type == TType.UNKNOWN) {
            if (t.val().charAt(0) == '/' && t.val().length() > 1) {
                t.type = TType.COMMAND;
            } else if (t.val().equals("$")) {
                t.type = TType.FINAL_VAR;
            } else if (VAR_PATTERN.matcher(t.val()).matches()) {
                t.type = TType.VARIABLE;
            } else if (IVAR_PATTERN.matcher(t.val()).matches()) {
                t.type = TType.IVARIABLE;
            } else if (t.val().charAt(0) == '@') {
                throw new ConfigCompileException("IVariables must match the regex: " + IVAR_PATTERN, target);
            } else if (keywords.contains(t.val())) {
                t.type = TType.KEYWORD;
            } else if (t.val().matches("[\t ]*")) {
                t.type = TType.WHITESPACE;
            } else {
                t.type = TType.LIT;
            }
        }
        // Skip this check if we're not in pure mscript.
        if (inPureMScript) {
            if (it.hasNext()) {
                // Select 'next' -->.
                Token next = it.next();
                // Select 'next' <--.
                it.previous();
                // Select 't' <--.
                it.previous();
                if (t.type.isSymbol() && !t.type.isUnary() && !next.type.isUnary()) {
                    if (it.hasPrevious()) {
                        // Select 'prev1' <--.
                        Token prev1 = it.previous();
                        if (prev1.type.equals(TType.FUNC_START) || prev1.type.equals(TType.COMMA) || next.type.equals(TType.FUNC_END) || next.type.equals(TType.COMMA) || prev1.type.isSymbol() || next.type.isSymbol()) {
                            throw new ConfigCompileException("Unexpected symbol (" + t.val() + ")", t.getTarget());
                        }
                        // Select 'prev1' -->.
                        it.next();
                    }
                }
                // Select 't' -->.
                it.next();
            }
        }
    }
    // Set file options
    token_list.setFileOptions(fileOptions.toString());
    // Make sure that the file options are the first non-comment code in the file
    {
        boolean foundCode = false;
        for (Token t : token_list) {
            if (t.type.isFileOption()) {
                if (foundCode) {
                    throw new ConfigCompileException("File options must be the first non-comment section in the" + " code", t.target);
                }
                break;
            }
            if (!t.type.isComment() && !t.type.isWhitespace()) {
                foundCode = true;
            }
        }
    }
    return token_list;
}

Example 3 with TokenStream

use of com.laytonsmith.core.compiler.TokenStream in project CommandHelper by EngineHub.

the class CommandHelperInterpreterListener method execute.

public void execute(String script, final MCPlayer p) throws ConfigCompileException, ConfigCompileGroupException {
    TokenStream stream = MethodScriptCompiler.lex(script, new File("Interpreter"), true);
    ParseTree tree = MethodScriptCompiler.compile(stream);
    interpreterMode.remove(p.getName());
    GlobalEnv gEnv = new GlobalEnv(plugin.executionQueue, plugin.profiler, plugin.persistenceNetwork, CommandHelperFileLocations.getDefault().getConfigDirectory(), plugin.profiles, new TaskManager());
    gEnv.SetDynamicScriptingMode(true);
    CommandHelperEnvironment cEnv = new CommandHelperEnvironment();
    cEnv.SetPlayer(p);
    Environment env = Environment.createEnvironment(gEnv, cEnv);
    try {
        MethodScriptCompiler.registerAutoIncludes(env, null);
        MethodScriptCompiler.execute(tree, env, new MethodScriptComplete() {

            @Override
            public void done(String output) {
                output = output.trim();
                if (output.isEmpty()) {
                    Static.SendMessage(p, ":");
                } else {
                    if (output.startsWith("/")) {
                        // Run the command
                        Static.SendMessage(p, ":" + MCChatColor.YELLOW + output);
                        p.chat(output);
                    } else {
                        // output the results
                        Static.SendMessage(p, ":" + MCChatColor.GREEN + output);
                    }
                }
                interpreterMode.add(p.getName());
            }
        }, null);
    } catch (CancelCommandException e) {
        interpreterMode.add(p.getName());
    } catch (ConfigRuntimeException e) {
        ConfigRuntimeException.HandleUncaughtException(e, env);
        Static.SendMessage(p, MCChatColor.RED + e.toString());
        interpreterMode.add(p.getName());
    } catch (Exception e) {
        Static.SendMessage(p, MCChatColor.RED + e.toString());
        Logger.getLogger(CommandHelperInterpreterListener.class.getName()).log(Level.SEVERE, null, e);
        interpreterMode.add(p.getName());
    }
}

Example 4 with TokenStream

use of com.laytonsmith.core.compiler.TokenStream in project CommandHelper by EngineHub.

the class Interpreter method execute.

/**
 * This executes an entire script. The cmdline_prompt_event is first triggered (if used) and if the event is
 * cancelled, nothing happens.
 *
 * @param script
 * @param args
 * @param fromFile
 * @throws ConfigCompileException
 * @throws IOException
 */
public void execute(String script, List<String> args, File fromFile) throws ConfigCompileException, IOException, ConfigCompileGroupException {
    CmdlineEvents.cmdline_prompt_input.CmdlinePromptInput input = new CmdlineEvents.cmdline_prompt_input.CmdlinePromptInput(script, inShellMode);
    EventUtils.TriggerListener(Driver.CMDLINE_PROMPT_INPUT, "cmdline_prompt_input", input);
    if (input.isCancelled()) {
        return;
    }
    ctrlCcount = 0;
    if ("exit".equals(script)) {
        if (inShellMode) {
            inShellMode = false;
            return;
        }
        pl(YELLOW + "Use exit() if you wish to exit.");
        return;
    }
    if ("help".equals(script)) {
        pl(getHelpMsg());
        return;
    }
    if (fromFile == null) {
        fromFile = new File("Interpreter");
    }
    boolean localShellMode = false;
    if (!inShellMode && script.startsWith("$$")) {
        localShellMode = true;
        script = script.substring(2);
    }
    if (inShellMode || localShellMode) {
        // Wrap this in shell_adv
        if (doBuiltin(script)) {
            return;
        }
        List<String> shellArgs = StringUtils.ArgParser(script);
        List<String> escapedArgs = new ArrayList<>();
        for (String arg : shellArgs) {
            escapedArgs.add(new CString(arg, Target.UNKNOWN).getQuote());
        }
        script = "shell_adv(" + "array(" + StringUtils.Join(escapedArgs, ",") + ")," + "array(" + "'stdout':closure(@l){sys_out(@l);}," + "'stderr':closure(@l){sys_err(@l);})" + ");";
    }
    isExecuting = true;
    ProfilePoint compile = env.getEnv(GlobalEnv.class).GetProfiler().start("Compilation", LogLevel.VERBOSE);
    final ParseTree tree;
    try {
        TokenStream stream = MethodScriptCompiler.lex(script, fromFile, true);
        tree = MethodScriptCompiler.compile(stream);
    } finally {
        compile.stop();
    }
    // Environment env = Environment.createEnvironment(this.env.getEnv(GlobalEnv.class));
    final List<Variable> vars = new ArrayList<>();
    if (args != null) {
        // Build the @arguments variable, the $ vars, and $ itself. Note that
        // we have special handling for $0, that is the script name, like bash.
        // However, it doesn't get added to either $ or @arguments, due to the
        // uncommon use of it.
        StringBuilder finalArgument = new StringBuilder();
        CArray arguments = new CArray(Target.UNKNOWN);
        {
            // Set the $0 argument
            Variable v = new Variable("$0", "", Target.UNKNOWN);
            v.setVal(fromFile.toString());
            v.setDefault(fromFile.toString());
            vars.add(v);
        }
        for (int i = 0; i < args.size(); i++) {
            String arg = args.get(i);
            if (i > 0) {
                finalArgument.append(" ");
            }
            Variable v = new Variable("$" + Integer.toString(i + 1), "", Target.UNKNOWN);
            v.setVal(new CString(arg, Target.UNKNOWN));
            v.setDefault(arg);
            vars.add(v);
            finalArgument.append(arg);
            arguments.push(new CString(arg, Target.UNKNOWN), Target.UNKNOWN);
        }
        Variable v = new Variable("$", "", false, true, Target.UNKNOWN);
        v.setVal(new CString(finalArgument.toString(), Target.UNKNOWN));
        v.setDefault(finalArgument.toString());
        vars.add(v);
        env.getEnv(GlobalEnv.class).GetVarList().set(new IVariable(CArray.TYPE, "@arguments", arguments, Target.UNKNOWN));
    }
    try {
        ProfilePoint p = this.env.getEnv(GlobalEnv.class).GetProfiler().start("Interpreter Script", LogLevel.ERROR);
        try {
            final MutableObject<Throwable> wasThrown = new MutableObject<>();
            scriptThread = new Thread(new Runnable() {

                @Override
                public void run() {
                    try {
                        MethodScriptCompiler.execute(tree, env, new MethodScriptComplete() {

                            @Override
                            public void done(String output) {
                                if (System.console() != null && !"".equals(output.trim())) {
                                    StreamUtils.GetSystemOut().println(output);
                                }
                            }
                        }, null, vars);
                    } catch (CancelCommandException e) {
                    // Nothing, though we could have been Ctrl+C cancelled, so we need to reset
                    // the interrupt flag. But we do that unconditionally below, in the finally,
                    // in the other thread.
                    } catch (ConfigRuntimeException e) {
                        ConfigRuntimeException.HandleUncaughtException(e, env);
                        // No need for the full stack trace
                        if (System.console() == null) {
                            System.exit(1);
                        }
                    } catch (NoClassDefFoundError e) {
                        StreamUtils.GetSystemErr().println(RED + Main.getNoClassDefFoundErrorMessage(e) + reset());
                        StreamUtils.GetSystemErr().println("Since you're running from standalone interpreter mode, this is not a fatal error, but one of the functions you just used required" + " an actual backing engine that isn't currently loaded. (It still might fail even if you load the engine though.) You simply won't be" + " able to use that function here.");
                        if (System.console() == null) {
                            System.exit(1);
                        }
                    } catch (InvalidEnvironmentException ex) {
                        StreamUtils.GetSystemErr().println(RED + ex.getMessage() + " " + ex.getData() + "() cannot be used in this context.");
                        if (System.console() == null) {
                            System.exit(1);
                        }
                    } catch (RuntimeException e) {
                        pl(RED + e.toString());
                        e.printStackTrace(StreamUtils.GetSystemErr());
                        if (System.console() == null) {
                            System.exit(1);
                        }
                    }
                }
            }, "MethodScript-Main");
            scriptThread.start();
            try {
                scriptThread.join();
            } catch (InterruptedException ex) {
            // 
            }
        } finally {
            p.stop();
        }
    } finally {
        env.getEnv(GlobalEnv.class).SetInterrupt(false);
        isExecuting = false;
    }
}

Example 5 with TokenStream

use of com.laytonsmith.core.compiler.TokenStream in project CommandHelper by EngineHub.

the class SimpleSyntaxHighlighter method highlight.

private String highlight() throws ConfigCompileException {
    TokenStream tokens = MethodScriptCompiler.lex(code, null, inPureMscript, true);
    // take out the last token, which is always a newline
    tokens.remove(tokens.size() - 1);
    // if the first token is a newline, also take that out.
    if (tokens.get(0).type == TType.NEWLINE) {
        tokens.remove(0);
    }
    String newlineString = "<div><span style=\"font-style: italic; " + getColor(ElementTypes.LINE_NUMBER) + "\">" + "%0" + Integer.toString(tokens.get(tokens.size() - 1).line_num - 1).length() + "d</span>&nbsp;&nbsp;&nbsp;";
    StringBuilder out = new StringBuilder();
    AtomicInteger lineNum = new AtomicInteger(1);
    out.append(String.format(newlineString, lineNum.get()));
    for (Token t : tokens) {
        if (null != t.type) {
            switch(t.type) {
                case SMART_COMMENT:
                case COMMENT:
                    for (String line : t.val().split("\n")) {
                        out.append(getOpenSpan(t.type == TType.SMART_COMMENT ? ElementTypes.SMART_COMMENT : ElementTypes.COMMENT)).append(escapeLit(line)).append(getCloseSpan());
                        // this is not a bug.
                        if (t != tokens.get(tokens.size() - 1)) {
                            out.append("</div>").append(String.format(newlineString, lineNum.addAndGet(1)));
                        }
                    }
                    break;
                case SMART_STRING:
                    out.append(getOpenSpan(ElementTypes.DOUBLE_STRING)).append("&quot;");
                    out.append(processDoubleString(t.toOutputString()));
                    out.append("&quot;").append(getCloseSpan());
                    break;
                case VARIABLE:
                    out.append(getOpenSpan(ElementTypes.DVAR));
                    out.append(escapeLit(t.val()));
                    out.append(getCloseSpan());
                    break;
                case FUNC_NAME:
                    if (t.val().equals("__autoconcat__")) {
                        // case, we just want to get rid of this token, so we skip it.
                        break;
                    }
                    out.append(getOpenSpan(ElementTypes.FUNCTION, "font-style: italic"));
                    out.append("{{function|").append(escapeLit(t.val())).append("}}");
                    out.append(getCloseSpan());
                    break;
                case KEYWORD:
                    out.append(getOpenSpan(ElementTypes.KEYWORD));
                    out.append("{{keyword|").append(escapeLit(t.val())).append("}}");
                    out.append(getCloseSpan());
                    break;
                case STRING:
                    out.append(getOpenSpan(ElementTypes.SINGLE_STRING));
                    out.append("&apos;").append(escapeLit(t.toOutputString())).append("&apos;");
                    out.append(getCloseSpan());
                    break;
                case IVARIABLE:
                    out.append(getOpenSpan(ElementTypes.VAR));
                    out.append(escapeLit(t.val()));
                    out.append(getCloseSpan());
                    break;
                case LIT:
                    if (NativeTypeList.getNativeTypeList().contains(t.val())) {
                        out.append(getOpenSpan(ElementTypes.OBJECT_TYPE));
                        out.append("{{object|").append(t.val()).append("}}");
                        out.append(getCloseSpan());
                    } else {
                        out.append(escapeLit(t.val()));
                    }
                    break;
                case COMMAND:
                    out.append(getOpenSpan(ElementTypes.COMMAND));
                    out.append(t.val());
                    out.append(getCloseSpan());
                    break;
                case NEWLINE:
                    out.append("</div>").append(String.format(newlineString, lineNum.addAndGet(1)));
                    break;
                case WHITESPACE:
                    out.append(escapeLit(t.val()));
                    break;
                case FILE_OPTIONS_START:
                    out.append(getOpenSpan(ElementTypes.FILE_OPTIONS_BLOCK));
                    out.append(escapeLit(t.val()));
                    out.append(getCloseSpan());
                    break;
                case FILE_OPTIONS_END:
                    out.append(getOpenSpan(ElementTypes.FILE_OPTIONS_BLOCK));
                    out.append(escapeLit(t.val()));
                    out.append(getCloseSpan());
                    break;
                case FILE_OPTIONS_STRING:
                    out.append(processFileOptionsString(newlineString, lineNum, t.val()));
                    break;
                default:
                    out.append(escapeLit(t.val()));
                    break;
            }
        }
    }
    out.append("</div>");
    String totalOutput = "<div style=\"font-family: 'Consolas','DejaVu Sans','Lucida Console',monospace; " + (classes == null ? "" : "background-color: #" + getRGB(classes.get(ElementTypes.BACKGROUND_COLOR)) + ";" + " border-color: #" + getRGB(classes.get(ElementTypes.BORDER_COLOR)) + "; ") + " border-style: solid; border-width: 1px 0px 1px 0px; margin: 1em 2em;" + " padding: 12px 2px 1em 1em;\" class=\"methodscript_code\">" + out.toString() + "</div>";
    return totalOutput;
}