Examples with CharToken com.googlecode.aviator.lexer.token.CharToken used on opensource projects

Example 1 with CharToken

use of com.googlecode.aviator.lexer.token.CharToken in project aviatorscript by killme2008.

the class ExpressionLexer method scan.

public Token<?> scan(final boolean analyse) {
    // If buffer is not empty,return
    if (this.tokenBuffer != null && !this.tokenBuffer.isEmpty()) {
        return this.tokenBuffer.pop();
    }
    // Skip white space or line
    for (; ; nextChar()) {
        if (this.peek == CharacterIterator.DONE) {
            return null;
        }
        if (analyse) {
            if (this.peek == ' ' || this.peek == '\t' || this.peek == '\r' || this.peek == '\n') {
                if (this.peek == '\n') {
                    this.lineNo++;
                }
                continue;
            }
            break;
        } else {
            char ch = this.peek;
            int index = this.iterator.getIndex();
            nextChar();
            return new CharToken(ch, this.lineNo, index);
        }
    }
    // if it is a hex digit
    if (Character.isDigit(this.peek) && this.peek == '0') {
        nextChar();
        if (this.peek == 'x' || this.peek == 'X') {
            nextChar();
            StringBuilder sb = new StringBuilder();
            int startIndex = this.iterator.getIndex() - 2;
            long value = 0L;
            do {
                sb.append(this.peek);
                value = 16 * value + Character.digit(this.peek, 16);
                nextChar();
            } while (isValidHexChar(this.peek));
            return new NumberToken(value, sb.toString(), this.lineNo, startIndex);
        } else {
            prevChar();
        }
    }
    // If it is a digit
    if (Character.isDigit(this.peek) || this.peek == '.') {
        StringBuilder sb = new StringBuilder();
        int startIndex = this.iterator.getIndex();
        long lval = 0L;
        double dval = 0d;
        boolean hasDot = false;
        double d = 10.0;
        boolean isBigInt = false;
        boolean isBigDecimal = false;
        boolean scientificNotation = false;
        boolean negExp = false;
        boolean isOverflow = false;
        do {
            sb.append(this.peek);
            if (this.peek == '.') {
                if (scientificNotation) {
                    throw new CompileExpressionErrorException("Illegal number " + sb + " at " + this.iterator.getIndex());
                }
                if (hasDot) {
                    throw new CompileExpressionErrorException("Illegal Number " + sb + " at " + this.iterator.getIndex());
                } else {
                    hasDot = true;
                    nextChar();
                }
            } else if (this.peek == 'N') {
                // big integer
                if (hasDot) {
                    throw new CompileExpressionErrorException("Illegal number " + sb + " at " + this.iterator.getIndex());
                }
                isBigInt = true;
                nextChar();
                break;
            } else if (this.peek == 'M') {
                isBigDecimal = true;
                nextChar();
                break;
            } else if (this.peek == 'e' || this.peek == 'E') {
                if (scientificNotation) {
                    throw new CompileExpressionErrorException("Illegal number " + sb + " at " + this.iterator.getIndex());
                }
                scientificNotation = true;
                nextChar();
                if (this.peek == '-') {
                    negExp = true;
                    sb.append(this.peek);
                    nextChar();
                }
            } else {
                int digit = Character.digit(this.peek, 10);
                if (scientificNotation) {
                    int n = digit;
                    nextChar();
                    while (Character.isDigit(this.peek)) {
                        sb.append(this.peek);
                        n = 10 * n + Character.digit(this.peek, 10);
                        nextChar();
                    }
                    while (n-- > 0) {
                        if (negExp) {
                            dval = dval / 10;
                        } else {
                            dval = 10 * dval;
                        }
                    }
                    hasDot = true;
                } else if (hasDot) {
                    dval = dval + digit / d;
                    d = d * 10;
                    nextChar();
                } else {
                    if (!isOverflow && (lval > OVERFLOW_FLAG || (lval == OVERFLOW_FLAG && digit > OVERFLOW_SINGLE))) {
                        isOverflow = true;
                    }
                    lval = 10 * lval + digit;
                    dval = 10 * dval + digit;
                    nextChar();
                }
            }
        } while (Character.isDigit(this.peek) || this.peek == '.' || this.peek == 'E' || this.peek == 'e' || this.peek == 'M' || this.peek == 'N');
        Number value;
        if (isBigDecimal) {
            value = new BigDecimal(getBigNumberLexeme(sb), this.mathContext);
        } else if (isBigInt) {
            value = new BigInteger(getBigNumberLexeme(sb));
        } else if (hasDot) {
            if (this.parseFloatIntoDecimal && sb.length() > 1) {
                value = new BigDecimal(sb.toString(), this.mathContext);
            } else if (sb.length() == 1) {
                // only have a dot character.
                return new CharToken('.', this.lineNo, startIndex);
            } else {
                value = dval;
            }
        } else {
            if (this.parseIntegralNumberIntoDecimal) {
                // we make integral number as a BigDecimal.
                value = new BigDecimal(sb.toString(), this.mathContext);
            } else {
                // The long value is overflow, we should prompt it to be a BigInteger.
                if (isOverflow) {
                    value = new BigInteger(sb.toString());
                } else {
                    value = lval;
                }
            }
        }
        String lexeme = sb.toString();
        if (isBigDecimal || isBigInt) {
            lexeme = lexeme.substring(0, lexeme.length() - 1);
        }
        return new NumberToken(value, lexeme, this.lineNo, startIndex);
    }
    // It is a variable
    if (this.peek == '#') {
        int startIndex = this.iterator.getIndex();
        // skip '#'
        nextChar();
        boolean hasBackquote = false;
        if (this.peek == '#') {
            // ## comments
            while (this.peek != CharacterIterator.DONE && this.peek != '\n') {
                nextChar();
            }
            return this.scan(analyse);
        } else if (this.peek == '`') {
            hasBackquote = true;
            nextChar();
        }
        StringBuilder sb = new StringBuilder();
        if (hasBackquote) {
            while (this.peek != '`') {
                if (this.peek == CharacterIterator.DONE) {
                    throw new CompileExpressionErrorException("EOF while reading string at index: " + this.iterator.getIndex());
                }
                sb.append(this.peek);
                nextChar();
            }
            // skip '`'
            nextChar();
        } else {
            while (Character.isJavaIdentifierPart(this.peek) || this.peek == '.' || this.peek == '[' || this.peek == ']') {
                sb.append(this.peek);
                nextChar();
            }
        }
        String lexeme = sb.toString();
        if (lexeme.isEmpty()) {
            throw new ExpressionSyntaxErrorException("Blank variable name after '#'");
        }
        Variable variable = new Variable(lexeme, this.lineNo, startIndex);
        variable.setQuote(true);
        return this.symbolTable.reserve(variable);
    }
    if (Character.isJavaIdentifierStart(this.peek)) {
        int startIndex = this.iterator.getIndex();
        StringBuilder sb = new StringBuilder();
        do {
            sb.append(this.peek);
            nextChar();
        } while (Character.isJavaIdentifierPart(this.peek) || this.peek == '.');
        String lexeme = sb.toString();
        Variable variable = new Variable(lexeme, this.lineNo, startIndex);
        return this.symbolTable.reserve(variable);
    }
    if (isBinaryOP(this.peek)) {
        CharToken opToken = new CharToken(this.peek, this.lineNo, this.iterator.getIndex());
        nextChar();
        return opToken;
    }
    // String
    if (this.peek == '"' || this.peek == '\'') {
        char left = this.peek;
        int startIndex = this.iterator.getIndex();
        StringBuilder sb = new StringBuilder();
        boolean hasInterpolation = false;
        // char prev = this.peek;
        while ((this.peek = this.iterator.next()) != left) {
            // It's not accurate,but acceptable.
            if (this.peek == '#' && !hasInterpolation) {
                hasInterpolation = true;
            }
            if (this.peek == '\\') {
                // escape
                nextChar();
                if (this.peek == CharacterIterator.DONE) {
                    throw new CompileExpressionErrorException("EOF while reading string at index: " + this.iterator.getIndex());
                }
                if (this.peek == left) {
                    sb.append(this.peek);
                    continue;
                }
                switch(this.peek) {
                    case 't':
                        this.peek = '\t';
                        break;
                    case 'r':
                        this.peek = '\r';
                        break;
                    case 'n':
                        this.peek = '\n';
                        break;
                    case '\\':
                        break;
                    case 'b':
                        this.peek = '\b';
                        break;
                    case 'f':
                        this.peek = '\f';
                        break;
                    case '#':
                        hasInterpolation = hasInterpolation || true;
                        if (this.instance.isFeatureEnabled(Feature.StringInterpolation)) {
                            sb.append('\\');
                            this.peek = '#';
                            break;
                        }
                    default:
                        {
                            throw new CompileExpressionErrorException("Unsupported escape character: \\" + this.peek);
                        }
                }
            }
            if (this.peek == CharacterIterator.DONE) {
                throw new CompileExpressionErrorException("EOF while reading string at index: " + this.iterator.getIndex());
            }
            sb.append(this.peek);
        }
        nextChar();
        return new StringToken(sb.toString(), this.lineNo, startIndex).withMeta(Constants.INTER_META, hasInterpolation);
    }
    Token<Character> token = new CharToken(this.peek, this.lineNo, this.iterator.getIndex());
    nextChar();
    return token;
}

Example 2 with CharToken

use of com.googlecode.aviator.lexer.token.CharToken in project aviatorscript by killme2008.

the class ExpressionParser method factor0.

private boolean factor0() {
    if (this.lookhead == null) {
        reportSyntaxError("illegal token");
    }
    if (this.lookhead == Variable.END) {
        return false;
    }
    if (expectChar('(')) {
        move(true);
        this.scope.enterParen();
        ternary();
        if (expectChar(')')) {
            move(true);
            this.scope.leaveParen();
        }
    } else if (this.lookhead.getType() == TokenType.Number || this.lookhead.getType() == TokenType.String || this.lookhead.getType() == TokenType.Variable || this.lookhead == Variable.TRUE || this.lookhead == Variable.FALSE || isOPVariable(this.lookhead)) {
        if (this.lookhead.getType() == TokenType.Variable) {
            checkVariableName(this.lookhead);
        }
        // binary operation as variable for seq functions
        if (this.lookhead.getType() == TokenType.Char) {
            CharToken charToken = (CharToken) this.lookhead;
            if (!ExpressionLexer.isBinaryOP(charToken.getCh())) {
                reportSyntaxError("unexpect char '" + charToken.getCh() + "'");
            }
            // make it as variable
            this.lookhead = this.lexer.getSymbolTable().reserve(new Variable(charToken.getLexeme(), charToken.getLineNo(), charToken.getStartIndex()));
        }
        move(true);
        // function
        Token<?> prev = getPrevToken();
        if (prev.getType() == TokenType.Variable && expectChar('(')) {
            if (prev == Variable.LAMBDA) {
                lambda(false);
            } else if (prev == Variable.FN) {
                lambda(true);
            } else {
                method(prev);
            }
        } else if (prev.getType() == TokenType.Variable) {
            if (!arrayAccess()) {
                getCodeGeneratorWithTimes().onConstant(prev);
            }
        } else {
            getCodeGeneratorWithTimes().onConstant(prev);
        }
    } else if (expectChar('/')) {
        pattern();
    } else if (expectChar('}')) {
        return false;
    } else {
        reportSyntaxError("invalid token");
    }
    return true;
}

Example 3 with CharToken

use of com.googlecode.aviator.lexer.token.CharToken in project aviatorscript by killme2008.

the class ExpressionParser method equality.

public void equality() {
    rel();
    while (true) {
        Token<?> opToken = this.lookhead;
        Token<?> prevToken = getPrevToken();
        if (expectChar('=')) {
            move(true);
            if (expectChar('=')) {
                move(true);
                rel();
                getCodeGeneratorWithTimes().onEq(opToken);
            } else if (expectChar('~')) {
                // It is a regular expression
                move(true);
                rel();
                getCodeGeneratorWithTimes().onMatch(opToken);
            } else {
                // this.back();
                // assignment
                boolean isVar = false;
                if (prevToken.getType() == TokenType.Variable) {
                    isVar = true;
                } else if (prevToken.getType() == TokenType.Char && ((CharToken) prevToken).getCh() == ']') {
                    int depth = 1;
                    boolean beginSearch = false;
                    boolean found = false;
                    for (Token<?> t : this.prevTokens) {
                        if (!beginSearch && t == prevToken) {
                            beginSearch = true;
                            continue;
                        }
                        if (beginSearch && t.getType() == TokenType.Char) {
                            CharToken chToken = (CharToken) t;
                            switch(chToken.getCh()) {
                                case ']':
                                    depth++;
                                    break;
                                case '[':
                                    depth--;
                                    break;
                            }
                            if (depth == 0) {
                                found = true;
                                continue;
                            }
                        }
                        if (found) {
                            if (t.getType() == TokenType.Variable) {
                                t.withMeta(Constants.TYPE_META, CompileTypes.Array);
                            }
                            break;
                        }
                    }
                }
                statement();
                // try to find var(prevToken) in right statement, it's not initialized if presents.
                if (isVar) {
                    checkVarIsInit(prevToken);
                }
                ensureFeatureEnabled(Feature.Assignment);
                getCodeGeneratorWithTimes().onAssignment(opToken);
            }
        } else if (expectChar('!')) {
            move(true);
            if (expectChar('=')) {
                move(true);
                rel();
                getCodeGeneratorWithTimes().onNeq(opToken);
            } else {
                reportSyntaxError("expect '='");
            }
        } else {
            break;
        }
    }
}

Example 4 with CharToken

use of com.googlecode.aviator.lexer.token.CharToken in project aviatorscript by killme2008.

the class ExpressionParser method pattern.

private void pattern() {
    // It is a pattern
    int startIndex = this.lookhead.getStartIndex();
    move(true);
    this.inPattern = true;
    StringBuilder sb = new StringBuilder();
    while (this.lookhead != null) {
        while (!expectChar('/')) {
            sb.append(this.lookhead.getLexeme());
            move(false);
        }
        if (getPrevToken().getType() == TokenType.Char && ((CharToken) getPrevToken()).getLexeme().equals("\\")) {
            sb.append("/");
            move(false);
            continue;
        }
        this.inPattern = false;
        break;
    }
    if (this.inPattern) {
        reportSyntaxError("invalid regular pattern:" + sb.toString());
    }
    getCodeGeneratorWithTimes().onConstant(new PatternToken(sb.toString(), this.lexer.getLineNo(), startIndex));
    move(true);
}

Example 5 with CharToken

use of com.googlecode.aviator.lexer.token.CharToken in project aviatorscript by killme2008.

the class AviatorEvaluatorInstance method compileStringSegments.

/**
 * Compile a string to string segments, if string doesn't have a interpolation,returns an empty
 * list.
 *
 * @param lexeme
 * @param sourceFile
 * @param lineNo;
 * @return
 */
public StringSegments compileStringSegments(final String lexeme, final String sourceFile, final int lineNo) {
    List<StringSegment> segs = new ArrayList<StringSegment>();
    boolean hasInterpolationOrEscaped = false;
    StringCharacterIterator it = new StringCharacterIterator(lexeme);
    char ch = it.current(), prev = StringCharacterIterator.DONE;
    int lastInterPos = 0;
    int i = 1;
    for (; ; ) {
        if (ch == '#') {
            if (prev == '\\') {
                // # is escaped, skip the backslash.
                final String segStr = lexeme.substring(lastInterPos, i - 2);
                segs.add(new LiteralSegment(segStr));
                lastInterPos = i - 1;
                hasInterpolationOrEscaped = true;
            } else {
                // # is not escaped.
                prev = ch;
                ch = it.next();
                i++;
                if (ch == '{') {
                    // Find a interpolation position.
                    if (i - 2 > lastInterPos) {
                        final String segStr = lexeme.substring(lastInterPos, i - 2);
                        segs.add(new LiteralSegment(segStr));
                    }
                    try {
                        ExpressionLexer lexer = new ExpressionLexer(this, lexeme.substring(i));
                        lexer.setLineNo(lineNo);
                        ExpressionParser parser = new ExpressionParser(this, lexer, newCodeGenerator(sourceFile, false));
                        Expression exp = parser.parse(false);
                        final Token<?> lookhead = parser.getLookhead();
                        if (lookhead == null || (lookhead.getType() != TokenType.Char || ((CharToken) lookhead).getCh() != '}')) {
                            parser.reportSyntaxError("expect '}' to complete string interpolation");
                        }
                        int expStrLen = lookhead.getStartIndex() + 1;
                        while (expStrLen-- > 0) {
                            prev = ch;
                            ch = it.next();
                            i++;
                        }
                        Token<?> previousToken = null;
                        if (parser.getParsedTokens() == 2 && (previousToken = parser.getPrevToken()) != null && previousToken.getType() == TokenType.Variable) {
                            // special case for inline variable.
                            if (previousToken == Variable.TRUE) {
                                segs.add(new LiteralSegment("true"));
                            } else if (previousToken == Variable.FALSE) {
                                segs.add(new LiteralSegment("false"));
                            } else if (previousToken == Variable.NIL) {
                                segs.add(new LiteralSegment("null"));
                            } else {
                                segs.add(new VarSegment(parser.getSymbolTable().reserve(previousToken.getLexeme()).getLexeme()));
                            }
                        } else {
                            segs.add(new ExpressionSegment(exp));
                        }
                        hasInterpolationOrEscaped = true;
                        lastInterPos = i;
                    } catch (Throwable t) {
                        throw new CompileExpressionErrorException("Fail to compile string interpolation: " + lexeme, t);
                    }
                // End of interpolation
                }
            // End of # is not escaped.
            }
        }
        if (ch == StringCharacterIterator.DONE) {
            if (i - 1 > lastInterPos) {
                final String segStr = lexeme.substring(lastInterPos, i - 1);
                segs.add(new LiteralSegment(segStr));
            }
            break;
        }
        prev = ch;
        ch = it.next();
        i++;
    }
    if (hasInterpolationOrEscaped) {
        return new StringSegments(segs, lexeme.length() * 2 / 3);
    } else {
        return new StringSegments(Collections.<StringSegment>emptyList(), 0);
    }
}