Examples with Token com.alibaba.druid.sql.parser.Token used on opensource projects

Example 21 with Token

use of com.alibaba.druid.sql.parser.Token in project druid by alibaba.

the class Lexer method scanIdentifier.

public void scanIdentifier() {
    this.hash_lower = 0;
    this.hash = 0;
    final char first = ch;
    if (ch == '`') {
        mark = pos;
        bufPos = 1;
        char ch;
        int startPos = pos + 1;
        int quoteIndex = text.indexOf('`', startPos);
        if (quoteIndex == -1) {
            throw new ParserException("illegal identifier. " + info());
        }
        hash_lower = 0xcbf29ce484222325L;
        hash = 0xcbf29ce484222325L;
        for (int i = startPos; i < quoteIndex; ++i) {
            ch = text.charAt(i);
            hash_lower ^= ((ch >= 'A' && ch <= 'Z') ? (ch + 32) : ch);
            hash_lower *= 0x100000001b3L;
            hash ^= ch;
            hash *= 0x100000001b3L;
        }
        stringVal = MySqlLexer.quoteTable.addSymbol(text, pos, quoteIndex + 1 - pos, hash);
        // stringVal = text.substring(mark, pos);
        pos = quoteIndex + 1;
        this.ch = charAt(pos);
        token = Token.IDENTIFIER;
        return;
    }
    final boolean firstFlag = isFirstIdentifierChar(first);
    if (!firstFlag) {
        throw new ParserException("illegal identifier. " + info());
    }
    hash_lower = 0xcbf29ce484222325L;
    hash = 0xcbf29ce484222325L;
    hash_lower ^= ((ch >= 'A' && ch <= 'Z') ? (ch + 32) : ch);
    hash_lower *= 0x100000001b3L;
    hash ^= ch;
    hash *= 0x100000001b3L;
    mark = pos;
    bufPos = 1;
    char ch = 0;
    for (; ; ) {
        char c0 = ch;
        ch = charAt(++pos);
        if (!isIdentifierChar(ch)) {
            if ((ch == '（' || ch == '）') && c0 > 256) {
                bufPos++;
                continue;
            }
            break;
        }
        hash_lower ^= ((ch >= 'A' && ch <= 'Z') ? (ch + 32) : ch);
        hash_lower *= 0x100000001b3L;
        hash ^= ch;
        hash *= 0x100000001b3L;
        bufPos++;
        continue;
    }
    this.ch = charAt(pos);
    if (bufPos == 1) {
        switch(first) {
            case '（':
                token = Token.LPAREN;
                return;
            case '）':
                token = Token.RPAREN;
                return;
            default:
                break;
        }
        token = Token.IDENTIFIER;
        stringVal = CharTypes.valueOf(first);
        if (stringVal == null) {
            stringVal = Character.toString(first);
        }
        return;
    }
    Token tok = keywords.getKeyword(hash_lower);
    if (tok != null) {
        token = tok;
        if (token == Token.IDENTIFIER) {
            stringVal = SymbolTable.global.addSymbol(text, mark, bufPos, hash);
        } else {
            stringVal = null;
        }
    } else {
        token = Token.IDENTIFIER;
        stringVal = SymbolTable.global.addSymbol(text, mark, bufPos, hash);
    }
}

Example 22 with Token

use of com.alibaba.druid.sql.parser.Token in project druid by alibaba.

the class Lexer method scanSingleLineComment.

private void scanSingleLineComment() {
    Token lastToken = this.token;
    mark = pos;
    bufPos = 2;
    scanChar();
    scanChar();
    for (; ; ) {
        if (ch == '\r') {
            if (charAt(pos + 1) == '\n') {
                line++;
                scanChar();
                break;
            }
            bufPos++;
            break;
        }
        if (ch == '\n') {
            line++;
            scanChar();
            break;
        }
        if (ch == EOI) {
            break;
        }
        scanChar();
        bufPos++;
    }
    stringVal = subString(mark, bufPos);
    token = Token.LINE_COMMENT;
    commentCount++;
    if (keepComments) {
        addComment(stringVal);
    }
    if (commentHandler != null && commentHandler.handle(lastToken, stringVal)) {
        return;
    }
    if (!isAllowComment() && !isSafeComment(stringVal)) {
        throw new NotAllowCommentException();
    }
}

Example 23 with Token

use of com.alibaba.druid.sql.parser.Token in project druid by alibaba.

the class WallProvider method checkInternal.

private WallCheckResult checkInternal(String sql) {
    checkCount.incrementAndGet();
    WallContext context = WallContext.current();
    if (config.isDoPrivilegedAllow() && ispPrivileged()) {
        WallCheckResult checkResult = new WallCheckResult();
        checkResult.setSql(sql);
        return checkResult;
    }
    // first step, check whiteList
    boolean mulltiTenant = config.getTenantTablePattern() != null && config.getTenantTablePattern().length() > 0;
    if (!mulltiTenant) {
        WallCheckResult checkResult = checkWhiteAndBlackList(sql);
        if (checkResult != null) {
            checkResult.setSql(sql);
            return checkResult;
        }
    }
    hardCheckCount.incrementAndGet();
    final List<Violation> violations = new ArrayList<Violation>();
    List<SQLStatement> statementList = new ArrayList<SQLStatement>();
    boolean syntaxError = false;
    boolean endOfComment = false;
    try {
        SQLStatementParser parser = createParser(sql);
        parser.getLexer().setCommentHandler(WallCommentHandler.instance);
        if (!config.isCommentAllow()) {
            // deny comment
            parser.getLexer().setAllowComment(false);
        }
        if (!config.isCompleteInsertValuesCheck()) {
            parser.setParseCompleteValues(false);
            parser.setParseValuesSize(config.getInsertValuesCheckSize());
        }
        parser.parseStatementList(statementList);
        final Token lastToken = parser.getLexer().token();
        if (lastToken != Token.EOF && config.isStrictSyntaxCheck()) {
            violations.add(new IllegalSQLObjectViolation(ErrorCode.SYNTAX_ERROR, "not terminal sql, token " + lastToken, sql));
        }
        endOfComment = parser.getLexer().isEndOfComment();
    } catch (NotAllowCommentException e) {
        violations.add(new IllegalSQLObjectViolation(ErrorCode.COMMENT_STATEMENT_NOT_ALLOW, "comment not allow", sql));
        incrementCommentDeniedCount();
    } catch (ParserException e) {
        syntaxErrorCount.incrementAndGet();
        syntaxError = true;
        if (config.isStrictSyntaxCheck()) {
            violations.add(new SyntaxErrorViolation(e, sql));
        }
    } catch (Exception e) {
        if (config.isStrictSyntaxCheck()) {
            violations.add(new SyntaxErrorViolation(e, sql));
        }
    }
    if (statementList.size() > 1 && !config.isMultiStatementAllow()) {
        violations.add(new IllegalSQLObjectViolation(ErrorCode.MULTI_STATEMENT, "multi-statement not allow", sql));
    }
    WallVisitor visitor = createWallVisitor();
    visitor.setSqlEndOfComment(endOfComment);
    if (statementList.size() > 0) {
        boolean lastIsHint = false;
        for (int i = 0; i < statementList.size(); i++) {
            SQLStatement stmt = statementList.get(i);
            if ((i == 0 || lastIsHint) && stmt instanceof MySqlHintStatement) {
                lastIsHint = true;
                continue;
            }
            try {
                stmt.accept(visitor);
            } catch (ParserException e) {
                violations.add(new SyntaxErrorViolation(e, sql));
            }
        }
    }
    if (visitor.getViolations().size() > 0) {
        violations.addAll(visitor.getViolations());
    }
    Map<String, WallSqlTableStat> tableStat = context.getTableStats();
    boolean updateCheckHandlerEnable = false;
    {
        WallUpdateCheckHandler updateCheckHandler = config.getUpdateCheckHandler();
        if (updateCheckHandler != null) {
            for (SQLStatement stmt : statementList) {
                if (stmt instanceof SQLUpdateStatement) {
                    SQLUpdateStatement updateStmt = (SQLUpdateStatement) stmt;
                    SQLName table = updateStmt.getTableName();
                    if (table != null) {
                        String tableName = table.getSimpleName();
                        Set<String> updateCheckColumns = config.getUpdateCheckTable(tableName);
                        if (updateCheckColumns != null && updateCheckColumns.size() > 0) {
                            updateCheckHandlerEnable = true;
                            break;
                        }
                    }
                }
            }
        }
    }
    WallSqlStat sqlStat = null;
    if (violations.size() > 0) {
        violationCount.incrementAndGet();
        if ((!updateCheckHandlerEnable) && sql.length() < MAX_SQL_LENGTH) {
            sqlStat = addBlackSql(sql, tableStat, context.getFunctionStats(), violations, syntaxError);
        }
    } else {
        if ((!updateCheckHandlerEnable) && sql.length() < MAX_SQL_LENGTH) {
            boolean selectLimit = false;
            if (config.getSelectLimit() > 0) {
                for (SQLStatement stmt : statementList) {
                    if (stmt instanceof SQLSelectStatement) {
                        selectLimit = true;
                        break;
                    }
                }
            }
            if (!selectLimit) {
                sqlStat = addWhiteSql(sql, tableStat, context.getFunctionStats(), syntaxError);
            }
        }
    }
    if (sqlStat == null && updateCheckHandlerEnable) {
        sqlStat = new WallSqlStat(tableStat, context.getFunctionStats(), violations, syntaxError);
    }
    Map<String, WallSqlTableStat> tableStats = null;
    Map<String, WallSqlFunctionStat> functionStats = null;
    if (context != null) {
        tableStats = context.getTableStats();
        functionStats = context.getFunctionStats();
        recordStats(tableStats, functionStats);
    }
    WallCheckResult result;
    if (sqlStat != null) {
        context.setSqlStat(sqlStat);
        result = new WallCheckResult(sqlStat, statementList);
    } else {
        result = new WallCheckResult(null, violations, tableStats, functionStats, statementList, syntaxError);
    }
    String resultSql;
    if (visitor.isSqlModified()) {
        resultSql = SQLUtils.toSQLString(statementList, dbType);
    } else {
        resultSql = sql;
    }
    result.setSql(resultSql);
    result.setUpdateCheckItems(visitor.getUpdateCheckItems());
    return result;
}