Examples with Token com.alibaba.druid.sql.parser.Token used on opensource projects

Example 6 with Token

use of com.alibaba.druid.sql.parser.Token in project druid by alibaba.

the class MySqlLexer method scanIdentifier.

public void scanIdentifier() {
    final char first = ch;
    if (ch == '`') {
        mark = pos;
        bufPos = 1;
        char ch;
        for (; ; ) {
            ch = charAt(++pos);
            if (ch == '`') {
                bufPos++;
                ch = charAt(++pos);
                break;
            } else if (ch == EOI) {
                throw new ParserException("illegal identifier");
            }
            bufPos++;
            continue;
        }
        this.ch = charAt(pos);
        stringVal = subString(mark, bufPos);
        Token tok = keywods.getKeyword(stringVal);
        if (tok != null) {
            token = tok;
        } else {
            token = Token.IDENTIFIER;
        }
    } else {
        final boolean firstFlag = isFirstIdentifierChar(first);
        if (!firstFlag) {
            throw new ParserException("illegal identifier");
        }
        mark = pos;
        bufPos = 1;
        char ch = '\0', last_ch;
        for (; ; ) {
            last_ch = ch;
            ch = charAt(++pos);
            if (!isIdentifierChar(ch)) {
                if (ch == '-' && pos < text.length() - 1) {
                    if (mark > 0 && text.charAt(mark - 1) == '.') {
                        break;
                    }
                    char next_char = text.charAt(pos + 1);
                    if (isIdentifierChar(next_char)) {
                        bufPos++;
                        continue;
                    }
                }
                if (last_ch == '-' && charAt(pos - 2) != '-') {
                    ch = last_ch;
                    bufPos--;
                    pos--;
                }
                break;
            }
            bufPos++;
            continue;
        }
        this.ch = charAt(pos);
        stringVal = addSymbol();
        Token tok = keywods.getKeyword(stringVal);
        if (tok != null) {
            token = tok;
        } else {
            token = Token.IDENTIFIER;
        }
    }
}

Example 7 with Token

use of com.alibaba.druid.sql.parser.Token in project druid by alibaba.

the class MySqlLexer method scanSharp.

public void scanSharp() {
    if (ch != '#') {
        throw new ParserException("illegal stat");
    }
    if (charAt(pos + 1) == '{') {
        scanVariable();
        return;
    }
    Token lastToken = this.token;
    scanChar();
    mark = pos;
    bufPos = 0;
    for (; ; ) {
        if (ch == '\r') {
            if (charAt(pos + 1) == '\n') {
                bufPos += 2;
                scanChar();
                break;
            }
            bufPos++;
            break;
        } else if (ch == EOI) {
            break;
        }
        if (ch == '\n') {
            scanChar();
            bufPos++;
            break;
        }
        scanChar();
        bufPos++;
    }
    stringVal = subString(mark - 1, bufPos + 1);
    token = Token.LINE_COMMENT;
    commentCount++;
    if (keepComments) {
        addComment(stringVal);
    }
    if (commentHandler != null && commentHandler.handle(lastToken, stringVal)) {
        return;
    }
    endOfComment = isEOF();
    if (!isAllowComment() && (isEOF() || !isSafeComment(stringVal))) {
        throw new NotAllowCommentException();
    }
}

Example 8 with Token

use of com.alibaba.druid.sql.parser.Token in project druid by alibaba.

the class OdpsLexer method scanIdentifier.

public void scanIdentifier() {
    final char first = ch;
    if (first == '`') {
        mark = pos;
        bufPos = 1;
        char ch;
        for (; ; ) {
            ch = charAt(++pos);
            if (ch == '`') {
                bufPos++;
                ch = charAt(++pos);
                break;
            } else if (ch == EOI) {
                throw new ParserException("illegal identifier");
            }
            bufPos++;
            continue;
        }
        this.ch = charAt(pos);
        stringVal = subString(mark, bufPos);
        token = Token.IDENTIFIER;
        return;
    }
    final boolean firstFlag = isFirstIdentifierChar(first);
    if (!firstFlag) {
        throw new ParserException("illegal identifier");
    }
    mark = pos;
    bufPos = 1;
    char ch;
    for (; ; ) {
        ch = charAt(++pos);
        if (!isIdentifierChar(ch)) {
            break;
        }
        bufPos++;
        continue;
    }
    this.ch = charAt(pos);
    if (ch == '@') {
        // for user identifier, like email, xx@alibaba-inc.com
        bufPos++;
        for (; ; ) {
            ch = charAt(++pos);
            if (ch != '-' && ch != '.' && !isIdentifierChar(ch)) {
                break;
            }
            bufPos++;
            continue;
        }
    }
    this.ch = charAt(pos);
    stringVal = addSymbol();
    Token tok = keywods.getKeyword(stringVal);
    if (tok != null) {
        token = tok;
    } else {
        token = Token.IDENTIFIER;
    }
}

Example 9 with Token

use of com.alibaba.druid.sql.parser.Token in project druid by alibaba.

the class OdpsLexer method scanComment.

public void scanComment() {
    if (ch != '/' && ch != '-') {
        throw new IllegalStateException();
    }
    Token lastToken = this.token;
    mark = pos;
    bufPos = 0;
    scanChar();
    // /*+ */
    if (ch == '*') {
        scanChar();
        bufPos++;
        while (ch == ' ') {
            scanChar();
            bufPos++;
        }
        boolean isHint = false;
        int startHintSp = bufPos + 1;
        if (ch == '+') {
            isHint = true;
            scanChar();
            bufPos++;
        }
        for (; ; ) {
            if (ch == '*' && charAt(pos + 1) == '/') {
                bufPos += 2;
                scanChar();
                scanChar();
                break;
            }
            scanChar();
            bufPos++;
        }
        if (isHint) {
            stringVal = subString(mark + startHintSp, (bufPos - startHintSp) - 1);
            token = Token.HINT;
        } else {
            stringVal = subString(mark, bufPos + 1);
            token = Token.MULTI_LINE_COMMENT;
            commentCount++;
            if (keepComments) {
                addComment(stringVal);
            }
        }
        if (commentHandler != null && commentHandler.handle(lastToken, stringVal)) {
            return;
        }
        if (token != Token.HINT && !isAllowComment()) {
            throw new NotAllowCommentException();
        }
        return;
    }
    if (!isAllowComment()) {
        throw new NotAllowCommentException();
    }
    if (ch == '/' || ch == '-') {
        scanChar();
        bufPos++;
        for (; ; ) {
            if (ch == '\r') {
                if (charAt(pos + 1) == '\n') {
                    line++;
                    bufPos += 2;
                    scanChar();
                    break;
                }
                bufPos++;
                break;
            } else if (ch == EOI) {
                break;
            }
            if (ch == '\n') {
                line++;
                scanChar();
                bufPos++;
                break;
            }
            scanChar();
            bufPos++;
        }
        stringVal = subString(mark, ch != EOI ? bufPos : bufPos + 1);
        token = Token.LINE_COMMENT;
        commentCount++;
        if (keepComments) {
            addComment(stringVal);
        }
        endOfComment = isEOF();
        if (commentHandler != null && commentHandler.handle(lastToken, stringVal)) {
            return;
        }
        return;
    }
}

Example 10 with Token

use of com.alibaba.druid.sql.parser.Token in project druid by alibaba.

the class OracleLexer method scanVariable.

public void scanVariable() {
    if (ch == '@') {
        scanChar();
        token = Token.MONKEYS_AT;
        return;
    }
    if (ch != ':' && ch != '#' && ch != '$') {
        throw new ParserException("illegal variable");
    }
    mark = pos;
    bufPos = 1;
    char ch;
    boolean quoteFlag = false;
    boolean mybatisFlag = false;
    if (charAt(pos + 1) == '"') {
        pos++;
        bufPos++;
        quoteFlag = true;
    } else if (charAt(pos + 1) == '{') {
        pos++;
        bufPos++;
        mybatisFlag = true;
    }
    for (; ; ) {
        ch = charAt(++pos);
        if (!isIdentifierChar(ch)) {
            break;
        }
        bufPos++;
        continue;
    }
    if (quoteFlag) {
        if (ch != '"') {
            throw new ParserException("syntax error");
        }
        ++pos;
        bufPos++;
    } else if (mybatisFlag) {
        if (ch != '}') {
            throw new ParserException("syntax error");
        }
        ++pos;
        bufPos++;
    }
    this.ch = charAt(pos);
    stringVal = addSymbol();
    Token tok = keywods.getKeyword(stringVal);
    if (tok != null) {
        token = tok;
    } else {
        token = Token.VARIANT;
    }
}