Examples with Tokenizer io.shardingjdbc.core.parsing.lexer.analyzer.Tokenizer used on opensource projects

Example 1 with Tokenizer

use of io.shardingjdbc.core.parsing.lexer.analyzer.Tokenizer in project sharding-jdbc by shardingjdbc.

the class Lexer method nextToken.

/**
 * Analyse next token.
 */
public final void nextToken() {
    skipIgnoredToken();
    if (isVariableBegin()) {
        currentToken = new Tokenizer(input, dictionary, offset).scanVariable();
    } else if (isNCharBegin()) {
        currentToken = new Tokenizer(input, dictionary, ++offset).scanChars();
    } else if (isIdentifierBegin()) {
        currentToken = new Tokenizer(input, dictionary, offset).scanIdentifier();
    } else if (isHexDecimalBegin()) {
        currentToken = new Tokenizer(input, dictionary, offset).scanHexDecimal();
    } else if (isNumberBegin()) {
        currentToken = new Tokenizer(input, dictionary, offset).scanNumber();
    } else if (isSymbolBegin()) {
        currentToken = new Tokenizer(input, dictionary, offset).scanSymbol();
    } else if (isCharsBegin()) {
        currentToken = new Tokenizer(input, dictionary, offset).scanChars();
    } else if (isEnd()) {
        currentToken = new Token(Assist.END, "", offset);
    } else {
        throw new SQLParsingException(this, Assist.ERROR);
    }
    offset = currentToken.getEndPosition();
}