Examples with Text org.eclipse.scout.rt.server.jdbc.parsers.sql.SqlParserToken.Text used on opensource projects

Example 1 with Text

use of org.eclipse.scout.rt.server.jdbc.parsers.sql.SqlParserToken.Text in project scout.rt by eclipse.

the class SqlParser method tokenize.

/**
 * Tokenize a string into Whitespace (containing Comment and HintComment), Text and Token items. There are no Raw
 * objects left.
 *
 * @throws ParseException
 */
private List<IToken> tokenize(String s, ParseContext ctx) {
    s = encodeBinds(s, ctx);
    s = s.replaceAll("[\\n\\r]+", " ");
    List<IToken> list = new ArrayList<IToken>();
    list.add(new Raw(s));
    // 
    list = tokenizeRaw(list, COMMENT_PAT, Comment.class, true);
    list = tokenizeRaw(list, APOS_PAT, Text.class, true);
    list = tokenizeRaw(list, APOS_PAT, Text.class, false);
    list = tokenizeRaw(list, QUOT_PAT, Name.class, true);
    list = tokenizeRaw(list, QUOT_PAT, Name.class, false);
    // replace all remaining whitespace by a single space and convert to upper case
    for (IToken item : list) {
        if (item instanceof Raw) {
            String text = item.getText();
            text = text.replaceAll("[\\s]+", " ");
            text = text.toUpperCase();
            item.setText(text);
        }
    }
    list = tokenizeRaw(list, UNION_PAT, UnionToken.class, false);
    list = tokenizeRaw(list, PART_PAT, PartToken.class, false);
    list = tokenizeRaw(list, OUTER_JOIN_PAT, OuterJoinToken.class, false);
    list = tokenizeRaw(list, OR_OP_PAT, OrOp.class, false);
    list = tokenizeRaw(list, AND_OP_PAT, AndOp.class, false);
    list = tokenizeRaw(list, MATH_OP_PAT1, MathOp.class, false);
    list = tokenizeRaw(list, MATH_OP_PAT2, MathOp.class, false);
    list = tokenizeRaw(list, UNARY_PREFIX_PAT, UnaryPrefix.class, false);
    list = tokenizeRaw(list, NAME_PAT, Name.class, false);
    list = tokenizeRaw(list, OPEN_BRACKET_PAT, OpenBracketToken.class, false);
    list = tokenizeRaw(list, CLOSE_BRACKET_PAT, CloseBracketToken.class, false);
    list = tokenizeRaw(list, LIST_SEPARATOR_PAT, ListSeparator.class, false);
    // eliminate all empty Raw
    for (Iterator<IToken> it = list.iterator(); it.hasNext(); ) {
        IToken item = it.next();
        if (item instanceof Raw && !StringUtility.hasText(item.getText())) {
            it.remove();
        }
    }
    // check the rest, no more Raw, convert to comment with warning
    for (int i = 0; i < list.size(); i++) {
        IToken tok = list.get(i);
        if (tok instanceof Raw) {
            list.remove(i);
            Comment c = new Comment();
            c.setText("/*X" + "X" + "X" + " unexpected token: " + tok.getText() + "*/");
            list.add(i, c);
        }
    }
    // associate comments with the successor token
    for (int i = 0; i < list.size(); i++) {
        if (list.get(i) instanceof Comment) {
            // find successor
            IToken succ = null;
            for (int k = i + 1; k < list.size(); k++) {
                if (!(list.get(k) instanceof Comment)) {
                    succ = list.get(k);
                    break;
                }
            }
            if (succ == null) {
                for (int k = 0; k < list.size(); k++) {
                    if (!(list.get(k) instanceof Comment)) {
                        succ = list.get(k);
                        break;
                    }
                }
            }
            if (succ != null) {
                succ.addComment((Comment) list.get(i));
            }
        }
    }
    for (Iterator<IToken> it = list.iterator(); it.hasNext(); ) {
        IToken item = it.next();
        if (item instanceof Comment) {
            it.remove();
        }
    }
    decodeBinds(list, ctx);
    return list;
}