Examples with Raw org.eclipse.scout.rt.server.jdbc.parsers.sql.SqlParserToken.Raw used on opensource projects

Example 1 with Raw

use of org.eclipse.scout.rt.server.jdbc.parsers.sql.SqlParserToken.Raw in project scout.rt by eclipse.

the class SqlParser method tokenize.

/**
 * Tokenize a string into Whitespace (containing Comment and HintComment), Text and Token items. There are no Raw
 * objects left.
 *
 * @throws ParseException
 */
private List<IToken> tokenize(String s, ParseContext ctx) {
    s = encodeBinds(s, ctx);
    s = s.replaceAll("[\\n\\r]+", " ");
    List<IToken> list = new ArrayList<IToken>();
    list.add(new Raw(s));
    // 
    list = tokenizeRaw(list, COMMENT_PAT, Comment.class, true);
    list = tokenizeRaw(list, APOS_PAT, Text.class, true);
    list = tokenizeRaw(list, APOS_PAT, Text.class, false);
    list = tokenizeRaw(list, QUOT_PAT, Name.class, true);
    list = tokenizeRaw(list, QUOT_PAT, Name.class, false);
    // replace all remaining whitespace by a single space and convert to upper case
    for (IToken item : list) {
        if (item instanceof Raw) {
            String text = item.getText();
            text = text.replaceAll("[\\s]+", " ");
            text = text.toUpperCase();
            item.setText(text);
        }
    }
    list = tokenizeRaw(list, UNION_PAT, UnionToken.class, false);
    list = tokenizeRaw(list, PART_PAT, PartToken.class, false);
    list = tokenizeRaw(list, OUTER_JOIN_PAT, OuterJoinToken.class, false);
    list = tokenizeRaw(list, OR_OP_PAT, OrOp.class, false);
    list = tokenizeRaw(list, AND_OP_PAT, AndOp.class, false);
    list = tokenizeRaw(list, MATH_OP_PAT1, MathOp.class, false);
    list = tokenizeRaw(list, MATH_OP_PAT2, MathOp.class, false);
    list = tokenizeRaw(list, UNARY_PREFIX_PAT, UnaryPrefix.class, false);
    list = tokenizeRaw(list, NAME_PAT, Name.class, false);
    list = tokenizeRaw(list, OPEN_BRACKET_PAT, OpenBracketToken.class, false);
    list = tokenizeRaw(list, CLOSE_BRACKET_PAT, CloseBracketToken.class, false);
    list = tokenizeRaw(list, LIST_SEPARATOR_PAT, ListSeparator.class, false);
    // eliminate all empty Raw
    for (Iterator<IToken> it = list.iterator(); it.hasNext(); ) {
        IToken item = it.next();
        if (item instanceof Raw && !StringUtility.hasText(item.getText())) {
            it.remove();
        }
    }
    // check the rest, no more Raw, convert to comment with warning
    for (int i = 0; i < list.size(); i++) {
        IToken tok = list.get(i);
        if (tok instanceof Raw) {
            list.remove(i);
            Comment c = new Comment();
            c.setText("/*X" + "X" + "X" + " unexpected token: " + tok.getText() + "*/");
            list.add(i, c);
        }
    }
    // associate comments with the successor token
    for (int i = 0; i < list.size(); i++) {
        if (list.get(i) instanceof Comment) {
            // find successor
            IToken succ = null;
            for (int k = i + 1; k < list.size(); k++) {
                if (!(list.get(k) instanceof Comment)) {
                    succ = list.get(k);
                    break;
                }
            }
            if (succ == null) {
                for (int k = 0; k < list.size(); k++) {
                    if (!(list.get(k) instanceof Comment)) {
                        succ = list.get(k);
                        break;
                    }
                }
            }
            if (succ != null) {
                succ.addComment((Comment) list.get(i));
            }
        }
    }
    for (Iterator<IToken> it = list.iterator(); it.hasNext(); ) {
        IToken item = it.next();
        if (item instanceof Comment) {
            it.remove();
        }
    }
    decodeBinds(list, ctx);
    return list;
}

Example 2 with Raw

use of org.eclipse.scout.rt.server.jdbc.parsers.sql.SqlParserToken.Raw in project scout.rt by eclipse.

the class SqlParser method tokenizeRaw.

@SuppressWarnings("squid:S1643")
private List<IToken> tokenizeRaw(List<IToken> list, Pattern p, Class<? extends IToken> tokenType, boolean transcodeDelimiters) {
    ArrayList<IToken> newList = new ArrayList<IToken>(list.size());
    for (IToken item : list) {
        if (item instanceof Raw) {
            String s = ((Raw) item).getText();
            if (transcodeDelimiters) {
                s = encodeDelimiters(s);
            }
            // extend s to start and end with an empty text (simpler regex can be used then)
            s = " " + s + " ";
            Matcher m = p.matcher(s);
            int lastEnd = 0;
            while (lastEnd < s.length() && m.find(lastEnd)) {
                String r = s.substring(lastEnd, m.start(1));
                if (transcodeDelimiters) {
                    r = decodeDelimiters(r);
                }
                newList.add(new Raw(r.trim()));
                // 
                r = m.group(1);
                if (transcodeDelimiters) {
                    r = decodeDelimiters(r);
                }
                IToken t;
                try {
                    t = tokenType.newInstance();
                } catch (Exception e) {
                    throw new ProcessingException("Could not create new instance of {}", tokenType, e);
                }
                t.setText(r);
                newList.add(t);
                // 
                lastEnd = m.end(1);
            }
            // remaining part
            if (lastEnd < s.length()) {
                String r = s.substring(lastEnd);
                if (transcodeDelimiters) {
                    r = decodeDelimiters(r);
                }
                newList.add(new Raw(r.trim()));
            }
        } else {
            newList.add(item);
        }
    }
    return newList;
}