Examples with IToken org.eclipse.jface.text.rules.IToken used on opensource projects

Example 46 with IToken

use of org.eclipse.jface.text.rules.IToken in project dbeaver by dbeaver.

the class SQLEditorBase method parseQuery.

protected SQLScriptElement parseQuery(final IDocument document, final int startPos, final int endPos, final int currentPos, final boolean scriptMode, final boolean keepDelimiters) {
    if (endPos - startPos <= 0) {
        return null;
    }
    SQLDialect dialect = getSQLDialect();
    // Parse range
    boolean useBlankLines = !scriptMode && syntaxManager.isBlankLineDelimiter();
    ruleManager.setRange(document, startPos, endPos - startPos);
    int statementStart = startPos;
    int bracketDepth = 0;
    boolean hasBlocks = false;
    boolean hasValuableTokens = false;
    boolean hasBlockHeader = false;
    String blockTogglePattern = null;
    int lastTokenLineFeeds = 0;
    int prevNotEmptyTokenType = SQLToken.T_UNKNOWN;
    for (; ; ) {
        IToken token = ruleManager.nextToken();
        int tokenOffset = ruleManager.getTokenOffset();
        int tokenLength = ruleManager.getTokenLength();
        int tokenType = token instanceof SQLToken ? ((SQLToken) token).getType() : SQLToken.T_UNKNOWN;
        if (tokenOffset < startPos) {
            // This may happen with EOF tokens (bug in jface?)
            return null;
        }
        boolean isDelimiter = tokenType == SQLToken.T_DELIMITER;
        boolean isControl = false;
        String delimiterText = null;
        try {
            if (isDelimiter) {
                // Save delimiter text
                try {
                    delimiterText = document.get(tokenOffset, tokenLength);
                } catch (BadLocationException e) {
                    log.debug(e);
                }
            } else if (useBlankLines && token.isWhitespace() && tokenLength >= 1) {
                // Check for blank line delimiter
                if (lastTokenLineFeeds + countLineFeeds(document, tokenOffset, tokenLength) >= 2) {
                    isDelimiter = true;
                }
            }
            lastTokenLineFeeds = 0;
            if (tokenLength == 1) {
                // Check for bracket block begin/end
                try {
                    char aChar = document.getChar(tokenOffset);
                    if (aChar == '(' || aChar == '{' || aChar == '[') {
                        bracketDepth++;
                    } else if (aChar == ')' || aChar == '}' || aChar == ']') {
                        bracketDepth--;
                    }
                } catch (BadLocationException e) {
                    log.warn(e);
                }
            }
            if (tokenType == SQLToken.T_BLOCK_BEGIN && prevNotEmptyTokenType == SQLToken.T_BLOCK_END) {
                // This is a tricky thing.
                // In some dialects block end looks like END CASE, END LOOP. It is parsed as
                // Block end followed by block begin (as CASE and LOOP are block begin tokens)
                // So let's ignore block begin if previos token was block end and there were no delimtiers.
                tokenType = SQLToken.T_UNKNOWN;
            }
            if (tokenType == SQLToken.T_BLOCK_HEADER) {
                bracketDepth++;
                hasBlocks = true;
                hasBlockHeader = true;
            } else if (tokenType == SQLToken.T_BLOCK_TOGGLE) {
                String togglePattern;
                try {
                    togglePattern = document.get(tokenOffset, tokenLength);
                } catch (BadLocationException e) {
                    log.warn(e);
                    togglePattern = "";
                }
                // Toggles can be nested (PostgreSQL) and we need to count only outer
                if (bracketDepth == 1 && togglePattern.equals(blockTogglePattern)) {
                    bracketDepth--;
                    blockTogglePattern = null;
                } else if (bracketDepth == 0 && blockTogglePattern == null) {
                    bracketDepth++;
                    blockTogglePattern = togglePattern;
                } else {
                    log.debug("Block toggle token inside another block. Can't process it");
                }
                hasBlocks = true;
            } else if (tokenType == SQLToken.T_BLOCK_BEGIN) {
                if (!hasBlockHeader) {
                    bracketDepth++;
                }
                hasBlocks = true;
                hasBlockHeader = false;
            } else if (bracketDepth > 0 && tokenType == SQLToken.T_BLOCK_END) {
                // This END doesn't mean block
                if (hasBlocks) {
                    bracketDepth--;
                }
                hasBlockHeader = false;
            } else if (isDelimiter && bracketDepth > 0) {
                // Delimiter in some brackets - ignore it
                continue;
            } else if (tokenType == SQLToken.T_SET_DELIMITER || tokenType == SQLToken.T_CONTROL) {
                isDelimiter = true;
                isControl = true;
            } else if (tokenType == SQLToken.T_COMMENT) {
                lastTokenLineFeeds = tokenLength < 2 ? 0 : countLineFeeds(document, tokenOffset + tokenLength - 2, 2);
            }
            boolean cursorInsideToken = currentPos >= tokenOffset && currentPos < tokenOffset + tokenLength;
            if (isControl && (scriptMode || cursorInsideToken) && !hasValuableTokens) {
                // Control query
                try {
                    String controlText = document.get(tokenOffset, tokenLength);
                    String commandId = null;
                    if (token instanceof SQLControlToken) {
                        commandId = ((SQLControlToken) token).getCommandId();
                    }
                    return new SQLControlCommand(getDataSource(), syntaxManager, controlText.trim(), commandId, tokenOffset, tokenLength, tokenType == SQLToken.T_SET_DELIMITER);
                } catch (BadLocationException e) {
                    // $NON-NLS-1$
                    log.warn("Can't extract control statement", e);
                    return null;
                }
            }
            if (hasValuableTokens && (token.isEOF() || (isDelimiter && tokenOffset >= currentPos) || tokenOffset > endPos)) {
                if (tokenOffset > endPos) {
                    tokenOffset = endPos;
                }
                if (tokenOffset >= document.getLength()) {
                    // Sometimes (e.g. when comment finishing script text)
                    // last token offset is beyond document range
                    tokenOffset = document.getLength();
                }
                assert (tokenOffset >= currentPos);
                try {
                    // remove leading spaces
                    while (statementStart < tokenOffset && Character.isWhitespace(document.getChar(statementStart))) {
                        statementStart++;
                    }
                    /*
                        while (statementStart < tokenOffset && Character.isWhitespace(document.getChar(tokenOffset - 1))) {
                            tokenOffset--;
                            tokenLength++;
                        }
*/
                    if (tokenOffset == statementStart) {
                        // Empty statement
                        if (token.isEOF()) {
                            return null;
                        }
                        statementStart = tokenOffset + tokenLength;
                        continue;
                    }
                    String queryText = document.get(statementStart, tokenOffset - statementStart);
                    queryText = SQLUtils.fixLineFeeds(queryText);
                    if (isDelimiter && (keepDelimiters || (hasBlocks ? dialect.isDelimiterAfterBlock() : dialect.isDelimiterAfterQuery()))) {
                        if (delimiterText != null && delimiterText.equals(SQLConstants.DEFAULT_STATEMENT_DELIMITER)) {
                            // Add delimiter in the end of query. Do this only for semicolon delimiters
                            // Quite dirty workaround needed for SQL server
                            queryText += delimiterText;
                        }
                    }
                    int queryEndPos = tokenOffset;
                    if (tokenType == SQLToken.T_DELIMITER) {
                        queryEndPos += tokenLength;
                    }
                    // make script line
                    return new SQLQuery(getDataSource(), queryText, statementStart, queryEndPos - statementStart);
                } catch (BadLocationException ex) {
                    // $NON-NLS-1$
                    log.warn("Can't extract query", ex);
                    return null;
                }
            }
            if (isDelimiter) {
                statementStart = tokenOffset + tokenLength;
            }
            if (token.isEOF()) {
                return null;
            }
            if (!hasValuableTokens && !token.isWhitespace() && !isControl) {
                if (tokenType == SQLToken.T_COMMENT) {
                    hasValuableTokens = dialect.supportsCommentQuery();
                } else {
                    hasValuableTokens = true;
                }
            }
        } finally {
            if (!token.isWhitespace() && !token.isEOF()) {
                prevNotEmptyTokenType = tokenType;
            }
        }
    }
}

Example 47 with IToken

use of org.eclipse.jface.text.rules.IToken in project dbeaver by dbeaver.

the class HTMLSQLConverter method convertText.

@NotNull
@Override
public String convertText(@NotNull SQLDialect dialect, @NotNull SQLSyntaxManager syntaxManager, @NotNull SQLRuleManager ruleManager, @NotNull IDocument document, int startPos, int length, @NotNull Map<String, Object> options) {
    StringBuilder result = new StringBuilder();
    ruleManager.setRange(document, startPos, length);
    try {
        result.append("<pre>");
        for (; ; ) {
            IToken token = ruleManager.nextToken();
            if (token.isEOF()) {
                break;
            }
            int tokenOffset = ruleManager.getTokenOffset();
            final int tokenLength = ruleManager.getTokenLength();
            boolean hasSpan = false;
            Object data = token.getData();
            if (data instanceof TextAttribute) {
                result.append("<span style='");
                TextAttribute ta = (TextAttribute) data;
                if (ta.getBackground() != null) {
                    result.append("background-color:").append(toHex(ta.getBackground())).append(";");
                }
                if (ta.getForeground() != null) {
                    result.append("color:").append(toHex(ta.getForeground())).append(";");
                }
                if ((ta.getStyle() & SWT.BOLD) == SWT.BOLD) {
                    result.append("font-weight:bold;");
                }
                if ((ta.getStyle() & SWT.ITALIC) == SWT.ITALIC) {
                    result.append("font-style: italic;");
                }
                // ta.getStyle()
                result.append("'>");
                hasSpan = true;
            }
            result.append(document.get(tokenOffset, tokenLength));
            if (hasSpan) {
                result.append("</span>");
            }
        }
        result.append("</pre>");
    } catch (BadLocationException e) {
        log.error("Error converting SQL to HTML", e);
    }
    return result.toString().trim();
}

Example 48 with IToken

use of org.eclipse.jface.text.rules.IToken in project dbeaver by dbeaver.

the class UnformattedSQLConverter method convertText.

@NotNull
@Override
public String convertText(@NotNull SQLDialect dialect, @NotNull SQLSyntaxManager syntaxManager, @NotNull SQLRuleManager ruleManager, @NotNull IDocument document, int startPos, int length, @NotNull Map<String, Object> options) {
    StringBuilder result = new StringBuilder();
    ruleManager.setRange(document, startPos, length);
    String[] singleLineComments = dialect.getSingleLineComments();
    Pair<String, String> multiLineComments = dialect.getMultiLineComments();
    boolean lastWhitespace = false;
    try {
        for (; ; ) {
            IToken token = ruleManager.nextToken();
            if (token.isEOF()) {
                break;
            }
            int tokenOffset = ruleManager.getTokenOffset();
            final int tokenLength = ruleManager.getTokenLength();
            if (token.isWhitespace()) {
                if (!lastWhitespace) {
                    result.append(' ');
                }
                lastWhitespace = true;
            } else if (token instanceof SQLCommentToken) {
                String comment = document.get(tokenOffset, tokenLength);
                for (String slc : singleLineComments) {
                    if (comment.startsWith(slc)) {
                        if (multiLineComments != null) {
                            comment = multiLineComments.getFirst() + comment.substring(slc.length()) + multiLineComments.getSecond();
                        }
                        break;
                    }
                }
                comment = TextUtils.compactWhiteSpaces(comment);
                result.append(comment);
            } else {
                lastWhitespace = false;
                result.append(document.get(tokenOffset, tokenLength));
            }
        }
    } catch (BadLocationException e) {
        log.error("Error unformatting SQL", e);
    }
    return result.toString().trim();
}

Example 49 with IToken

use of org.eclipse.jface.text.rules.IToken in project eclipse.platform.text by eclipse.

the class FastPartitionerTest method testBug409538_2.

@Test
public void testBug409538_2() throws Exception {
    fPartitioner.disconnect();
    IPartitionTokenScanner scanner = new RuleBasedPartitionScanner() {

        {
            IToken comment = new Token(COMMENT);
            IPredicateRule[] rules = new IPredicateRule[] { new MultiLineRule("<!--", "-->", comment, (char) 0, true) };
            setPredicateRules(rules);
        }
    };
    fPartitioner = createPartitioner(scanner);
    fDoc.setDocumentPartitioner(fPartitioner);
    fPartitioner.connect(fDoc);
    fDoc.set("<!-- blah");
    assertEqualPartition(0, 9, COMMENT);
}

Example 50 with IToken

use of org.eclipse.jface.text.rules.IToken in project eclipse.platform.text by eclipse.

the class FastPartitionerTest method testPR130900.

@Test
public void testPR130900() throws Exception {
    fPartitioner.disconnect();
    IPartitionTokenScanner scanner = new RuleBasedPartitionScanner() {

        {
            IToken comment = new Token(COMMENT);
            IPredicateRule[] rules = new IPredicateRule[] { new SingleLineRule("#", null, comment, (char) 0, true, false) };
            setPredicateRules(rules);
        }
    };
    fPartitioner = createPartitioner(scanner);
    fDoc.setDocumentPartitioner(fPartitioner);
    fPartitioner.connect(fDoc);
    fDoc.set("#");
    int[] offsets = new int[] { 0, 1 };
    assertComputePartitioning_InterleavingPartitions(offsets);
}