Examples with IToken org.eclipse.jface.text.rules.IToken used on opensource projects

Example 21 with IToken

use of org.eclipse.jface.text.rules.IToken in project dbeaver by serge-rider.

the class SQLEditorBase method parseParameters.

protected List<SQLQueryParameter> parseParameters(IDocument document, SQLQuery query) {
    boolean execQuery = SQLUtils.isExecQuery(getSQLDialect(), query.getQuery());
    List<SQLQueryParameter> parameters = null;
    ruleManager.setRange(document, query.getOffset(), query.getLength());
    int blockDepth = 0;
    for (; ; ) {
        IToken token = ruleManager.nextToken();
        int tokenOffset = ruleManager.getTokenOffset();
        final int tokenLength = ruleManager.getTokenLength();
        if (token.isEOF() || tokenOffset > query.getOffset() + query.getLength()) {
            break;
        }
        // Handle only parameters which are not in SQL blocks
        if (token instanceof SQLBlockBeginToken) {
            blockDepth++;
        } else if (token instanceof SQLBlockEndToken) {
            blockDepth--;
        }
        if (token instanceof SQLParameterToken && tokenLength > 0 && blockDepth <= 0) {
            try {
                String paramName = document.get(tokenOffset, tokenLength);
                if (execQuery && paramName.equals("?")) {
                    // Skip ? parameters for stored procedures (they have special meaning? [DB2])
                    continue;
                }
                if (parameters == null) {
                    parameters = new ArrayList<>();
                }
                SQLQueryParameter parameter = new SQLQueryParameter(parameters.size(), paramName, tokenOffset - query.getOffset(), tokenLength);
                SQLQueryParameter previous = null;
                if (parameter.isNamed()) {
                    for (int i = parameters.size(); i > 0; i--) {
                        if (parameters.get(i - 1).getName().equals(paramName)) {
                            previous = parameters.get(i - 1);
                            break;
                        }
                    }
                }
                parameter.setPrevious(previous);
                parameters.add(parameter);
            } catch (BadLocationException e) {
                log.warn("Can't extract query parameter", e);
            }
        }
    }
    return parameters;
}

Example 22 with IToken

use of org.eclipse.jface.text.rules.IToken in project dbeaver by serge-rider.

the class SQLEditorBase method parseQuery.

protected SQLQuery parseQuery(IDocument document, int startPos, int endPos, int currentPos, boolean scriptMode) {
    if (endPos - startPos <= 0) {
        return null;
    }
    SQLDialect dialect = getSQLDialect();
    // Parse range
    boolean useBlankLines = !scriptMode && syntaxManager.isBlankLineDelimiter();
    ruleManager.setRange(document, startPos, endPos - startPos);
    int statementStart = startPos;
    int bracketDepth = 0;
    boolean hasBlocks = false;
    boolean hasValuableTokens = false;
    boolean hasBlockHeader = false;
    String blockTogglePattern = null;
    for (; ; ) {
        IToken token = ruleManager.nextToken();
        int tokenOffset = ruleManager.getTokenOffset();
        final int tokenLength = ruleManager.getTokenLength();
        boolean isDelimiter = token instanceof SQLDelimiterToken;
        String delimiterText = null;
        if (isDelimiter) {
            try {
                delimiterText = document.get(tokenOffset, tokenLength);
            } catch (BadLocationException e) {
                log.debug(e);
                delimiterText = "";
            }
        }
        if (!isDelimiter) {
            if (useBlankLines && token.isWhitespace() && tokenLength >= 2) {
                // Check for blank line delimiter
                try {
                    int lfCount = 0;
                    for (int i = tokenOffset; i < tokenOffset + tokenLength; i++) {
                        if (document.getChar(i) == '\n') {
                            lfCount++;
                        }
                    }
                    if (lfCount >= 2) {
                        isDelimiter = true;
                    }
                } catch (BadLocationException e) {
                    log.error(e);
                }
            }
        }
        if (tokenLength == 1) {
            try {
                char aChar = document.getChar(tokenOffset);
                if (aChar == '(' || aChar == '{' || aChar == '[') {
                    bracketDepth++;
                } else if (aChar == ')' || aChar == '}' || aChar == ']') {
                    bracketDepth--;
                }
            } catch (BadLocationException e) {
                log.warn(e);
            }
        }
        if (token instanceof SQLBlockHeaderToken) {
            bracketDepth++;
            hasBlocks = true;
            hasBlockHeader = true;
        } else if (token instanceof SQLBlockToggleToken) {
            String togglePattern;
            try {
                togglePattern = document.get(tokenOffset, tokenLength);
            } catch (BadLocationException e) {
                log.warn(e);
                togglePattern = "";
            }
            // Toggles can be nested (PostgreSQL) and we need to count only outer
            if (bracketDepth == 1 && togglePattern.equals(blockTogglePattern)) {
                bracketDepth--;
                blockTogglePattern = null;
            } else if (bracketDepth == 0 && blockTogglePattern == null) {
                bracketDepth++;
                blockTogglePattern = togglePattern;
            } else {
                log.debug("Block toggle token inside another block. Can't process it");
            }
            hasBlocks = true;
        } else if (token instanceof SQLBlockBeginToken) {
            if (!hasBlockHeader) {
                bracketDepth++;
            }
            hasBlocks = true;
        } else if (bracketDepth > 0 && token instanceof SQLBlockEndToken) {
            // This END doesn't mean block
            if (hasBlocks) {
                bracketDepth--;
            }
            hasBlockHeader = false;
        } else if (isDelimiter && bracketDepth > 0) {
            // Delimiter in some brackets - ignore it
            continue;
        } else if (token instanceof SQLSetDelimiterToken) {
            isDelimiter = true;
        }
        if (hasValuableTokens && (token.isEOF() || (isDelimiter && tokenOffset >= currentPos) || tokenOffset > endPos)) {
            if (tokenOffset > endPos) {
                tokenOffset = endPos;
            }
            if (tokenOffset >= document.getLength()) {
                // Sometimes (e.g. when comment finishing script text)
                // last token offset is beyond document range
                tokenOffset = document.getLength();
            }
            assert (tokenOffset >= currentPos);
            try {
                // remove leading spaces
                while (statementStart < tokenOffset && Character.isWhitespace(document.getChar(statementStart))) {
                    statementStart++;
                }
                // remove trailing spaces
                while (statementStart < tokenOffset && Character.isWhitespace(document.getChar(tokenOffset - 1))) {
                    tokenOffset--;
                }
                if (tokenOffset == statementStart) {
                    // Empty statement
                    if (token.isEOF()) {
                        return null;
                    }
                    statementStart = tokenOffset + tokenLength;
                    continue;
                }
                String queryText = document.get(statementStart, tokenOffset - statementStart);
                queryText = SQLUtils.fixLineFeeds(queryText);
                // FIXME: includes last delimiter in query (Oracle?)
                if (isDelimiter && hasBlocks && dialect.isDelimiterAfterBlock()) {
                    if (delimiterText != null) {
                        queryText += delimiterText;
                    }
                }
                // make script line
                return new SQLQuery(getDataSource(), queryText.trim(), statementStart, tokenOffset - statementStart);
            } catch (BadLocationException ex) {
                //$NON-NLS-1$
                log.warn("Can't extract query", ex);
                return null;
            }
        }
        if (isDelimiter) {
            statementStart = tokenOffset + tokenLength;
        }
        if (token.isEOF()) {
            return null;
        }
        if (!hasValuableTokens && !token.isWhitespace() && !(token instanceof SQLSetDelimiterToken)) {
            if (token instanceof SQLCommentToken) {
                hasValuableTokens = dialect.supportsCommentQuery();
            } else {
                hasValuableTokens = true;
            }
        }
    }
}

Example 23 with IToken

use of org.eclipse.jface.text.rules.IToken in project KaiZen-OpenAPI-Editor by RepreZen.

the class JsonScanner method init.

protected void init() {
    TextAttribute keyAttr = tokenAttribute(PreferenceConstants.COLOR_KEY, PreferenceConstants.BOLD_KEY, PreferenceConstants.ITALIC_KEY, PreferenceConstants.UNDERLINE_KEY);
    IToken keyToken = new YAMLToken(keyAttr, YAMLToken.KEY);
    TextAttribute scalarAttr = tokenAttribute(PreferenceConstants.COLOR_SCALAR, PreferenceConstants.BOLD_SCALAR, PreferenceConstants.ITALIC_SCALAR, PreferenceConstants.UNDERLINE_SCALAR);
    IToken scalarToken = new YAMLToken(scalarAttr, YAMLToken.SCALAR);
    TextAttribute commentAttr = tokenAttribute(PreferenceConstants.COLOR_COMMENT, PreferenceConstants.BOLD_COMMENT, PreferenceConstants.ITALIC_COMMENT, PreferenceConstants.UNDERLINE_COMMENT);
    IToken commentToken = new YAMLToken(commentAttr, YAMLToken.COMMENT);
    TextAttribute documentAttr = tokenAttribute(PreferenceConstants.COLOR_DOCUMENT, PreferenceConstants.BOLD_DOCUMENT, PreferenceConstants.ITALIC_DOCUMENT, PreferenceConstants.UNDERLINE_DOCUMENT);
    IToken documentStartToken = new YAMLToken(documentAttr, YAMLToken.DOCUMENT_START);
    IToken documentEndToken = new YAMLToken(documentAttr, YAMLToken.DOCUMENT_END);
    TextAttribute anchorAttr = tokenAttribute(PreferenceConstants.COLOR_ANCHOR, PreferenceConstants.BOLD_ANCHOR, PreferenceConstants.ITALIC_ANCHOR, PreferenceConstants.UNDERLINE_ANCHOR);
    IToken anchorToken = new YAMLToken(anchorAttr, YAMLToken.ANCHOR);
    TextAttribute aliasAttr = tokenAttribute(PreferenceConstants.COLOR_ALIAS, PreferenceConstants.BOLD_ALIAS, PreferenceConstants.ITALIC_ALIAS, PreferenceConstants.UNDERLINE_ALIAS);
    IToken aliasToken = new YAMLToken(aliasAttr, YAMLToken.ALIAS);
    IToken indicatorCharToken = new YAMLToken(new TextAttribute(null), YAMLToken.INDICATOR_CHARACTER);
    TextAttribute tagAttr = tokenAttribute(PreferenceConstants.COLOR_TAG_PROPERTY, PreferenceConstants.BOLD_TAG_PROPERTY, PreferenceConstants.ITALIC_TAG_PROPERTY, PreferenceConstants.UNDERLINE_TAG_PROPERTY);
    IToken tagPropToken = new YAMLToken(tagAttr, YAMLToken.TAG_PROPERTY);
    TextAttribute constantAttr = tokenAttribute(PreferenceConstants.COLOR_CONSTANT, PreferenceConstants.BOLD_CONSTANT, PreferenceConstants.ITALIC_CONSTANT, PreferenceConstants.UNDERLINE_CONSTANT);
    IToken predefinedValToken = new YAMLToken(constantAttr, YAMLToken.CONSTANT);
    IToken whitespaceToken = new YAMLToken(new TextAttribute(null), YAMLToken.WHITESPACE);
    IToken directiveToken = new YAMLToken(new TextAttribute(null), YAMLToken.DIRECTIVE);
    ArrayList<IRule> rules = new ArrayList<IRule>();
    rules.add(new KeyRule(keyToken));
    rules.add(new SingleQuotedKeyRule(keyToken));
    rules.add(new DoubleQuotedKeyRule(keyToken));
    rules.add(new MultiLineRule("\"", "\"", scalarToken, '\\'));
    rules.add(new MultiLineRule("'", "'", scalarToken));
    rules.add(new EndOfLineRule("#", commentToken));
    rules.add(new EndOfLineRule("%TAG", directiveToken));
    rules.add(new EndOfLineRule("%YAML", directiveToken));
    rules.add(new DocumentStartAndEndRule("---", documentStartToken));
    rules.add(new DocumentStartAndEndRule("...", documentEndToken));
    rules.add(new IndicatorCharacterRule(indicatorCharToken));
    rules.add(new WhitespaceRule(whitespaceToken));
    rules.add(new WordPatternRule(new AnchorWordDetector(), "&", "", anchorToken));
    rules.add(new WordPatternRule(new AnchorWordDetector(), "*", "", aliasToken));
    rules.add(new WordPatternRule(new TagWordDetector(), "!", "", tagPropToken));
    rules.add(new PredefinedValueRule(predefinedValToken));
    rules.add(new ScalarRule(scalarToken));
    IRule[] rulesArray = new IRule[rules.size()];
    rules.toArray(rulesArray);
    setRules(rulesArray);
    setDefaultReturnToken(scalarToken);
}

Example 24 with IToken

use of org.eclipse.jface.text.rules.IToken in project eclipse.platform.text by eclipse.

the class FastPartitionerTest method testBug409538_2.

@Test
public void testBug409538_2() throws Exception {
    fPartitioner.disconnect();
    IPartitionTokenScanner scanner = new RuleBasedPartitionScanner() {

        {
            IToken comment = new Token(COMMENT);
            IPredicateRule[] rules = new IPredicateRule[] { new MultiLineRule("<!--", "-->", comment, (char) 0, true) };
            setPredicateRules(rules);
        }
    };
    fPartitioner = createPartitioner(scanner);
    fDoc.setDocumentPartitioner(fPartitioner);
    fPartitioner.connect(fDoc);
    fDoc.set("<!-- blah");
    assertEqualPartition(0, 9, COMMENT);
}

Example 25 with IToken

use of org.eclipse.jface.text.rules.IToken in project eclipse.platform.text by eclipse.

the class FastPartitionerTest method testPR130900.

@Test
public void testPR130900() throws Exception {
    fPartitioner.disconnect();
    IPartitionTokenScanner scanner = new RuleBasedPartitionScanner() {

        {
            IToken comment = new Token(COMMENT);
            IPredicateRule[] rules = new IPredicateRule[] { new SingleLineRule("#", null, comment, (char) 0, true, false) };
            setPredicateRules(rules);
        }
    };
    fPartitioner = createPartitioner(scanner);
    fDoc.setDocumentPartitioner(fPartitioner);
    fPartitioner.connect(fDoc);
    fDoc.set("#");
    int[] offsets = new int[] { 0, 1 };
    assertComputePartitioning_InterleavingPartitions(offsets);
}