Examples with TokenSet com.intellij.psi.tree.TokenSet used on opensource projects

Example 6 with TokenSet

use of com.intellij.psi.tree.TokenSet in project intellij-community by JetBrains.

the class StructuralSearchProfileBase method isStringLiteral.

protected boolean isStringLiteral(PsiElement element) {
    if (element == null)
        return false;
    final ASTNode astNode = element.getNode();
    if (astNode == null) {
        return false;
    }
    final IElementType elementType = astNode.getElementType();
    final ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(element.getLanguage());
    if (parserDefinition != null) {
        final TokenSet literals = parserDefinition.getStringLiteralElements();
        return literals.contains(elementType);
    }
    return false;
}

Example 7 with TokenSet

use of com.intellij.psi.tree.TokenSet in project intellij-community by JetBrains.

the class FindManagerImpl method findInCommentsAndLiterals.

@NotNull
private FindResult findInCommentsAndLiterals(@NotNull CharSequence text, char[] textArray, int offset, @NotNull FindModel model, @NotNull final VirtualFile file) {
    synchronized (model) {
        FileType ftype = file.getFileType();
        Language lang = null;
        if (ftype instanceof LanguageFileType) {
            lang = ((LanguageFileType) ftype).getLanguage();
        }
        CommentsLiteralsSearchData data = model.getUserData(ourCommentsLiteralsSearchDataKey);
        if (data == null || !Comparing.equal(data.lastFile, file) || !data.model.equals(model)) {
            SyntaxHighlighter highlighter = getHighlighter(file, lang);
            if (highlighter == null) {
                // no syntax highlighter -> no search
                return NOT_FOUND_RESULT;
            }
            TokenSet tokensOfInterest = TokenSet.EMPTY;
            Set<Language> relevantLanguages;
            if (lang != null) {
                final Language finalLang = lang;
                relevantLanguages = ApplicationManager.getApplication().runReadAction(new Computable<Set<Language>>() {

                    @Override
                    public Set<Language> compute() {
                        THashSet<Language> result = new THashSet<>();
                        FileViewProvider viewProvider = PsiManager.getInstance(myProject).findViewProvider(file);
                        if (viewProvider != null) {
                            result.addAll(viewProvider.getLanguages());
                        }
                        if (result.isEmpty()) {
                            result.add(finalLang);
                        }
                        return result;
                    }
                });
                for (Language relevantLanguage : relevantLanguages) {
                    tokensOfInterest = addTokenTypesForLanguage(model, relevantLanguage, tokensOfInterest);
                }
                if (model.isInStringLiteralsOnly()) {
                    // TODO: xml does not have string literals defined so we add XmlAttributeValue element type as convenience
                    final Lexer xmlLexer = getHighlighter(null, Language.findLanguageByID("XML")).getHighlightingLexer();
                    final String marker = "xxx";
                    xmlLexer.start("<a href=\"" + marker + "\" />");
                    while (!marker.equals(xmlLexer.getTokenText())) {
                        xmlLexer.advance();
                        if (xmlLexer.getTokenType() == null)
                            break;
                    }
                    IElementType convenienceXmlAttrType = xmlLexer.getTokenType();
                    if (convenienceXmlAttrType != null) {
                        tokensOfInterest = TokenSet.orSet(tokensOfInterest, TokenSet.create(convenienceXmlAttrType));
                    }
                }
            } else {
                relevantLanguages = ContainerUtil.newHashSet();
                if (ftype instanceof AbstractFileType) {
                    if (model.isInCommentsOnly()) {
                        tokensOfInterest = TokenSet.create(CustomHighlighterTokenType.LINE_COMMENT, CustomHighlighterTokenType.MULTI_LINE_COMMENT);
                    }
                    if (model.isInStringLiteralsOnly()) {
                        tokensOfInterest = TokenSet.orSet(tokensOfInterest, TokenSet.create(CustomHighlighterTokenType.STRING, CustomHighlighterTokenType.SINGLE_QUOTED_STRING));
                    }
                }
            }
            Matcher matcher = model.isRegularExpressions() ? compileRegExp(model, "") : null;
            StringSearcher searcher = matcher != null ? null : new StringSearcher(model.getStringToFind(), model.isCaseSensitive(), true);
            SyntaxHighlighterOverEditorHighlighter highlighterAdapter = new SyntaxHighlighterOverEditorHighlighter(highlighter, file, myProject);
            data = new CommentsLiteralsSearchData(file, relevantLanguages, highlighterAdapter, tokensOfInterest, searcher, matcher, model.clone());
            data.highlighter.restart(text);
            model.putUserData(ourCommentsLiteralsSearchDataKey, data);
        }
        int initialStartOffset = model.isForward() && data.startOffset < offset ? data.startOffset : 0;
        data.highlighter.resetPosition(initialStartOffset);
        final Lexer lexer = data.highlighter.getHighlightingLexer();
        IElementType tokenType;
        TokenSet tokens = data.tokensOfInterest;
        int lastGoodOffset = 0;
        boolean scanningForward = model.isForward();
        FindResultImpl prevFindResult = NOT_FOUND_RESULT;
        while ((tokenType = lexer.getTokenType()) != null) {
            if (lexer.getState() == 0)
                lastGoodOffset = lexer.getTokenStart();
            final TextAttributesKey[] keys = data.highlighter.getTokenHighlights(tokenType);
            if (tokens.contains(tokenType) || (model.isInStringLiteralsOnly() && ChunkExtractor.isHighlightedAsString(keys)) || (model.isInCommentsOnly() && ChunkExtractor.isHighlightedAsComment(keys))) {
                int start = lexer.getTokenStart();
                int end = lexer.getTokenEnd();
                if (model.isInStringLiteralsOnly()) {
                    // skip literal quotes itself from matching
                    char c = text.charAt(start);
                    if (c == '"' || c == '\'') {
                        while (start < end && c == text.charAt(start)) {
                            ++start;
                            if (c == text.charAt(end - 1) && start < end)
                                --end;
                        }
                    }
                }
                while (true) {
                    FindResultImpl findResult = null;
                    if (data.searcher != null) {
                        int matchStart = data.searcher.scan(text, textArray, start, end);
                        if (matchStart != -1 && matchStart >= start) {
                            final int matchEnd = matchStart + model.getStringToFind().length();
                            if (matchStart >= offset || !scanningForward)
                                findResult = new FindResultImpl(matchStart, matchEnd);
                            else {
                                start = matchEnd;
                                continue;
                            }
                        }
                    } else if (start <= end) {
                        data.matcher.reset(StringPattern.newBombedCharSequence(text.subSequence(start, end)));
                        if (data.matcher.find()) {
                            final int matchEnd = start + data.matcher.end();
                            int matchStart = start + data.matcher.start();
                            if (matchStart >= offset || !scanningForward) {
                                findResult = new FindResultImpl(matchStart, matchEnd);
                            } else {
                                int diff = 0;
                                if (start == end) {
                                    diff = scanningForward ? 1 : -1;
                                }
                                start = matchEnd + diff;
                                continue;
                            }
                        }
                    }
                    if (findResult != null) {
                        if (scanningForward) {
                            data.startOffset = lastGoodOffset;
                            return findResult;
                        } else {
                            if (findResult.getEndOffset() >= offset)
                                return prevFindResult;
                            prevFindResult = findResult;
                            start = findResult.getEndOffset();
                            continue;
                        }
                    }
                    break;
                }
            } else {
                Language tokenLang = tokenType.getLanguage();
                if (tokenLang != lang && tokenLang != Language.ANY && !data.relevantLanguages.contains(tokenLang)) {
                    tokens = addTokenTypesForLanguage(model, tokenLang, tokens);
                    data.tokensOfInterest = tokens;
                    data.relevantLanguages.add(tokenLang);
                }
            }
            lexer.advance();
        }
        return prevFindResult;
    }
}

Example 8 with TokenSet

use of com.intellij.psi.tree.TokenSet in project intellij-community by JetBrains.

the class SpacingBuilder method aroundInside.

public RuleBuilder aroundInside(IElementType token, TokenSet parent) {
    final TokenSet tokenSet = TokenSet.create(token);
    RuleCondition before = new RuleCondition(parent, null, tokenSet);
    RuleCondition after = new RuleCondition(parent, tokenSet, null);
    return new RuleBuilder(before, after);
}

Example 9 with TokenSet

use of com.intellij.psi.tree.TokenSet in project intellij-community by JetBrains.

the class TypedHandler method isInsideStringLiteral.

private static boolean isInsideStringLiteral(@NotNull Editor editor, @NotNull PsiFile file) {
    int offset = editor.getCaretModel().getOffset();
    PsiElement element = file.findElementAt(offset);
    if (element == null)
        return false;
    final ParserDefinition definition = LanguageParserDefinitions.INSTANCE.forLanguage(element.getLanguage());
    if (definition != null) {
        final TokenSet stringLiteralElements = definition.getStringLiteralElements();
        final ASTNode node = element.getNode();
        if (node == null)
            return false;
        final IElementType elementType = node.getElementType();
        if (stringLiteralElements.contains(elementType)) {
            return true;
        }
        PsiElement parent = element.getParent();
        if (parent != null) {
            ASTNode parentNode = parent.getNode();
            if (parentNode != null && stringLiteralElements.contains(parentNode.getElementType())) {
                return true;
            }
        }
    }
    return false;
}

Example 10 with TokenSet

use of com.intellij.psi.tree.TokenSet in project intellij-community by JetBrains.

the class JspIndexPatternBuilder method getCommentTokenSet.

@Override
public TokenSet getCommentTokenSet(@NotNull final PsiFile file) {
    final JspFile jspFile = JspPsiUtil.getJspFile(file);
    TokenSet commentTokens = TokenSet.orSet(JavaIndexPatternBuilder.XML_COMMENT_BIT_SET, StdTokenSets.COMMENT_BIT_SET);
    final ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(jspFile.getViewProvider().getTemplateDataLanguage());
    if (parserDefinition != null) {
        commentTokens = TokenSet.orSet(commentTokens, parserDefinition.getCommentTokens());
    }
    return commentTokens;
}