Examples with TokenSet com.intellij.psi.tree.TokenSet used on opensource projects

Example 16 with TokenSet

use of com.intellij.psi.tree.TokenSet in project intellij-community by JetBrains.

the class ClassElement method addInternal.

@Override
public TreeElement addInternal(TreeElement first, ASTNode last, ASTNode anchor, Boolean before) {
    PsiClass psiClass = (PsiClass) SourceTreeToPsiMap.treeElementToPsi(this);
    if (anchor == null) {
        if (first.getElementType() != JavaDocElementType.DOC_COMMENT) {
            if (before == null) {
                if (first == last) {
                    PsiElement firstPsi = SourceTreeToPsiMap.treeElementToPsi(first);
                    if (firstPsi instanceof PsiEnumConstant) {
                        anchor = findEnumConstantListDelimiterPlace();
                        before = anchor != findChildByRole(ChildRole.LBRACE);
                    } else {
                        PsiElement psiElement = firstPsi instanceof PsiMember ? JavaPsiImplementationHelper.getInstance(psiClass.getProject()).getDefaultMemberAnchor(psiClass, (PsiMember) firstPsi) : null;
                        anchor = psiElement != null ? SourceTreeToPsiMap.psiElementToTree(psiElement) : null;
                        before = Boolean.TRUE;
                    }
                } else {
                    anchor = findChildByRole(ChildRole.RBRACE);
                    before = Boolean.TRUE;
                }
            } else if (!before.booleanValue()) {
                anchor = findChildByRole(ChildRole.LBRACE);
            } else {
                anchor = findChildByRole(ChildRole.RBRACE);
            }
        }
    }
    if (isEnum()) {
        if (!ENUM_CONSTANT_LIST_ELEMENTS_BIT_SET.contains(first.getElementType())) {
            ASTNode semicolonPlace = findEnumConstantListDelimiterPlace();
            boolean commentsOrWhiteSpaces = true;
            for (ASTNode child = first; child != null; child = child.getTreeNext()) {
                if (!PsiImplUtil.isWhitespaceOrComment(child)) {
                    commentsOrWhiteSpaces = false;
                    break;
                }
            }
            if (!commentsOrWhiteSpaces && (semicolonPlace == null || semicolonPlace.getElementType() != SEMICOLON)) {
                final LeafElement semicolon = Factory.createSingleLeafElement(SEMICOLON, ";", 0, 1, SharedImplUtil.findCharTableByTree(this), getManager());
                addInternal(semicolon, semicolon, semicolonPlace, Boolean.FALSE);
                semicolonPlace = semicolon;
            }
            for (ASTNode run = anchor; run != null; run = run.getTreeNext()) {
                if (run == semicolonPlace) {
                    anchor = before.booleanValue() ? semicolonPlace.getTreeNext() : semicolonPlace;
                    if (anchor != null && PsiImplUtil.isWhitespaceOrComment(anchor)) {
                        anchor = PsiTreeUtil.skipSiblingsForward(anchor.getPsi(), PsiWhiteSpace.class, PsiComment.class).getNode();
                    }
                    break;
                }
            }
        }
    }
    ASTNode afterLast = last.getTreeNext();
    ASTNode next;
    for (ASTNode child = first; child != afterLast; child = next) {
        next = child.getTreeNext();
        if (child.getElementType() == JavaElementType.METHOD && ((PsiMethod) SourceTreeToPsiMap.treeElementToPsi(child)).isConstructor()) {
            ASTNode oldIdentifier = ((CompositeElement) child).findChildByRole(ChildRole.NAME);
            ASTNode newIdentifier = findChildByRole(ChildRole.NAME).copyElement();
            newIdentifier.putUserData(CharTable.CHAR_TABLE_KEY, SharedImplUtil.findCharTableByTree(this));
            child.replaceChild(oldIdentifier, newIdentifier);
        }
    }
    if (psiClass.isEnum()) {
        for (ASTNode child = first; child != afterLast; child = next) {
            next = child.getTreeNext();
            if ((child.getElementType() == JavaElementType.METHOD && ((PsiMethod) SourceTreeToPsiMap.treeElementToPsi(child)).isConstructor()) || child.getElementType() == JavaElementType.ENUM_CONSTANT) {
                CompositeElement modifierList = (CompositeElement) ((CompositeElement) child).findChildByRole(ChildRole.MODIFIER_LIST);
                while (true) {
                    ASTNode modifier = modifierList.findChildByType(MODIFIERS_TO_REMOVE_IN_ENUM_BIT_SET);
                    if (modifier == null)
                        break;
                    modifierList.deleteChildInternal(modifier);
                }
            }
        }
    } else if (psiClass.isInterface()) {
        final boolean level8OrHigher = PsiUtil.isLanguageLevel8OrHigher(psiClass);
        for (ASTNode child = first; child != afterLast; child = next) {
            next = child.getTreeNext();
            final IElementType childElementType = child.getElementType();
            if (childElementType == JavaElementType.METHOD || childElementType == JavaElementType.FIELD) {
                CompositeElement modifierList = (CompositeElement) ((CompositeElement) child).findChildByRole(ChildRole.MODIFIER_LIST);
                final TokenSet removeModifiersBitSet = level8OrHigher && childElementType == JavaElementType.METHOD ? MODIFIERS_TO_REMOVE_IN_INTERFACE_BIT_SET_18_METHOD : MODIFIERS_TO_REMOVE_IN_INTERFACE_BIT_SET;
                while (true) {
                    ASTNode modifier = modifierList.findChildByType(removeModifiersBitSet);
                    if (modifier == null)
                        break;
                    modifierList.deleteChildInternal(modifier);
                }
            }
        }
    }
    final TreeElement firstAdded = super.addInternal(first, last, anchor, before);
    if (firstAdded.getElementType() == ENUM_CONSTANT) {
        final CharTable treeCharTab = SharedImplUtil.findCharTableByTree(this);
        for (ASTNode child = ((ASTNode) first).getTreeNext(); child != null; child = child.getTreeNext()) {
            final IElementType elementType = child.getElementType();
            if (elementType == COMMA || elementType == SEMICOLON)
                break;
            if (elementType == ENUM_CONSTANT) {
                TreeElement comma = Factory.createSingleLeafElement(COMMA, ",", 0, 1, treeCharTab, getManager());
                super.addInternal(comma, comma, first, Boolean.FALSE);
                break;
            }
        }
        for (ASTNode child = ((ASTNode) first).getTreePrev(); child != null; child = child.getTreePrev()) {
            final IElementType elementType = child.getElementType();
            if (elementType == COMMA || elementType == SEMICOLON)
                break;
            if (elementType == ENUM_CONSTANT) {
                TreeElement comma = Factory.createSingleLeafElement(COMMA, ",", 0, 1, treeCharTab, getManager());
                super.addInternal(comma, comma, child, Boolean.FALSE);
                break;
            }
        }
    }
    return firstAdded;
}

Example 17 with TokenSet

use of com.intellij.psi.tree.TokenSet in project intellij-community by JetBrains.

the class IndexPatternSearcher method findCommentTokenRanges.

private static void findCommentTokenRanges(final PsiFile file, final CharSequence chars, final TextRange range, final TIntArrayList commentStarts, final TIntArrayList commentEnds) {
    if (file instanceof PsiPlainTextFile) {
        FileType fType = file.getFileType();
        if (fType instanceof CustomSyntaxTableFileType) {
            Lexer lexer = SyntaxHighlighterFactory.getSyntaxHighlighter(fType, file.getProject(), file.getVirtualFile()).getHighlightingLexer();
            findComments(lexer, chars, range, COMMENT_TOKENS, commentStarts, commentEnds, null);
        } else {
            commentStarts.add(0);
            commentEnds.add(file.getTextLength());
        }
    } else {
        final FileViewProvider viewProvider = file.getViewProvider();
        final Set<Language> relevantLanguages = viewProvider.getLanguages();
        for (Language lang : relevantLanguages) {
            final TIntArrayList commentStartsList = new TIntArrayList();
            final TIntArrayList commentEndsList = new TIntArrayList();
            final SyntaxHighlighter syntaxHighlighter = SyntaxHighlighterFactory.getSyntaxHighlighter(lang, file.getProject(), file.getVirtualFile());
            Lexer lexer = syntaxHighlighter.getHighlightingLexer();
            TokenSet commentTokens = null;
            IndexPatternBuilder builderForFile = null;
            for (IndexPatternBuilder builder : Extensions.getExtensions(IndexPatternBuilder.EP_NAME)) {
                Lexer lexerFromBuilder = builder.getIndexingLexer(file);
                if (lexerFromBuilder != null) {
                    lexer = lexerFromBuilder;
                    commentTokens = builder.getCommentTokenSet(file);
                    builderForFile = builder;
                }
            }
            if (builderForFile == null) {
                final ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(lang);
                if (parserDefinition != null) {
                    commentTokens = parserDefinition.getCommentTokens();
                }
            }
            if (commentTokens != null) {
                findComments(lexer, chars, range, commentTokens, commentStartsList, commentEndsList, builderForFile);
                mergeCommentLists(commentStarts, commentEnds, commentStartsList, commentEndsList);
            }
        }
    }
}

Example 18 with TokenSet

use of com.intellij.psi.tree.TokenSet in project intellij-community by JetBrains.

the class SpacingBuilder method withinPairInside.

public RuleBuilder withinPairInside(IElementType pairFirst, IElementType pairSecond, IElementType parent) {
    TokenSet parentSet = TokenSet.create(parent);
    RuleCondition before = new RuleCondition(parentSet, TokenSet.create(pairFirst), null);
    RuleCondition after = new RuleCondition(parentSet, null, TokenSet.create(pairSecond));
    return new RuleBuilder(before, after);
}

Example 19 with TokenSet

use of com.intellij.psi.tree.TokenSet in project intellij-community by JetBrains.

the class SpacingBuilder method aroundInside.

public RuleBuilder aroundInside(IElementType token, IElementType parent) {
    final TokenSet tokenSet = TokenSet.create(token);
    RuleCondition before = new RuleCondition(TokenSet.create(parent), null, tokenSet);
    RuleCondition after = new RuleCondition(TokenSet.create(parent), tokenSet, null);
    return new RuleBuilder(before, after);
}

Example 20 with TokenSet

use of com.intellij.psi.tree.TokenSet in project intellij-community by JetBrains.

the class MinifiedFilesUtil method isMinified.

protected static boolean isMinified(@NotNull Lexer lexer, @NotNull ParserDefinition parserDefinition, @NotNull TokenSet noWSRequireBeforeTokenSet, @NotNull TokenSet noWSRequireAfterTokenSet, @NotNull TokenSet stringLiteralElements) {
    int offsetIgnoringComments = 0;
    int offsetIgnoringCommentsAndStrings = 0;
    int unneededWhitespaceCount = 0;
    String lastTokenText = null;
    IElementType lastTokenType = null;
    TokenSet whitespaceTokens = parserDefinition.getWhitespaceTokens();
    TokenSet commentTokens = parserDefinition.getCommentTokens();
    boolean lastWhiteSpaceWasHandled = false;
    for (IElementType tokenType = lexer.getTokenType(); tokenType != null; lexer.advance(), tokenType = lexer.getTokenType()) {
        if (commentTokens.contains(tokenType)) {
            lastTokenType = tokenType;
            lastTokenText = lexer.getTokenText();
            continue;
        }
        int tokenLength = lexer.getTokenEnd() - lexer.getTokenStart();
        if (isNewLine(lexer, tokenLength) && commentTokens.contains(lastTokenType) && !noWSRequireAfterTokenSet.contains(lastTokenType)) {
            // do not count new line after line comment token since it's required and it's part of comment
            continue;
        }
        offsetIgnoringComments += tokenLength;
        if (!stringLiteralElements.contains(tokenType)) {
            offsetIgnoringCommentsAndStrings += tokenLength;
        }
        if (whitespaceTokens.contains(tokenType)) {
            lastWhiteSpaceWasHandled = false;
            if (!commentTokens.contains(lastTokenType) && tokenLength > 1) {
                lexer.advance();
                if (lexer.getTokenType() == null) {
                    // it was last token
                    break;
                } else {
                    return false;
                }
            }
            if (isNewLine(lexer, tokenLength) && StringUtil.equals(lastTokenText, "\n") || tokenLength > 0 && noWSRequireAfterTokenSet.contains(lastTokenType)) {
                unneededWhitespaceCount++;
                lastWhiteSpaceWasHandled = true;
            }
        } else {
            if (!lastWhiteSpaceWasHandled && whitespaceTokens.contains(lastTokenType) && StringUtil.isNotEmpty(lastTokenText) && noWSRequireBeforeTokenSet.contains(tokenType)) {
                unneededWhitespaceCount++;
            }
        }
        if (stringLiteralElements.contains(tokenType)) {
            lastTokenType = tokenType;
            lastTokenText = lexer.getTokenText();
            continue;
        }
        if (offsetIgnoringComments >= MAX_OFFSET) {
            break;
        }
        lastTokenType = tokenType;
        lastTokenText = lexer.getTokenText();
    }
    return offsetIgnoringComments >= MIN_SIZE && (unneededWhitespaceCount + 0.0d) / offsetIgnoringCommentsAndStrings < MAX_UNNEEDED_OFFSET_PERCENTAGE;
}