Examples with TokenSet com.intellij.psi.tree.TokenSet used on opensource projects

Example 1 with TokenSet

use of com.intellij.psi.tree.TokenSet in project intellij-community by JetBrains.

the class PyConditionalStatementPartFixer method doApply.

@Override
public void doApply(@NotNull Editor editor, @NotNull PySmartEnterProcessor processor, @NotNull PyConditionalStatementPart statementPart) throws IncorrectOperationException {
    final PyExpression condition = statementPart.getCondition();
    final Document document = editor.getDocument();
    final PsiElement colon = PyPsiUtils.getFirstChildOfType(statementPart, PyTokenTypes.COLON);
    if (colon == null) {
        if (condition != null) {
            final PsiElement firstNonComment = PyPsiUtils.getNextNonCommentSibling(condition.getNextSibling(), false);
            if (firstNonComment != null && !":".equals(firstNonComment.getNode().getText())) {
                document.insertString(firstNonComment.getTextRange().getEndOffset(), ":");
            }
        } else {
            final TokenSet keywords = TokenSet.create(PyTokenTypes.IF_KEYWORD, PyTokenTypes.ELIF_KEYWORD, PyTokenTypes.WHILE_KEYWORD);
            final PsiElement keywordToken = PyPsiUtils.getChildByFilter(statementPart, keywords, 0);
            final int offset = sure(keywordToken).getTextRange().getEndOffset();
            document.insertString(offset, " :");
            processor.registerUnresolvedError(offset + 1);
        }
    } else if (condition == null) {
        processor.registerUnresolvedError(colon.getTextRange().getStartOffset());
    }
}

Example 2 with TokenSet

use of com.intellij.psi.tree.TokenSet in project kotlin by JetBrains.

the class KotlinParsing method parseFunction.

/*
     * function
     *   : modifiers "fun" typeParameters?
     *       (type ".")?
     *       SimpleName
     *       typeParameters? functionParameters (":" type)?
     *       typeConstraints
     *       functionBody?
     *   ;
     */
@Contract("false -> !null")
IElementType parseFunction(boolean failIfIdentifierExists) {
    assert _at(FUN_KEYWORD);
    // FUN_KEYWORD
    advance();
    // Recovery for the case of class A { fun| }
    if (at(RBRACE)) {
        error("Function body expected");
        return FUN;
    }
    boolean typeParameterListOccurred = false;
    if (at(LT)) {
        parseTypeParameterList(TokenSet.create(LBRACKET, LBRACE, RBRACE, LPAR));
        typeParameterListOccurred = true;
    }
    myBuilder.disableJoiningComplexTokens();
    TokenSet functionNameFollow = TokenSet.create(LT, LPAR, RPAR, COLON, EQ);
    boolean receiverFound = parseReceiverType("function", functionNameFollow);
    if (at(IDENTIFIER) && failIfIdentifierExists) {
        myBuilder.restoreJoiningComplexTokensState();
        return null;
    }
    // function as expression has no name
    parseFunctionOrPropertyName(receiverFound, "function", functionNameFollow, /*nameRequired = */
    false);
    myBuilder.restoreJoiningComplexTokensState();
    TokenSet valueParametersFollow = TokenSet.create(EQ, LBRACE, RBRACE, SEMICOLON, RPAR);
    if (at(LT)) {
        PsiBuilder.Marker error = mark();
        parseTypeParameterList(TokenSet.orSet(TokenSet.create(LPAR), valueParametersFollow));
        if (typeParameterListOccurred) {
            int offset = myBuilder.getCurrentOffset();
            error.rollbackTo();
            error = mark();
            advance(offset - myBuilder.getCurrentOffset());
            error.error("Only one type parameter list is allowed for a function");
        } else {
            error.drop();
        }
        typeParameterListOccurred = true;
    }
    if (at(LPAR)) {
        parseValueParameterList(false, /* typeRequired  = */
        false, valueParametersFollow);
    } else {
        error("Expecting '('");
    }
    if (at(COLON)) {
        // COLON
        advance();
        parseTypeRef();
    }
    parseTypeConstraintsGuarded(typeParameterListOccurred);
    if (at(SEMICOLON)) {
        // SEMICOLON
        advance();
    } else if (at(EQ) || at(LBRACE)) {
        parseFunctionBody();
    }
    return FUN;
}

Example 3 with TokenSet

use of com.intellij.psi.tree.TokenSet in project kotlin by JetBrains.

the class KotlinParsing method parseProperty.

public IElementType parseProperty(PropertyParsingMode mode) {
    assert (at(VAL_KEYWORD) || at(VAR_KEYWORD));
    advance();
    boolean typeParametersDeclared = at(LT) && parseTypeParameterList(TokenSet.create(IDENTIFIER, EQ, COLON, SEMICOLON));
    TokenSet propertyNameFollow = TokenSet.create(COLON, EQ, LBRACE, RBRACE, SEMICOLON, VAL_KEYWORD, VAR_KEYWORD, FUN_KEYWORD, CLASS_KEYWORD);
    myBuilder.disableJoiningComplexTokens();
    PsiBuilder.Marker receiver = mark();
    boolean receiverTypeDeclared = parseReceiverType("property", propertyNameFollow);
    boolean multiDeclaration = at(LPAR);
    errorIf(receiver, multiDeclaration && receiverTypeDeclared, "Receiver type is not allowed on a destructuring declaration");
    boolean isNameOnTheNextLine = eol();
    PsiBuilder.Marker beforeName = mark();
    if (multiDeclaration) {
        PsiBuilder.Marker multiDecl = mark();
        parseMultiDeclarationName(propertyNameFollow);
        errorIf(multiDecl, !mode.desctructuringAllowed, "Destructuring declarations are only allowed for local variables/values");
    } else {
        parseFunctionOrPropertyName(receiverTypeDeclared, "property", propertyNameFollow, /*nameRequired = */
        true);
    }
    myBuilder.restoreJoiningComplexTokensState();
    boolean noTypeReference = true;
    if (at(COLON)) {
        noTypeReference = false;
        PsiBuilder.Marker type = mark();
        // COLON
        advance();
        parseTypeRef();
        errorIf(type, multiDeclaration, "Type annotations are not allowed on destructuring declarations");
    }
    parseTypeConstraintsGuarded(typeParametersDeclared);
    if (!parsePropertyDelegateOrAssignment() && isNameOnTheNextLine && noTypeReference && !receiverTypeDeclared) {
        // Do not parse property identifier on the next line if declaration is invalid
        // In most cases this identifier relates to next statement/declaration
        beforeName.rollbackTo();
        error("Expecting property name or receiver type");
        return PROPERTY;
    }
    beforeName.drop();
    if (mode.accessorsAllowed) {
        // It's only needed for non-local properties, because in local ones:
        // "val a = 1; b" must not be an infix call of b on "val ...;"
        myBuilder.enableNewlines();
        boolean hasNewLineWithSemicolon = consumeIf(SEMICOLON) && myBuilder.newlineBeforeCurrentToken();
        myBuilder.restoreNewlinesState();
        if (!hasNewLineWithSemicolon) {
            AccessorKind accessorKind = parsePropertyGetterOrSetter(null);
            if (accessorKind != null) {
                parsePropertyGetterOrSetter(accessorKind);
            }
            if (!atSet(EOL_OR_SEMICOLON, RBRACE)) {
                if (getLastToken() != SEMICOLON) {
                    errorUntil("Property getter or setter expected", TokenSet.orSet(DECLARATION_FIRST, TokenSet.create(EOL_OR_SEMICOLON, LBRACE, RBRACE)));
                }
            } else {
                consumeIf(SEMICOLON);
            }
        }
    }
    return multiDeclaration ? DESTRUCTURING_DECLARATION : PROPERTY;
}

Example 4 with TokenSet

use of com.intellij.psi.tree.TokenSet in project WebStormRequireJsPlugin by Fedott.

the class RequirejsProjectComponent method getJSPropertyLiteralValue.

@Nullable
private static String getJSPropertyLiteralValue(TreeElement jsProperty) {
    TokenSet availablePropertyValueTokenSet = TokenSet.create(JSElementTypes.LITERAL_EXPRESSION, JSElementTypes.BINARY_EXPRESSION, JSElementTypes.CONDITIONAL_EXPRESSION);
    TreeElement jsPropertyValue = (TreeElement) jsProperty.findChildByType(availablePropertyValueTokenSet);
    if (null == jsPropertyValue) {
        return null;
    }
    if (jsPropertyValue.getElementType() != JSElementTypes.LITERAL_EXPRESSION) {
        jsPropertyValue = (TreeElement) jsPropertyValue.findChildByType(JSElementTypes.LITERAL_EXPRESSION);
        if (null == jsPropertyValue) {
            return null;
        }
    }
    return dequote(jsPropertyValue.getText());
}

Example 5 with TokenSet

use of com.intellij.psi.tree.TokenSet in project intellij-community by JetBrains.

the class FindManagerImpl method findInCommentsAndLiterals.

@NotNull
private FindResult findInCommentsAndLiterals(@NotNull CharSequence text, char[] textArray, int offset, @NotNull FindModel model, @NotNull final VirtualFile file) {
    synchronized (model) {
        FileType ftype = file.getFileType();
        Language lang = null;
        if (ftype instanceof LanguageFileType) {
            lang = ((LanguageFileType) ftype).getLanguage();
        }
        CommentsLiteralsSearchData data = model.getUserData(ourCommentsLiteralsSearchDataKey);
        if (data == null || !Comparing.equal(data.lastFile, file) || !data.model.equals(model)) {
            SyntaxHighlighter highlighter = getHighlighter(file, lang);
            if (highlighter == null) {
                // no syntax highlighter -> no search
                return NOT_FOUND_RESULT;
            }
            TokenSet tokensOfInterest = TokenSet.EMPTY;
            Set<Language> relevantLanguages;
            if (lang != null) {
                final Language finalLang = lang;
                relevantLanguages = ApplicationManager.getApplication().runReadAction(new Computable<Set<Language>>() {

                    @Override
                    public Set<Language> compute() {
                        THashSet<Language> result = new THashSet<>();
                        FileViewProvider viewProvider = PsiManager.getInstance(myProject).findViewProvider(file);
                        if (viewProvider != null) {
                            result.addAll(viewProvider.getLanguages());
                        }
                        if (result.isEmpty()) {
                            result.add(finalLang);
                        }
                        return result;
                    }
                });
                for (Language relevantLanguage : relevantLanguages) {
                    tokensOfInterest = addTokenTypesForLanguage(model, relevantLanguage, tokensOfInterest);
                }
                if (model.isInStringLiteralsOnly()) {
                    // TODO: xml does not have string literals defined so we add XmlAttributeValue element type as convenience
                    final Lexer xmlLexer = getHighlighter(null, Language.findLanguageByID("XML")).getHighlightingLexer();
                    final String marker = "xxx";
                    xmlLexer.start("<a href=\"" + marker + "\" />");
                    while (!marker.equals(xmlLexer.getTokenText())) {
                        xmlLexer.advance();
                        if (xmlLexer.getTokenType() == null)
                            break;
                    }
                    IElementType convenienceXmlAttrType = xmlLexer.getTokenType();
                    if (convenienceXmlAttrType != null) {
                        tokensOfInterest = TokenSet.orSet(tokensOfInterest, TokenSet.create(convenienceXmlAttrType));
                    }
                }
            } else {
                relevantLanguages = ContainerUtil.newHashSet();
                if (ftype instanceof AbstractFileType) {
                    if (model.isInCommentsOnly()) {
                        tokensOfInterest = TokenSet.create(CustomHighlighterTokenType.LINE_COMMENT, CustomHighlighterTokenType.MULTI_LINE_COMMENT);
                    }
                    if (model.isInStringLiteralsOnly()) {
                        tokensOfInterest = TokenSet.orSet(tokensOfInterest, TokenSet.create(CustomHighlighterTokenType.STRING, CustomHighlighterTokenType.SINGLE_QUOTED_STRING));
                    }
                }
            }
            Matcher matcher = model.isRegularExpressions() ? compileRegExp(model, "") : null;
            StringSearcher searcher = matcher != null ? null : new StringSearcher(model.getStringToFind(), model.isCaseSensitive(), true);
            SyntaxHighlighterOverEditorHighlighter highlighterAdapter = new SyntaxHighlighterOverEditorHighlighter(highlighter, file, myProject);
            data = new CommentsLiteralsSearchData(file, relevantLanguages, highlighterAdapter, tokensOfInterest, searcher, matcher, model.clone());
            data.highlighter.restart(text);
            model.putUserData(ourCommentsLiteralsSearchDataKey, data);
        }
        int initialStartOffset = model.isForward() && data.startOffset < offset ? data.startOffset : 0;
        data.highlighter.resetPosition(initialStartOffset);
        final Lexer lexer = data.highlighter.getHighlightingLexer();
        IElementType tokenType;
        TokenSet tokens = data.tokensOfInterest;
        int lastGoodOffset = 0;
        boolean scanningForward = model.isForward();
        FindResultImpl prevFindResult = NOT_FOUND_RESULT;
        while ((tokenType = lexer.getTokenType()) != null) {
            if (lexer.getState() == 0)
                lastGoodOffset = lexer.getTokenStart();
            final TextAttributesKey[] keys = data.highlighter.getTokenHighlights(tokenType);
            if (tokens.contains(tokenType) || (model.isInStringLiteralsOnly() && ChunkExtractor.isHighlightedAsString(keys)) || (model.isInCommentsOnly() && ChunkExtractor.isHighlightedAsComment(keys))) {
                int start = lexer.getTokenStart();
                int end = lexer.getTokenEnd();
                if (model.isInStringLiteralsOnly()) {
                    // skip literal quotes itself from matching
                    char c = text.charAt(start);
                    if (c == '"' || c == '\'') {
                        while (start < end && c == text.charAt(start)) {
                            ++start;
                            if (c == text.charAt(end - 1) && start < end)
                                --end;
                        }
                    }
                }
                while (true) {
                    FindResultImpl findResult = null;
                    if (data.searcher != null) {
                        int matchStart = data.searcher.scan(text, textArray, start, end);
                        if (matchStart != -1 && matchStart >= start) {
                            final int matchEnd = matchStart + model.getStringToFind().length();
                            if (matchStart >= offset || !scanningForward)
                                findResult = new FindResultImpl(matchStart, matchEnd);
                            else {
                                start = matchEnd;
                                continue;
                            }
                        }
                    } else if (start <= end) {
                        data.matcher.reset(StringPattern.newBombedCharSequence(text.subSequence(start, end)));
                        if (data.matcher.find()) {
                            final int matchEnd = start + data.matcher.end();
                            int matchStart = start + data.matcher.start();
                            if (matchStart >= offset || !scanningForward) {
                                findResult = new FindResultImpl(matchStart, matchEnd);
                            } else {
                                int diff = 0;
                                if (start == end) {
                                    diff = scanningForward ? 1 : -1;
                                }
                                start = matchEnd + diff;
                                continue;
                            }
                        }
                    }
                    if (findResult != null) {
                        if (scanningForward) {
                            data.startOffset = lastGoodOffset;
                            return findResult;
                        } else {
                            if (findResult.getEndOffset() >= offset)
                                return prevFindResult;
                            prevFindResult = findResult;
                            start = findResult.getEndOffset();
                            continue;
                        }
                    }
                    break;
                }
            } else {
                Language tokenLang = tokenType.getLanguage();
                if (tokenLang != lang && tokenLang != Language.ANY && !data.relevantLanguages.contains(tokenLang)) {
                    tokens = addTokenTypesForLanguage(model, tokenLang, tokens);
                    data.tokensOfInterest = tokens;
                    data.relevantLanguages.add(tokenLang);
                }
            }
            lexer.advance();
        }
        return prevFindResult;
    }
}