Examples with Lexer com.intellij.lexer.Lexer used on opensource projects

Example 31 with Lexer

use of com.intellij.lexer.Lexer in project intellij-community by JetBrains.

the class InnerClassesScannerTest method mayContainClassesInside.

private static boolean mayContainClassesInside(PsiElement psiElement, final CharSequence fileBuffer) {
    PsiFile psiFile = psiElement.getContainingFile();
    boolean mayHaveClassesInside = false;
    if (psiFile instanceof PsiJavaFileImpl) {
        Lexer originalLexer = JavaParserDefinition.createLexer(((PsiJavaFileImpl) psiFile).getLanguageLevel());
        FilterLexer lexer = new FilterLexer(originalLexer, new FilterLexer.SetFilter(StdTokenSets.WHITE_SPACE_OR_COMMENT_BIT_SET));
        final TextRange range = psiElement.getTextRange();
        lexer.start(fileBuffer, range.getStartOffset(), range.getEndOffset());
        boolean isInNewExpression = false;
        boolean isRightAfterNewExpression = false;
        int angleLevel = 0;
        int parenLevel = 0;
        do {
            IElementType tokenType = lexer.getTokenType();
            if (tokenType == null)
                break;
            if (tokenType == JavaTokenType.NEW_KEYWORD) {
                isInNewExpression = true;
            } else if (tokenType == JavaTokenType.LPARENTH) {
                if (isInNewExpression)
                    parenLevel++;
            } else if (tokenType == JavaTokenType.LT) {
                if (isInNewExpression)
                    angleLevel++;
            } else if (tokenType == JavaTokenType.GT) {
                if (isInNewExpression)
                    angleLevel--;
            } else if (tokenType == JavaTokenType.RPARENTH) {
                if (isInNewExpression) {
                    parenLevel--;
                    if (parenLevel == 0) {
                        isRightAfterNewExpression = true;
                    }
                }
            } else if (tokenType == JavaTokenType.LBRACE) {
                if (isInNewExpression || isRightAfterNewExpression) {
                    mayHaveClassesInside = true;
                }
            } else if (tokenType == JavaTokenType.LBRACKET) {
                if (parenLevel == 0 && angleLevel == 0)
                    isInNewExpression = false;
            } else if (tokenType == JavaTokenType.INTERFACE_KEYWORD || tokenType == JavaTokenType.CLASS_KEYWORD || tokenType == JavaTokenType.ENUM_KEYWORD) {
                mayHaveClassesInside = true;
            }
            if (isInNewExpression && isRightAfterNewExpression) {
                isInNewExpression = false;
            } else {
                isRightAfterNewExpression = false;
            }
            lexer.advance();
        } while (!mayHaveClassesInside);
    }
    return mayHaveClassesInside;
}

Example 32 with Lexer

use of com.intellij.lexer.Lexer in project intellij-community by JetBrains.

the class IndexPatternSearcher method findCommentTokenRanges.

private static void findCommentTokenRanges(final PsiFile file, final CharSequence chars, final TextRange range, final TIntArrayList commentStarts, final TIntArrayList commentEnds) {
    if (file instanceof PsiPlainTextFile) {
        FileType fType = file.getFileType();
        if (fType instanceof CustomSyntaxTableFileType) {
            Lexer lexer = SyntaxHighlighterFactory.getSyntaxHighlighter(fType, file.getProject(), file.getVirtualFile()).getHighlightingLexer();
            findComments(lexer, chars, range, COMMENT_TOKENS, commentStarts, commentEnds, null);
        } else {
            commentStarts.add(0);
            commentEnds.add(file.getTextLength());
        }
    } else {
        final FileViewProvider viewProvider = file.getViewProvider();
        final Set<Language> relevantLanguages = viewProvider.getLanguages();
        for (Language lang : relevantLanguages) {
            final TIntArrayList commentStartsList = new TIntArrayList();
            final TIntArrayList commentEndsList = new TIntArrayList();
            final SyntaxHighlighter syntaxHighlighter = SyntaxHighlighterFactory.getSyntaxHighlighter(lang, file.getProject(), file.getVirtualFile());
            Lexer lexer = syntaxHighlighter.getHighlightingLexer();
            TokenSet commentTokens = null;
            IndexPatternBuilder builderForFile = null;
            for (IndexPatternBuilder builder : Extensions.getExtensions(IndexPatternBuilder.EP_NAME)) {
                Lexer lexerFromBuilder = builder.getIndexingLexer(file);
                if (lexerFromBuilder != null) {
                    lexer = lexerFromBuilder;
                    commentTokens = builder.getCommentTokenSet(file);
                    builderForFile = builder;
                }
            }
            if (builderForFile == null) {
                final ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(lang);
                if (parserDefinition != null) {
                    commentTokens = parserDefinition.getCommentTokens();
                }
            }
            if (commentTokens != null) {
                findComments(lexer, chars, range, commentTokens, commentStartsList, commentEndsList, builderForFile);
                mergeCommentLists(commentStarts, commentEnds, commentStartsList, commentEndsList);
            }
        }
    }
}

Example 33 with Lexer

use of com.intellij.lexer.Lexer in project intellij-community by JetBrains.

the class JavaParserDefinition method spaceExistanceTypeBetweenTokens.

@Override
public SpaceRequirements spaceExistanceTypeBetweenTokens(ASTNode left, ASTNode right) {
    if (right.getElementType() == JavaDocTokenType.DOC_TAG_VALUE_SHARP_TOKEN || left.getElementType() == JavaDocTokenType.DOC_TAG_VALUE_SHARP_TOKEN) {
        return SpaceRequirements.MUST_NOT;
    }
    PsiFile containingFile = left.getTreeParent().getPsi().getContainingFile();
    LanguageLevel level = containingFile instanceof PsiJavaFile ? ((PsiJavaFile) containingFile).getLanguageLevel() : LanguageLevel.HIGHEST;
    Lexer lexer = createLexer(level);
    SpaceRequirements spaceRequirements = LanguageUtil.canStickTokensTogetherByLexer(left, right, lexer);
    if (left.getElementType() == JavaTokenType.END_OF_LINE_COMMENT) {
        return SpaceRequirements.MUST_LINE_BREAK;
    }
    if (left.getElementType() == JavaDocTokenType.DOC_COMMENT_DATA) {
        String text = left.getText();
        if (text.length() > 0 && Character.isWhitespace(text.charAt(text.length() - 1))) {
            return SpaceRequirements.MAY;
        }
    }
    if (right.getElementType() == JavaDocTokenType.DOC_COMMENT_DATA) {
        String text = right.getText();
        if (text.length() > 0 && Character.isWhitespace(text.charAt(0))) {
            return SpaceRequirements.MAY;
        }
    } else if (right.getElementType() == JavaDocTokenType.DOC_INLINE_TAG_END) {
        return SpaceRequirements.MAY;
    }
    return spaceRequirements;
}

Example 34 with Lexer

use of com.intellij.lexer.Lexer in project intellij-community by JetBrains.

the class JavaParserUtil method parseFragment.

@Nullable
public static ASTNode parseFragment(final ASTNode chameleon, final ParserWrapper wrapper, final boolean eatAll, final LanguageLevel level) {
    final PsiElement psi = (chameleon.getTreeParent() != null ? chameleon.getTreeParent().getPsi() : chameleon.getPsi());
    assert psi != null : chameleon;
    final Project project = psi.getProject();
    final PsiBuilderFactory factory = PsiBuilderFactory.getInstance();
    final Lexer lexer = chameleon.getElementType() == JavaDocElementType.DOC_COMMENT ? JavaParserDefinition.createDocLexer(level) : JavaParserDefinition.createLexer(level);
    final PsiBuilder builder = factory.createBuilder(project, chameleon, lexer, chameleon.getElementType().getLanguage(), chameleon.getChars());
    setLanguageLevel(builder, level);
    final PsiBuilder.Marker root = builder.mark();
    wrapper.parse(builder);
    if (!builder.eof()) {
        if (!eatAll)
            throw new AssertionError("Unexpected tokens");
        final PsiBuilder.Marker extras = builder.mark();
        while (!builder.eof()) builder.advanceLexer();
        extras.error(JavaErrorMessages.message("unexpected.tokens"));
    }
    root.done(chameleon.getElementType());
    return builder.getTreeBuilt().getFirstChildNode();
}

Example 35 with Lexer

use of com.intellij.lexer.Lexer in project intellij-community by JetBrains.

the class MinifiedFilesUtil method isMinified.

/**
     * Finds out whether the file minified by using common (not language-specific) heuristics.
     * Can be used for checking of css/less/scss/sass and js files.
     *
     * @param fileContent              target file content
     * @param parserDefinition         Parser definition of target language
     * @param noWSRequireAfterTokenSet TokenSet of types that doesn't require whitespaces after them.
     * @param stringsTokenSet TokenSet of types considered as string elements
     */
public static boolean isMinified(@NotNull CharSequence fileContent, @NotNull ParserDefinition parserDefinition, @NotNull TokenSet noWSRequireBeforeTokenSet, @NotNull TokenSet noWSRequireAfterTokenSet, @NotNull TokenSet stringsTokenSet) {
    Lexer lexer = parserDefinition.createLexer(null);
    lexer.start(fileContent);
    if (!isMinified(lexer, parserDefinition, noWSRequireBeforeTokenSet, noWSRequireAfterTokenSet, stringsTokenSet)) {
        return false;
    } else if (lexer.getTokenType() == null) {
        // whole file had been considered
        return true;
    }
    int startOffset = fileContent.length() - COUNT_OF_CONSIDERING_CHARACTERS_FROM_END_OF_FILE;
    if (startOffset <= 0) {
        return true;
    }
    lexer.start(fileContent, startOffset, fileContent.length());
    return isMinified(lexer, parserDefinition, noWSRequireBeforeTokenSet, noWSRequireAfterTokenSet, stringsTokenSet);
}