Examples with FilterLexer com.intellij.lexer.FilterLexer used on opensource projects

Example 1 with FilterLexer

use of com.intellij.lexer.FilterLexer in project intellij-community by JetBrains.

the class XmlParsingTest method _testLexerPerformance2.

public void _testLexerPerformance2() throws Exception {
    final String text = loadFile("performance2.xml");
    XmlLexer lexer = new XmlLexer();
    doLex(lexer, text);
    final FilterLexer filterLexer = new FilterLexer(new XmlLexer(), new FilterLexer.SetFilter(LanguageParserDefinitions.INSTANCE.forLanguage(XMLLanguage.INSTANCE).getWhitespaceTokens()));
    doLex(filterLexer, text);
    doLex(lexer, text);
    for (int i = 0; i < 20; i++) {
        doLex(filterLexer, text);
    }
}

Example 2 with FilterLexer

use of com.intellij.lexer.FilterLexer in project intellij-community by JetBrains.

the class FunctionDeclarationParsing method parseFuntionDeclaration.

public static Pair<String, ? extends Function> parseFuntionDeclaration(String decl) {
    final Lexer lexer = new FilterLexer(XPathLexer.create(true), new FilterLexer.SetFilter(TokenSet.create(XPathTokenTypes.WHITESPACE)));
    lexer.start(decl);
    String prefix = "";
    if (lexer.getTokenType() == XPathTokenTypes.EXT_PREFIX) {
        prefix = lexer.getTokenText();
        lexer.advance();
        match(lexer, XPathTokenTypes.COL);
    }
    final String name = match(lexer, XPathTokenTypes.FUNCTION_NAME);
    match(lexer, XPathTokenTypes.LPAREN);
    final List<Parameter> parameters = new ArrayList<>();
    while (lexer.getTokenType() != XPathTokenTypes.RPAREN) {
        if (lexer.getTokenType() == XPathTokenTypes.DOTDOT) {
            lexer.advance();
            match(lexer, XPathTokenTypes.DOT);
            parameters.add(new Parameter(XPathType.ANY, Parameter.Kind.VARARG));
        } else {
            match(lexer, XPathTokenTypes.DOLLAR);
            match(lexer, XPathTokenTypes.VARIABLE_NAME);
            match(lexer, XPath2TokenTypes.AS);
            final String type = parseType(lexer);
            final XPath2SequenceType.Cardinality indicator = parseCardinality(lexer);
            parameters.add(new Parameter(mapType(type, indicator), Parameter.Kind.REQUIRED));
        }
        if (lexer.getTokenType() == XPathTokenTypes.COMMA) {
            lexer.advance();
        }
    }
    lexer.advance();
    match(lexer, XPath2TokenTypes.AS);
    final String ret = parseType(lexer);
    final XPath2SequenceType.Cardinality indicator = parseCardinality(lexer);
    final XPathType returnType = mapType(ret, indicator);
    return Pair.create(prefix, new FunctionImpl(name, returnType, parameters.toArray(new Parameter[parameters.size()])));
}

Example 3 with FilterLexer

use of com.intellij.lexer.FilterLexer in project intellij-community by JetBrains.

the class InnerClassesScannerTest method mayContainClassesInside.

private static boolean mayContainClassesInside(PsiElement psiElement, final CharSequence fileBuffer) {
    PsiFile psiFile = psiElement.getContainingFile();
    boolean mayHaveClassesInside = false;
    if (psiFile instanceof PsiJavaFileImpl) {
        Lexer originalLexer = JavaParserDefinition.createLexer(((PsiJavaFileImpl) psiFile).getLanguageLevel());
        FilterLexer lexer = new FilterLexer(originalLexer, new FilterLexer.SetFilter(StdTokenSets.WHITE_SPACE_OR_COMMENT_BIT_SET));
        final TextRange range = psiElement.getTextRange();
        lexer.start(fileBuffer, range.getStartOffset(), range.getEndOffset());
        boolean isInNewExpression = false;
        boolean isRightAfterNewExpression = false;
        int angleLevel = 0;
        int parenLevel = 0;
        do {
            IElementType tokenType = lexer.getTokenType();
            if (tokenType == null)
                break;
            if (tokenType == JavaTokenType.NEW_KEYWORD) {
                isInNewExpression = true;
            } else if (tokenType == JavaTokenType.LPARENTH) {
                if (isInNewExpression)
                    parenLevel++;
            } else if (tokenType == JavaTokenType.LT) {
                if (isInNewExpression)
                    angleLevel++;
            } else if (tokenType == JavaTokenType.GT) {
                if (isInNewExpression)
                    angleLevel--;
            } else if (tokenType == JavaTokenType.RPARENTH) {
                if (isInNewExpression) {
                    parenLevel--;
                    if (parenLevel == 0) {
                        isRightAfterNewExpression = true;
                    }
                }
            } else if (tokenType == JavaTokenType.LBRACE) {
                if (isInNewExpression || isRightAfterNewExpression) {
                    mayHaveClassesInside = true;
                }
            } else if (tokenType == JavaTokenType.LBRACKET) {
                if (parenLevel == 0 && angleLevel == 0)
                    isInNewExpression = false;
            } else if (tokenType == JavaTokenType.INTERFACE_KEYWORD || tokenType == JavaTokenType.CLASS_KEYWORD || tokenType == JavaTokenType.ENUM_KEYWORD) {
                mayHaveClassesInside = true;
            }
            if (isInNewExpression && isRightAfterNewExpression) {
                isInNewExpression = false;
            } else {
                isRightAfterNewExpression = false;
            }
            lexer.advance();
        } while (!mayHaveClassesInside);
    }
    return mayHaveClassesInside;
}

Example 4 with FilterLexer

use of com.intellij.lexer.FilterLexer in project intellij-community by JetBrains.

the class XmlParsingTest method _testLexerPerformance1.

public void _testLexerPerformance1() throws Exception {
    final String text = loadFile("pallada.xml");
    XmlLexer lexer = new XmlLexer();
    doLex(lexer, text);
    final FilterLexer filterLexer = new FilterLexer(new XmlLexer(), new FilterLexer.SetFilter(LanguageParserDefinitions.INSTANCE.forLanguage(XMLLanguage.INSTANCE).getWhitespaceTokens()));
    doLex(filterLexer, text);
    doLex(lexer, text);
    doLex(filterLexer, text);
    doLex(filterLexer, text);
}