Examples with Lexer com.intellij.lexer.Lexer used on opensource projects

Example 6 with Lexer

use of com.intellij.lexer.Lexer in project intellij-community by JetBrains.

the class PyRegexpTest method testVerbose.

public void testVerbose() {
    Lexer lexer = new PythonVerboseRegexpParserDefinition().createLexer(myFixture.getProject());
    PyLexerTestCase.doLexerTest("# abc", lexer, "COMMENT", "COMMENT");
}

Example 7 with Lexer

use of com.intellij.lexer.Lexer in project intellij-community by JetBrains.

the class PyRegexpTest method doTestLexer.

private void doTestLexer(final String text, String... expectedTokens) {
    Lexer lexer = new PythonRegexpParserDefinition().createLexer(myFixture.getProject());
    PyLexerTestCase.doLexerTest(text, lexer, expectedTokens);
}

Example 8 with Lexer

use of com.intellij.lexer.Lexer in project intellij-community by JetBrains.

the class FormSourceCodeGenerator method lexemsEqual.

private static boolean lexemsEqual(final PsiClass classToBind, final PsiClass newClass) {
    Lexer oldTextLexer = JavaParserDefinition.createLexer(LanguageLevel.HIGHEST);
    Lexer newTextLexer = JavaParserDefinition.createLexer(LanguageLevel.HIGHEST);
    String oldBuffer = classToBind.getText();
    String newBuffer = newClass.getText();
    oldTextLexer.start(oldBuffer);
    newTextLexer.start(newBuffer);
    while (true) {
        IElementType oldLexem = oldTextLexer.getTokenType();
        IElementType newLexem = newTextLexer.getTokenType();
        if (oldLexem == null || newLexem == null) {
            // must terminate at the same time
            return oldLexem == null && newLexem == null;
        }
        if (oldLexem != newLexem) {
            return false;
        }
        if (oldLexem != TokenType.WHITE_SPACE && oldLexem != JavaDocElementType.DOC_COMMENT) {
            int oldStart = oldTextLexer.getTokenStart();
            int newStart = newTextLexer.getTokenStart();
            int oldLength = oldTextLexer.getTokenEnd() - oldTextLexer.getTokenStart();
            int newLength = newTextLexer.getTokenEnd() - newTextLexer.getTokenStart();
            if (oldLength != newLength) {
                return false;
            }
            for (int i = 0; i < oldLength; i++) {
                if (oldBuffer.charAt(oldStart + i) != newBuffer.charAt(newStart + i)) {
                    return false;
                }
            }
        }
        oldTextLexer.advance();
        newTextLexer.advance();
    }
}

Example 9 with Lexer

use of com.intellij.lexer.Lexer in project intellij-community by JetBrains.

the class CommentByBlockCommentHandler method findCommentedRange.

@Nullable
private TextRange findCommentedRange(final Commenter commenter) {
    final CharSequence text = myDocument.getCharsSequence();
    final FileType fileType = myFile.getFileType();
    if (fileType instanceof CustomSyntaxTableFileType) {
        Lexer lexer = new CustomFileTypeLexer(((CustomSyntaxTableFileType) fileType).getSyntaxTable());
        final int caretOffset = myCaret.getOffset();
        int commentStart = CharArrayUtil.lastIndexOf(text, commenter.getBlockCommentPrefix(), caretOffset);
        if (commentStart == -1)
            return null;
        lexer.start(text, commentStart, text.length());
        if (lexer.getTokenType() == CustomHighlighterTokenType.MULTI_LINE_COMMENT && lexer.getTokenEnd() >= caretOffset) {
            return new TextRange(commentStart, lexer.getTokenEnd());
        }
        return null;
    }
    final String prefix;
    final String suffix;
    // Custom uncommenter is able to find commented block inside of selected text
    final String selectedText = myCaret.getSelectedText();
    if ((commenter instanceof CustomUncommenter) && selectedText != null) {
        final TextRange commentedRange = ((CustomUncommenter) commenter).findMaximumCommentedRange(selectedText);
        if (commentedRange == null) {
            return null;
        }
        // Uncommenter returns range relative to text start, so we need to shift it to make abosolute.
        return commentedRange.shiftRight(myCaret.getSelectionStart());
    }
    if (commenter instanceof SelfManagingCommenter) {
        SelfManagingCommenter selfManagingCommenter = (SelfManagingCommenter) commenter;
        prefix = selfManagingCommenter.getBlockCommentPrefix(myCaret.getSelectionStart(), myDocument, mySelfManagedCommenterData);
        suffix = selfManagingCommenter.getBlockCommentSuffix(myCaret.getSelectionEnd(), myDocument, mySelfManagedCommenterData);
    } else {
        prefix = trim(commenter.getBlockCommentPrefix());
        suffix = trim(commenter.getBlockCommentSuffix());
    }
    if (prefix == null || suffix == null)
        return null;
    TextRange commentedRange;
    if (commenter instanceof SelfManagingCommenter) {
        commentedRange = ((SelfManagingCommenter) commenter).getBlockCommentRange(myCaret.getSelectionStart(), myCaret.getSelectionEnd(), myDocument, mySelfManagedCommenterData);
    } else {
        if (!testSelectionForNonComments()) {
            return null;
        }
        commentedRange = getSelectedComments(text, prefix, suffix);
    }
    if (commentedRange == null) {
        PsiElement comment = findCommentAtCaret();
        if (comment != null) {
            String commentText = comment.getText();
            if (commentText.startsWith(prefix) && commentText.endsWith(suffix)) {
                commentedRange = comment.getTextRange();
            }
        }
    }
    return commentedRange;
}

Example 10 with Lexer

use of com.intellij.lexer.Lexer in project intellij-community by JetBrains.

the class MacroParser method parse.

@NotNull
public static Expression parse(@Nullable String expression) {
    if (StringUtil.isEmpty(expression)) {
        return new ConstantNode("");
    }
    Lexer lexer = new MacroLexer();
    lexer.start(expression);
    skipWhitespaces(lexer);
    return parseMacro(lexer, expression);
}