Examples with Lexer com.intellij.lexer.Lexer used on opensource projects

Example 1 with Lexer

use of com.intellij.lexer.Lexer in project intellij-community by JetBrains.

the class PyRegexpTest method testVerbose.

public void testVerbose() {
    Lexer lexer = new PythonVerboseRegexpParserDefinition().createLexer(myFixture.getProject());
    PyLexerTestCase.doLexerTest("# abc", lexer, "COMMENT", "COMMENT");
}

Example 2 with Lexer

use of com.intellij.lexer.Lexer in project intellij-community by JetBrains.

the class PyRegexpTest method doTestLexer.

private void doTestLexer(final String text, String... expectedTokens) {
    Lexer lexer = new PythonRegexpParserDefinition().createLexer(myFixture.getProject());
    PyLexerTestCase.doLexerTest(text, lexer, expectedTokens);
}

Example 3 with Lexer

use of com.intellij.lexer.Lexer in project intellij-community by JetBrains.

the class FormSourceCodeGenerator method lexemsEqual.

private static boolean lexemsEqual(final PsiClass classToBind, final PsiClass newClass) {
    Lexer oldTextLexer = JavaParserDefinition.createLexer(LanguageLevel.HIGHEST);
    Lexer newTextLexer = JavaParserDefinition.createLexer(LanguageLevel.HIGHEST);
    String oldBuffer = classToBind.getText();
    String newBuffer = newClass.getText();
    oldTextLexer.start(oldBuffer);
    newTextLexer.start(newBuffer);
    while (true) {
        IElementType oldLexem = oldTextLexer.getTokenType();
        IElementType newLexem = newTextLexer.getTokenType();
        if (oldLexem == null || newLexem == null) {
            // must terminate at the same time
            return oldLexem == null && newLexem == null;
        }
        if (oldLexem != newLexem) {
            return false;
        }
        if (oldLexem != TokenType.WHITE_SPACE && oldLexem != JavaDocElementType.DOC_COMMENT) {
            int oldStart = oldTextLexer.getTokenStart();
            int newStart = newTextLexer.getTokenStart();
            int oldLength = oldTextLexer.getTokenEnd() - oldTextLexer.getTokenStart();
            int newLength = newTextLexer.getTokenEnd() - newTextLexer.getTokenStart();
            if (oldLength != newLength) {
                return false;
            }
            for (int i = 0; i < oldLength; i++) {
                if (oldBuffer.charAt(oldStart + i) != newBuffer.charAt(newStart + i)) {
                    return false;
                }
            }
        }
        oldTextLexer.advance();
        newTextLexer.advance();
    }
}

Example 4 with Lexer

use of com.intellij.lexer.Lexer in project intellij-community by JetBrains.

the class PropertiesLexerTest method doTestHL.

private static void doTestHL(@NonNls String text, @NonNls String[] expectedTokens) {
    Lexer lexer = new PropertiesHighlightingLexer();
    doTest(text, expectedTokens, lexer);
}

Example 5 with Lexer

use of com.intellij.lexer.Lexer in project intellij-community by JetBrains.

the class PyFileElementType method parseContents.

@Nullable
@Override
public ASTNode parseContents(ASTNode node) {
    final LanguageLevel languageLevel = getLanguageLevel(node.getPsi());
    if (PydevConsoleRunner.isPythonConsole(node)) {
        return parseConsoleCode(node, PydevConsoleRunner.getPythonConsoleData(node));
    } else {
        final PsiElement psi = node.getPsi();
        if (psi != null) {
            final Project project = psi.getProject();
            final PsiBuilderFactory factory = PsiBuilderFactory.getInstance();
            final Language language = getLanguage();
            final ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(language);
            if (parserDefinition == null) {
                return null;
            }
            final Lexer lexer = parserDefinition.createLexer(project);
            final PsiParser parser = parserDefinition.createParser(project);
            final PsiBuilder builder = factory.createBuilder(project, node, lexer, language, node.getChars());
            if (parser instanceof PyParser) {
                final PyParser pythonParser = (PyParser) parser;
                pythonParser.setLanguageLevel(languageLevel);
                if (languageLevel == LanguageLevel.PYTHON26 && psi.getContainingFile().getName().equals("__builtin__.py")) {
                    pythonParser.setFutureFlag(StatementParsing.FUTURE.PRINT_FUNCTION);
                }
            }
            return parser.parse(this, builder).getFirstChildNode();
        }
        return null;
    }
}