Examples with Lexer com.intellij.lexer.Lexer used on opensource projects

Example 16 with Lexer

use of com.intellij.lexer.Lexer in project intellij-community by JetBrains.

the class JavaParserUtil method createBuilder.

@NotNull
public static PsiBuilder createBuilder(final ASTNode chameleon) {
    final PsiElement psi = chameleon.getPsi();
    assert psi != null : chameleon;
    final Project project = psi.getProject();
    CharSequence text;
    if (TreeUtil.isCollapsedChameleon(chameleon)) {
        text = chameleon.getChars();
    } else {
        text = psi.getUserData(IndexingDataKeys.FILE_TEXT_CONTENT_KEY);
        if (text == null)
            text = chameleon.getChars();
    }
    final PsiBuilderFactory factory = PsiBuilderFactory.getInstance();
    final LanguageLevel level = PsiUtil.getLanguageLevel(psi);
    final Lexer lexer = JavaParserDefinition.createLexer(level);
    Language language = psi.getLanguage();
    if (!language.isKindOf(JavaLanguage.INSTANCE))
        language = JavaLanguage.INSTANCE;
    final PsiBuilder builder = factory.createBuilder(project, chameleon, lexer, language, text);
    setLanguageLevel(builder, level);
    return builder;
}

Example 17 with Lexer

use of com.intellij.lexer.Lexer in project intellij-community by JetBrains.

the class JavaSourceRootDetectionUtil method getPackageName.

@Nullable
public static String getPackageName(CharSequence text) {
    Lexer lexer = JavaParserDefinition.createLexer(LanguageLevel.JDK_1_3);
    lexer.start(text);
    skipWhiteSpaceAndComments(lexer);
    final IElementType firstToken = lexer.getTokenType();
    if (firstToken != JavaTokenType.PACKAGE_KEYWORD) {
        if (JAVA_FILE_FIRST_TOKEN_SET.contains(firstToken)) {
            return "";
        }
        return null;
    }
    lexer.advance();
    skipWhiteSpaceAndComments(lexer);
    final StringBuilder buffer = StringBuilderSpinAllocator.alloc();
    try {
        while (true) {
            if (lexer.getTokenType() != JavaTokenType.IDENTIFIER)
                break;
            buffer.append(text, lexer.getTokenStart(), lexer.getTokenEnd());
            lexer.advance();
            skipWhiteSpaceAndComments(lexer);
            if (lexer.getTokenType() != JavaTokenType.DOT)
                break;
            buffer.append('.');
            lexer.advance();
            skipWhiteSpaceAndComments(lexer);
        }
        String packageName = buffer.toString();
        if (packageName.length() == 0 || StringUtil.endsWithChar(packageName, '.'))
            return null;
        return packageName;
    } finally {
        StringBuilderSpinAllocator.dispose(buffer);
    }
}

Example 18 with Lexer

use of com.intellij.lexer.Lexer in project intellij-community by JetBrains.

the class PsiElementFactoryImpl method createDummyHolder.

@NotNull
@Override
public PsiElement createDummyHolder(@NotNull final String text, @NotNull final IElementType type, @Nullable final PsiElement context) {
    final DummyHolder result = DummyHolderFactory.createHolder(myManager, context);
    final FileElement holder = result.getTreeElement();
    final Language language = type.getLanguage();
    final ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(language);
    assert parserDefinition != null : "No parser definition for language " + language;
    final Project project = myManager.getProject();
    final Lexer lexer = parserDefinition.createLexer(project);
    final PsiBuilder builder = PsiBuilderFactory.getInstance().createBuilder(project, holder, lexer, language, text);
    final ASTNode node = parserDefinition.createParser(project).parse(type, builder);
    holder.rawAddChildren((TreeElement) node);
    final PsiElement psi = node.getPsi();
    assert psi != null : text;
    return psi;
}

Example 19 with Lexer

use of com.intellij.lexer.Lexer in project intellij-community by JetBrains.

the class LightKeyword method getTokenType.

@Override
public IElementType getTokenType() {
    Lexer lexer = JavaParserDefinition.createLexer(LanguageLevel.HIGHEST);
    lexer.start(myText);
    return lexer.getTokenType();
}

Example 20 with Lexer

use of com.intellij.lexer.Lexer in project intellij-community by JetBrains.

the class IElementTypeTest method testCount.

// load all parser definitions, instantiate all lexers & parsers to initialize all IElementType constants
@SuppressWarnings("UnusedDeclaration")
public void testCount() throws Exception {
    int count = IElementType.getAllocatedTypesCount();
    System.out.println("Preloaded: " + count + " element types");
    LanguageExtensionPoint[] extensions = Extensions.getExtensions(new ExtensionPointName<LanguageExtensionPoint>("com.intellij.lang.parserDefinition"));
    System.out.println("ParserDefinitions: " + extensions.length);
    THashMap<Language, String> languageMap = new THashMap<>();
    languageMap.put(Language.ANY, "platform");
    final TObjectIntHashMap<String> map = new TObjectIntHashMap<>();
    for (LanguageExtensionPoint e : extensions) {
        String key = e.getPluginDescriptor().getPluginId().getIdString();
        int curCount = IElementType.getAllocatedTypesCount();
        ParserDefinition definition = (ParserDefinition) e.getInstance();
        IFileElementType type = definition.getFileNodeType();
        Language language = type.getLanguage();
        languageMap.put(language, key);
        if (language.getBaseLanguage() != null && !languageMap.containsKey(language.getBaseLanguage())) {
            languageMap.put(language.getBaseLanguage(), key);
        }
        try {
            Lexer lexer = definition.createLexer(getProject());
            PsiParser parser = definition.createParser(getProject());
        } catch (UnsupportedOperationException e1) {
        }
    // language-based calculation: per-class-loading stuff commented
    //int diff = IElementType.getAllocatedTypesCount() - curCount;
    //map.put(key, map.get(key) + diff);
    }
    // language-based calculation
    count = IElementType.getAllocatedTypesCount();
    for (short i = 0; i < count; i++) {
        IElementType type = IElementType.find(i);
        Language language = type.getLanguage();
        String key = null;
        for (Language cur = language; cur != null && key == null; cur = cur.getBaseLanguage()) {
            key = languageMap.get(cur);
        }
        key = StringUtil.notNullize(key, "unknown");
        map.put(key, map.get(key) + 1);
    //if (key.equals("unknown")) System.out.println(type +"   " + language);
    }
    System.out.println("Total: " + IElementType.getAllocatedTypesCount() + " element types");
    // Show per-plugin statistics
    Object[] keys = map.keys();
    Arrays.sort(keys, (o1, o2) -> map.get((String) o2) - map.get((String) o1));
    int sum = 0;
    for (Object key : keys) {
        int value = map.get((String) key);
        if (value == 0)
            continue;
        sum += value;
        System.out.println("  " + key + ": " + value);
    }
    // leave some index-space for plugin developers
    assertTrue(IElementType.getAllocatedTypesCount() < 10000);
    assertEquals(IElementType.getAllocatedTypesCount(), sum);
// output on 11.05.2012
//   Preloaded: 3485 types
//   95 definitions
//   Total: 7694 types
// 14.05.2015:
// Preloaded: 4106 element types
// ParserDefinitions: 128
// Total: 8864 element types
// 19.04.2016:
// Preloaded: 4397 element types
// ParserDefinitions: 135
// Total: 9231 element types
}