Examples with CharsetToolkit com.intellij.openapi.vfs.CharsetToolkit used on opensource projects

Example 1 with CharsetToolkit

use of com.intellij.openapi.vfs.CharsetToolkit in project intellij-community by JetBrains.

the class VcsVirtualFile method loadContent.

private void loadContent() throws IOException {
    if (myContent != null)
        return;
    assert myFileRevision != null;
    final VcsFileSystem vcsFileSystem = ((VcsFileSystem) getFileSystem());
    try {
        myFileRevision.loadContent();
        fireBeforeContentsChange();
        myModificationStamp++;
        final VcsRevisionNumber revisionNumber = myFileRevision.getRevisionNumber();
        if (revisionNumber instanceof ShortVcsRevisionNumber) {
            setRevision(((ShortVcsRevisionNumber) revisionNumber).toShortString());
        } else {
            setRevision(revisionNumber.asString());
        }
        myContent = myFileRevision.getContent();
        myCharset = new CharsetToolkit(myContent).guessEncoding(myContent.length);
        ApplicationManager.getApplication().runWriteAction(new Runnable() {

            public void run() {
                vcsFileSystem.fireContentsChanged(this, VcsVirtualFile.this, 0);
            }
        });
    } catch (VcsException e) {
        myContentLoadFailed = true;
        ApplicationManager.getApplication().runWriteAction(new Runnable() {

            public void run() {
                vcsFileSystem.fireBeforeFileDeletion(this, VcsVirtualFile.this);
            }
        });
        myContent = ArrayUtil.EMPTY_BYTE_ARRAY;
        setRevision("0");
        Messages.showMessageDialog(VcsBundle.message("message.text.could.not.load.virtual.file.content", getPresentableUrl(), e.getLocalizedMessage()), VcsBundle.message("message.title.could.not.load.content"), Messages.getInformationIcon());
        ApplicationManager.getApplication().runWriteAction(new Runnable() {

            public void run() {
                vcsFileSystem.fireFileDeleted(this, VcsVirtualFile.this, getName(), getParent());
            }
        });
    } catch (ProcessCanceledException ex) {
        myContent = null;
    }
}

Example 2 with CharsetToolkit

use of com.intellij.openapi.vfs.CharsetToolkit in project intellij-community by JetBrains.

the class DiffContentFactoryImpl method guessCharsetFromContent.

@Nullable
private static Charset guessCharsetFromContent(@NotNull byte[] content) {
    // can't use CharsetToolkit.guessEncoding here because of false-positive INVALID_UTF8
    CharsetToolkit toolkit = new CharsetToolkit(content);
    Charset fromBOM = toolkit.guessFromBOM();
    if (fromBOM != null)
        return fromBOM;
    CharsetToolkit.GuessedEncoding guessedEncoding = toolkit.guessFromContent(content.length);
    switch(guessedEncoding) {
        case SEVEN_BIT:
            return Charset.forName("US-ASCII");
        case VALID_UTF8:
            return CharsetToolkit.UTF8_CHARSET;
        default:
            return null;
    }
}

Example 3 with CharsetToolkit

use of com.intellij.openapi.vfs.CharsetToolkit in project intellij-community by JetBrains.

the class FileTypesTest method test7BitIsText.

public void test7BitIsText() throws IOException {
    File d = createTempDirectory();
    File f = new File(d, "xx.asfdasdfas");
    byte[] bytes = { 9, 10, 13, 'x', 'a', 'b' };
    assertEquals(CharsetToolkit.GuessedEncoding.SEVEN_BIT, new CharsetToolkit(bytes).guessFromContent(bytes.length));
    FileUtil.writeToFile(f, bytes);
    VirtualFile vFile = getVirtualFile(f);
    assertEquals(PlainTextFileType.INSTANCE, vFile.getFileType());
}

Example 4 with CharsetToolkit

use of com.intellij.openapi.vfs.CharsetToolkit in project intellij-community by JetBrains.

the class LoadTextUtil method guessFromContent.

@Nullable("null means no luck, otherwise it's tuple(guessed encoding, hint about content if was unable to guess, BOM)")
public static Trinity<Charset, CharsetToolkit.GuessedEncoding, byte[]> guessFromContent(@NotNull VirtualFile virtualFile, @NotNull byte[] content, int length) {
    Charset defaultCharset = ObjectUtils.notNull(EncodingManager.getInstance().getEncoding(virtualFile, true), CharsetToolkit.getDefaultSystemCharset());
    CharsetToolkit toolkit = GUESS_UTF ? new CharsetToolkit(content, defaultCharset) : null;
    String detectedFromBytes = null;
    try {
        if (GUESS_UTF) {
            toolkit.setEnforce8Bit(true);
            Charset charset = toolkit.guessFromBOM();
            if (charset != null) {
                detectedFromBytes = AUTO_DETECTED_FROM_BOM;
                byte[] bom = ObjectUtils.notNull(CharsetToolkit.getMandatoryBom(charset), CharsetToolkit.UTF8_BOM);
                return Trinity.create(charset, null, bom);
            }
            CharsetToolkit.GuessedEncoding guessed = toolkit.guessFromContent(length);
            if (guessed == CharsetToolkit.GuessedEncoding.VALID_UTF8) {
                detectedFromBytes = "auto-detected from bytes";
                //UTF detected, ignore all directives
                return Trinity.create(CharsetToolkit.UTF8_CHARSET, guessed, null);
            }
            if (guessed == CharsetToolkit.GuessedEncoding.SEVEN_BIT) {
                return Trinity.create(null, guessed, null);
            }
        }
        return null;
    } finally {
        setCharsetWasDetectedFromBytes(virtualFile, detectedFromBytes);
    }
}

Example 5 with CharsetToolkit

use of com.intellij.openapi.vfs.CharsetToolkit in project intellij-community by JetBrains.

the class FileTypesTest method test7BitBinaryIsNotText.

public void test7BitBinaryIsNotText() throws IOException {
    File d = createTempDirectory();
    File f = new File(d, "xx.asfdasdfas");
    byte[] bytes = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 'x', 'a', 'b' };
    assertEquals(CharsetToolkit.GuessedEncoding.BINARY, new CharsetToolkit(bytes).guessFromContent(bytes.length));
    FileUtil.writeToFile(f, bytes);
    VirtualFile vFile = getVirtualFile(f);
    assertEquals(UnknownFileType.INSTANCE, vFile.getFileType());
}