Examples with ITokenizer org.omegat.tokenizer.ITokenizer used on opensource projects

Example 6 with ITokenizer

use of org.omegat.tokenizer.ITokenizer in project omegat by omegat-org.

the class GlossaryManager method searchTargetMatches.

/**
 * Get all target terms for the provided glossary entry that can be found in the provided string.
 *
 * @param trg
 *            The text to search
 * @param protectedParts
 *            A list of protected parts from which matches should be disregarded (can be null)
 * @param entry
 *            The glossary entry whose target terms should be searched
 * @return A list of matching target terms
 */
public List<String> searchTargetMatches(String trg, ProtectedPart[] protectedParts, GlossaryEntry entry) {
    ITokenizer tok = Core.getProject().getTargetTokenizer();
    if (tok == null) {
        return Collections.emptyList();
    }
    GlossarySearcher searcher = new GlossarySearcher(tok, Core.getProject().getProjectProperties().getTargetLanguage());
    return searcher.searchTargetMatches(trg, protectedParts, entry);
}

Example 7 with ITokenizer

use of org.omegat.tokenizer.ITokenizer in project omegat by omegat-org.

the class FalseFriendsTest method setUp.

@Before
public final void setUp() {
    final ProjectProperties props = new ProjectProperties() {

        public Language getSourceLanguage() {
            return new Language("en");
        }

        public Language getTargetLanguage() {
            return new Language("pl");
        }
    };
    Core.setProject(new IProject() {

        public void setTranslation(SourceTextEntry entry, PrepareTMXEntry trans, boolean defaultTranslation, TMXEntry.ExternalLinked externalLinked) {
        }

        public void setTranslation(SourceTextEntry entry, PrepareTMXEntry trans, boolean defaultTranslation, ExternalLinked externalLinked, AllTranslations previousTranslations) throws OptimisticLockingFail {
        }

        public void setNote(SourceTextEntry entry, TMXEntry oldTrans, String note) {
        }

        public void saveProjectProperties() throws Exception {
        }

        public void saveProject(boolean doTeamSync) {
        }

        public void iterateByMultipleTranslations(MultipleTranslationsIterator it) {
        }

        public void iterateByDefaultTranslations(DefaultTranslationsIterator it) {
        }

        public boolean isProjectModified() {
            return false;
        }

        public boolean isProjectLoaded() {
            return true;
        }

        public boolean isOrphaned(EntryKey entry) {
            return false;
        }

        public boolean isOrphaned(String source) {
            return false;
        }

        public TMXEntry getTranslationInfo(SourceTextEntry ste) {
            return null;
        }

        public AllTranslations getAllTranslations(SourceTextEntry ste) {
            return null;
        }

        public Map<String, ExternalTMX> getTransMemories() {
            return null;
        }

        public ITokenizer getTargetTokenizer() {
            return null;
        }

        public StatisticsInfo getStatistics() {
            return null;
        }

        public ITokenizer getSourceTokenizer() {
            return null;
        }

        public ProjectProperties getProjectProperties() {
            return props;
        }

        public List<FileInfo> getProjectFiles() {
            return null;
        }

        public Map<Language, ProjectTMX> getOtherTargetLanguageTMs() {
            return null;
        }

        public List<SourceTextEntry> getAllEntries() {
            return null;
        }

        public void compileProject(String sourcePattern) throws Exception {
        }

        public void closeProject() {
        }

        public List<String> getSourceFilesOrder() {
            return null;
        }

        public void setSourceFilesOrder(List<String> filesList) {
        }

        @Override
        public String getTargetPathForSourceFile(String sourceFile) {
            return null;
        }

        @Override
        public boolean isTeamSyncPrepared() {
            return false;
        }

        @Override
        public void teamSync() {
        }

        @Override
        public void teamSyncPrepare() throws Exception {
        }

        @Override
        public boolean isRemoteProject() {
            return false;
        }

        @Override
        public void commitSourceFiles() throws Exception {
        }

        @Override
        public void compileProjectAndCommit(String sourcePattern, boolean doPostProcessing, boolean commitTargetFiles) throws Exception {
        }
    });
    LanguageToolWrapper.setBridgeFromCurrentProject();
}

Example 8 with ITokenizer

use of org.omegat.tokenizer.ITokenizer in project omegat by omegat-org.

the class EditorUtilsTest method testChangeCase.

@Test
public void testChangeCase() {
    Locale locale = Locale.ENGLISH;
    ITokenizer tokenizer = new LuceneEnglishTokenizer();
    String input = "a I've GOT a {crazy} text hErE including 1 \u65e5\u672c\u8a9e!";
    String round1 = EditorUtils.doChangeCase(input, CHANGE_CASE_TO.CYCLE, locale, tokenizer);
    assertEquals("A I'VE GOT A {CRAZY} TEXT HERE INCLUDING 1 \u65e5\u672c\u8a9e!", round1);
    assertEquals(round1, EditorUtils.doChangeCase(input, CHANGE_CASE_TO.UPPER, locale, tokenizer));
    String round2 = EditorUtils.doChangeCase(round1, CHANGE_CASE_TO.CYCLE, locale, tokenizer);
    assertEquals("a i've got a {crazy} text here including 1 \u65e5\u672c\u8a9e!", round2);
    assertEquals(round2, EditorUtils.doChangeCase(input, CHANGE_CASE_TO.LOWER, locale, tokenizer));
    String round3 = EditorUtils.doChangeCase(round2, CHANGE_CASE_TO.CYCLE, locale, tokenizer);
    assertEquals("A i've got a {crazy} text here including 1 \u65e5\u672c\u8a9e!", round3);
    assertEquals(round3, EditorUtils.doChangeCase(input, CHANGE_CASE_TO.SENTENCE, locale, tokenizer));
    String round4 = EditorUtils.doChangeCase(round3, CHANGE_CASE_TO.CYCLE, locale, tokenizer);
    assertEquals("A I've Got A {Crazy} Text Here Including 1 \u65e5\u672c\u8a9e!", round4);
    assertEquals(round4, EditorUtils.doChangeCase(input, CHANGE_CASE_TO.TITLE, locale, tokenizer));
    String round5 = EditorUtils.doChangeCase(round4, CHANGE_CASE_TO.CYCLE, locale, tokenizer);
    assertEquals(round1, round5);
    input = "lower case only";
    assertEquals(input, EditorUtils.doChangeCase(input, CHANGE_CASE_TO.LOWER, locale, tokenizer));
    assertEquals("LOWER CASE ONLY", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.UPPER, locale, tokenizer));
    assertEquals("Lower case only", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.SENTENCE, locale, tokenizer));
    assertEquals("Lower Case Only", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.TITLE, locale, tokenizer));
    assertEquals("Lower case only", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.CYCLE, locale, tokenizer));
    input = "UPPER CASE ONLY";
    assertEquals("upper case only", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.LOWER, locale, tokenizer));
    assertEquals(input, EditorUtils.doChangeCase(input, CHANGE_CASE_TO.UPPER, locale, tokenizer));
    assertEquals("Upper case only", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.SENTENCE, locale, tokenizer));
    assertEquals("Upper Case Only", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.TITLE, locale, tokenizer));
    assertEquals("upper case only", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.CYCLE, locale, tokenizer));
    input = "Title Case Only";
    assertEquals("title case only", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.LOWER, locale, tokenizer));
    assertEquals("TITLE CASE ONLY", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.UPPER, locale, tokenizer));
    assertEquals("Title case only", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.SENTENCE, locale, tokenizer));
    assertEquals(input, EditorUtils.doChangeCase(input, CHANGE_CASE_TO.TITLE, locale, tokenizer));
    assertEquals("TITLE CASE ONLY", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.CYCLE, locale, tokenizer));
    input = "Sentence case string";
    assertEquals("sentence case string", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.LOWER, locale, tokenizer));
    assertEquals("SENTENCE CASE STRING", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.UPPER, locale, tokenizer));
    assertEquals(input, EditorUtils.doChangeCase(input, CHANGE_CASE_TO.SENTENCE, locale, tokenizer));
    assertEquals("Sentence Case String", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.TITLE, locale, tokenizer));
    assertEquals("Sentence Case String", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.CYCLE, locale, tokenizer));
    input = "mIxed CaSe oNly";
    assertEquals("mixed case only", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.LOWER, locale, tokenizer));
    assertEquals("MIXED CASE ONLY", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.UPPER, locale, tokenizer));
    assertEquals("Mixed case only", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.SENTENCE, locale, tokenizer));
    assertEquals("Mixed Case Only", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.TITLE, locale, tokenizer));
    assertEquals("MIXED CASE ONLY", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.CYCLE, locale, tokenizer));
    // Ambiguous only
    input = "A B C";
    assertEquals("a b c", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.LOWER, locale, tokenizer));
    assertEquals(input, EditorUtils.doChangeCase(input, CHANGE_CASE_TO.UPPER, locale, tokenizer));
    assertEquals("A b c", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.SENTENCE, locale, tokenizer));
    assertEquals(input, EditorUtils.doChangeCase(input, CHANGE_CASE_TO.TITLE, locale, tokenizer));
    round2 = EditorUtils.doChangeCase(input, CHANGE_CASE_TO.CYCLE, locale, tokenizer);
    assertEquals("a b c", round2);
    round3 = EditorUtils.doChangeCase(round2, CHANGE_CASE_TO.CYCLE, locale, tokenizer);
    assertEquals("A b c", round3);
    round4 = EditorUtils.doChangeCase(round3, CHANGE_CASE_TO.CYCLE, locale, tokenizer);
    assertEquals(input, round4);
    // No letter-containing tokens
    input = "{!} 1 \u65e5\u672c\u8a9e";
    assertEquals(input, EditorUtils.doChangeCase(input, CHANGE_CASE_TO.LOWER, locale, tokenizer));
    assertEquals(input, EditorUtils.doChangeCase(input, CHANGE_CASE_TO.UPPER, locale, tokenizer));
    assertEquals(input, EditorUtils.doChangeCase(input, CHANGE_CASE_TO.SENTENCE, locale, tokenizer));
    assertEquals(input, EditorUtils.doChangeCase(input, CHANGE_CASE_TO.TITLE, locale, tokenizer));
    assertEquals(input, EditorUtils.doChangeCase(input, CHANGE_CASE_TO.CYCLE, locale, tokenizer));
    // Single tokens
    input = "lower";
    assertEquals(input, EditorUtils.doChangeCase(input, CHANGE_CASE_TO.LOWER, locale, tokenizer));
    assertEquals("LOWER", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.UPPER, locale, tokenizer));
    assertEquals("Lower", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.SENTENCE, locale, tokenizer));
    assertEquals("Lower", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.TITLE, locale, tokenizer));
    assertEquals("Lower", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.CYCLE, locale, tokenizer));
    input = "UPPER";
    assertEquals("upper", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.LOWER, locale, tokenizer));
    assertEquals(input, EditorUtils.doChangeCase(input, CHANGE_CASE_TO.UPPER, locale, tokenizer));
    assertEquals("Upper", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.SENTENCE, locale, tokenizer));
    assertEquals("Upper", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.TITLE, locale, tokenizer));
    assertEquals("upper", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.CYCLE, locale, tokenizer));
    input = "Title";
    assertEquals("title", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.LOWER, locale, tokenizer));
    assertEquals("TITLE", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.UPPER, locale, tokenizer));
    assertEquals(input, EditorUtils.doChangeCase(input, CHANGE_CASE_TO.SENTENCE, locale, tokenizer));
    assertEquals(input, EditorUtils.doChangeCase(input, CHANGE_CASE_TO.TITLE, locale, tokenizer));
    assertEquals("TITLE", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.CYCLE, locale, tokenizer));
    input = "mIxed";
    assertEquals("mixed", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.LOWER, locale, tokenizer));
    assertEquals("MIXED", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.UPPER, locale, tokenizer));
    assertEquals("Mixed", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.SENTENCE, locale, tokenizer));
    assertEquals("Mixed", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.TITLE, locale, tokenizer));
    assertEquals("MIXED", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.CYCLE, locale, tokenizer));
    // Ambiguous
    input = "A";
    assertEquals("a", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.LOWER, locale, tokenizer));
    assertEquals(input, EditorUtils.doChangeCase(input, CHANGE_CASE_TO.UPPER, locale, tokenizer));
    assertEquals(input, EditorUtils.doChangeCase(input, CHANGE_CASE_TO.SENTENCE, locale, tokenizer));
    assertEquals(input, EditorUtils.doChangeCase(input, CHANGE_CASE_TO.TITLE, locale, tokenizer));
    assertEquals("a", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.CYCLE, locale, tokenizer));
    // Includes number (https://sourceforge.net/p/omegat/bugs/612/)
    input = "MQL5";
    assertEquals("mql5", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.LOWER, locale, tokenizer));
    assertEquals(input, EditorUtils.doChangeCase(input, CHANGE_CASE_TO.UPPER, locale, tokenizer));
    assertEquals("Mql5", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.SENTENCE, locale, tokenizer));
    assertEquals("Mql5", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.TITLE, locale, tokenizer));
    assertEquals("mql5", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.CYCLE, locale, tokenizer));
    // Includes OmegaT tag
    input = "<g0>Foo</g0>";
    assertEquals("<g0>foo</g0>", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.LOWER, locale, tokenizer));
    assertEquals("<g0>FOO</g0>", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.UPPER, locale, tokenizer));
    assertEquals(input, EditorUtils.doChangeCase(input, CHANGE_CASE_TO.SENTENCE, locale, tokenizer));
    assertEquals(input, EditorUtils.doChangeCase(input, CHANGE_CASE_TO.TITLE, locale, tokenizer));
    assertEquals("<g0>FOO</g0>", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.CYCLE, locale, tokenizer));
    // Includes surrounding punctuation
    input = "\"Foo, Bar\"";
    assertEquals("\"foo, bar\"", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.LOWER, locale, tokenizer));
    assertEquals("\"FOO, BAR\"", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.UPPER, locale, tokenizer));
    assertEquals("\"Foo, bar\"", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.SENTENCE, locale, tokenizer));
    assertEquals(input, EditorUtils.doChangeCase(input, CHANGE_CASE_TO.TITLE, locale, tokenizer));
    assertEquals("\"FOO, BAR\"", EditorUtils.doChangeCase(input, CHANGE_CASE_TO.CYCLE, locale, tokenizer));
}

Example 9 with ITokenizer

use of org.omegat.tokenizer.ITokenizer in project omegat by omegat-org.

the class EditorUtilsTest method testReplaceGlossaryEntries.

@Test
public void testReplaceGlossaryEntries() {
    List<GlossaryEntry> entries = new ArrayList<GlossaryEntry>();
    entries.add(new GlossaryEntry("snowman", "sneeuwpop", "", false, null));
    entries.add(new GlossaryEntry("Bob", "Blub", "", false, null));
    ITokenizer tokenizer = new LuceneEnglishTokenizer();
    Locale locale = Locale.ENGLISH;
    String srcText = "Snowman Bob went to the snowman party. SnOwMaN!";
    String expected = "Sneeuwpop Blub went to the sneeuwpop party. sneeuwpop!";
    assertEquals(expected, EditorUtils.replaceGlossaryEntries(srcText, entries, locale, tokenizer));
    // Empty cases
    assertNull(EditorUtils.replaceGlossaryEntries(null, entries, locale, tokenizer));
    assertEquals("", EditorUtils.replaceGlossaryEntries("", entries, locale, tokenizer));
    assertSame(srcText, EditorUtils.replaceGlossaryEntries(srcText, null, locale, tokenizer));
    assertSame(srcText, EditorUtils.replaceGlossaryEntries(srcText, new ArrayList<GlossaryEntry>(), locale, tokenizer));
    try {
        EditorUtils.replaceGlossaryEntries(srcText, entries, null, tokenizer);
        fail("Should give NPE when given null locale");
    } catch (NullPointerException ex) {
    }
    try {
        EditorUtils.replaceGlossaryEntries(srcText, entries, locale, null);
        fail("Should give NPE when given null tokenizer");
    } catch (NullPointerException ex) {
    }
    // Multiword entry
    entries.add(0, new GlossaryEntry("snowman party", "sneeuwpop parti", "", false, null));
    srcText = "Snowman Bob went to the snowman party. SnOwMaN!";
    expected = "Sneeuwpop Blub went to the sneeuwpop parti. sneeuwpop!";
    assertEquals(expected, EditorUtils.replaceGlossaryEntries(srcText, entries, locale, tokenizer));
    // Replace final token (see https://sourceforge.net/p/omegat/bugs/819/)
    srcText = "Snowman Bob went to the snowman party. SnOwMaN";
    expected = "Sneeuwpop Blub went to the sneeuwpop parti. sneeuwpop";
    assertEquals(expected, EditorUtils.replaceGlossaryEntries(srcText, entries, locale, tokenizer));
}

Example 10 with ITokenizer

use of org.omegat.tokenizer.ITokenizer in project omegat by omegat-org.

the class GlossaryAutoCompleterViewTest method setUp.

@Before
public final void setUp() {
    Core.setProject(new NotLoadedProject() {

        @Override
        public ITokenizer getTargetTokenizer() {
            return new DefaultTokenizer();
        }

        @Override
        public ProjectProperties getProjectProperties() {
            try {
                return new ProjectProperties() {

                    @Override
                    public Language getTargetLanguage() {
                        return new Language(Locale.ENGLISH);
                    }
                };
            } catch (Exception e) {
                return null;
            }
        }
    });
    TestCoreInitializer.initEditor(new IEditor() {

        @Override
        public String getCurrentFile() {
            return null;
        }

        @Override
        public String getCurrentTargetFile() {
            return null;
        }

        @Override
        public SourceTextEntry getCurrentEntry() {
            return null;
        }

        @Override
        public int getCurrentEntryNumber() {
            return 0;
        }

        @Override
        public void activateEntry() {
        }

        @Override
        public void commitAndDeactivate() {
        }

        @Override
        public void commitAndLeave() {
        }

        @Override
        public void nextEntry() {
        }

        @Override
        public void prevEntry() {
        }

        @Override
        public void nextEntryWithNote() {
        }

        @Override
        public void prevEntryWithNote() {
        }

        @Override
        public void nextUntranslatedEntry() {
        }

        @Override
        public void nextTranslatedEntry() {
        }

        @Override
        public void nextUniqueEntry() {
        }

        @Override
        public void gotoFile(int fileIndex) {
        }

        @Override
        public void gotoEntry(int entryNum) {
        }

        @Override
        public void gotoEntry(int entryNum, CaretPosition pos) {
        }

        @Override
        public void gotoEntry(String srcString, EntryKey key) {
        }

        @Override
        public void gotoEntryAfterFix(int fixedEntry, String fixedSource) {
        }

        @Override
        public void refreshViewAfterFix(List<Integer> fixedEntries) {
        }

        @Override
        public void refreshView(boolean doCommit) {
        }

        @Override
        public void requestFocus() {
        }

        @Override
        public void changeCase(CHANGE_CASE_TO newCase) {
        }

        @Override
        public void replaceEditText(String text) {
        }

        @Override
        public void replaceEditTextAndMark(String text) {
        }

        @Override
        public void insertText(String text) {
        }

        @Override
        public void insertTextAndMark(String text) {
        }

        @Override
        public void insertTag(String tag) {
        }

        @Override
        public void gotoHistoryForward() {
        }

        @Override
        public void gotoHistoryBack() {
        }

        @Override
        public IEditorSettings getSettings() {
            return null;
        }

        @Override
        public void undo() {
        }

        @Override
        public void redo() {
        }

        @Override
        public String getSelectedText() {
            return null;
        }

        @Override
        public void setAlternateTranslationForCurrentEntry(boolean alternate) {
        }

        @Override
        public void markActiveEntrySource(SourceTextEntry requiredActiveEntry, List<Mark> marks, String markerClassName) {
        }

        @Override
        public void registerPopupMenuConstructors(int priority, IPopupMenuConstructor constructor) {
        }

        @Override
        public void remarkOneMarker(String markerClassName) {
        }

        @Override
        public IEditorFilter getFilter() {
            return null;
        }

        @Override
        public void setFilter(IEditorFilter filter) {
        }

        @Override
        public void removeFilter() {
        }

        @Override
        public String getCurrentTranslation() {
            return null;
        }

        @Override
        public void windowDeactivated() {
        }

        @Override
        public void registerUntranslated() {
        }

        @Override
        public void registerEmptyTranslation() {
        }

        @Override
        public void registerIdenticalTranslation() {
        }

        @Override
        public IAutoCompleter getAutoCompleter() {
            return null;
        }
    });
    TestCoreInitializer.initGlossary(new IGlossaries() {

        @Override
        public List<GlossaryEntry> getDisplayedEntries() {
            return currentEntries;
        }

        @Override
        public void showCreateGlossaryEntryDialog(Frame parent) {
        }
    });
    currentEntries.clear();
}