Examples with DefaultTokenizer org.omegat.tokenizer.DefaultTokenizer used on opensource projects

Example 1 with DefaultTokenizer

use of org.omegat.tokenizer.DefaultTokenizer in project omegat by omegat-org.

the class DictionariesManagerTest method setUp.

@Before
public final void setUp() throws Exception {
    manager = new DictionariesManager(new IDictionaries() {

        @Override
        public void refresh() {
        }

        @Override
        public void addDictionaryFactory(IDictionaryFactory factory) {
        }

        @Override
        public void removeDictionaryFactory(IDictionaryFactory factory) {
        }
    }) {

        @Override
        protected boolean doFuzzyMatching() {
            return true;
        }
    };
    manager.setIndexLanguage(new Language(Locale.ENGLISH));
    manager.setTokenizer(new DefaultTokenizer());
    PrintWriter fw = new PrintWriter(IGNORE_FILE, "UTF-8");
    fw.println(IGNORE_WORD);
    fw.close();
    assertFalse(fw.checkError());
}

Example 2 with DefaultTokenizer

use of org.omegat.tokenizer.DefaultTokenizer in project omegat by omegat-org.

the class GlossaryAutoCompleterViewTest method setUp.

@Before
public final void setUp() {
    Core.setProject(new NotLoadedProject() {

        @Override
        public ITokenizer getTargetTokenizer() {
            return new DefaultTokenizer();
        }

        @Override
        public ProjectProperties getProjectProperties() {
            try {
                return new ProjectProperties() {

                    @Override
                    public Language getTargetLanguage() {
                        return new Language(Locale.ENGLISH);
                    }
                };
            } catch (Exception e) {
                return null;
            }
        }
    });
    TestCoreInitializer.initEditor(new IEditor() {

        @Override
        public String getCurrentFile() {
            return null;
        }

        @Override
        public String getCurrentTargetFile() {
            return null;
        }

        @Override
        public SourceTextEntry getCurrentEntry() {
            return null;
        }

        @Override
        public int getCurrentEntryNumber() {
            return 0;
        }

        @Override
        public void activateEntry() {
        }

        @Override
        public void commitAndDeactivate() {
        }

        @Override
        public void commitAndLeave() {
        }

        @Override
        public void nextEntry() {
        }

        @Override
        public void prevEntry() {
        }

        @Override
        public void nextEntryWithNote() {
        }

        @Override
        public void prevEntryWithNote() {
        }

        @Override
        public void nextUntranslatedEntry() {
        }

        @Override
        public void nextTranslatedEntry() {
        }

        @Override
        public void nextUniqueEntry() {
        }

        @Override
        public void gotoFile(int fileIndex) {
        }

        @Override
        public void gotoEntry(int entryNum) {
        }

        @Override
        public void gotoEntry(int entryNum, CaretPosition pos) {
        }

        @Override
        public void gotoEntry(String srcString, EntryKey key) {
        }

        @Override
        public void gotoEntryAfterFix(int fixedEntry, String fixedSource) {
        }

        @Override
        public void refreshViewAfterFix(List<Integer> fixedEntries) {
        }

        @Override
        public void refreshView(boolean doCommit) {
        }

        @Override
        public void requestFocus() {
        }

        @Override
        public void changeCase(CHANGE_CASE_TO newCase) {
        }

        @Override
        public void replaceEditText(String text) {
        }

        @Override
        public void replaceEditTextAndMark(String text) {
        }

        @Override
        public void insertText(String text) {
        }

        @Override
        public void insertTextAndMark(String text) {
        }

        @Override
        public void insertTag(String tag) {
        }

        @Override
        public void gotoHistoryForward() {
        }

        @Override
        public void gotoHistoryBack() {
        }

        @Override
        public IEditorSettings getSettings() {
            return null;
        }

        @Override
        public void undo() {
        }

        @Override
        public void redo() {
        }

        @Override
        public String getSelectedText() {
            return null;
        }

        @Override
        public void setAlternateTranslationForCurrentEntry(boolean alternate) {
        }

        @Override
        public void markActiveEntrySource(SourceTextEntry requiredActiveEntry, List<Mark> marks, String markerClassName) {
        }

        @Override
        public void registerPopupMenuConstructors(int priority, IPopupMenuConstructor constructor) {
        }

        @Override
        public void remarkOneMarker(String markerClassName) {
        }

        @Override
        public IEditorFilter getFilter() {
            return null;
        }

        @Override
        public void setFilter(IEditorFilter filter) {
        }

        @Override
        public void removeFilter() {
        }

        @Override
        public String getCurrentTranslation() {
            return null;
        }

        @Override
        public void windowDeactivated() {
        }

        @Override
        public void registerUntranslated() {
        }

        @Override
        public void registerEmptyTranslation() {
        }

        @Override
        public void registerIdenticalTranslation() {
        }

        @Override
        public IAutoCompleter getAutoCompleter() {
            return null;
        }
    });
    TestCoreInitializer.initGlossary(new IGlossaries() {

        @Override
        public List<GlossaryEntry> getDisplayedEntries() {
            return currentEntries;
        }

        @Override
        public void showCreateGlossaryEntryDialog(Frame parent) {
        }
    });
    currentEntries.clear();
}

Example 3 with DefaultTokenizer

use of org.omegat.tokenizer.DefaultTokenizer in project omegat by omegat-org.

the class MatchesTextAreaTest method testReplaceNumbers.

@Test
public void testReplaceNumbers() {
    ITokenizer tok = new DefaultTokenizer();
    // Simple case
    String source = "chapter 5";
    String srcMatch = "chapter 1";
    String trgMatch = "foo 1";
    assertEquals("foo 5", MatchesTextArea.substituteNumbers(source, srcMatch, trgMatch, tok, tok));
    // Double
    source = "chapter 5.5";
    srcMatch = "chapter 1.1";
    trgMatch = "foo 1.1";
    assertEquals("foo 5.5", MatchesTextArea.substituteNumbers(source, srcMatch, trgMatch, tok, tok));
    // Different order
    source = "hoge 9 fuga 8 piyo 7";
    srcMatch = "foo 1 bar 2 baz 3";
    trgMatch = "bing 3 bang 2 bop 1";
    assertEquals("bing 7 bang 8 bop 9", MatchesTextArea.substituteNumbers(source, srcMatch, trgMatch, tok, tok));
    // User-reported problem string (duplicate numbers)
    source = "Point C : Recommence les \u00E9tapes 16 \u00E0 21 \u2013 pages 16 et 17";
    srcMatch = "Point B : Recommence les \u00E9tapes 9 \u00E0 15 \u2013 page 14 et 15";
    trgMatch = "Point B: Repeat steps 9 to 15 \u2013 pages 14 and 15";
    assertEquals("Point B: Repeat steps 16 to 21 \u2013 pages 16 and 17", MatchesTextArea.substituteNumbers(source, srcMatch, trgMatch, tok, tok));
    // Substitution not possible: differing number counts
    source = "hoge 9 fuga 8 piyo 7";
    srcMatch = "foo 1 bar 2 baz";
    trgMatch = "bing 3 bang 2 bop 1";
    assertEquals("bing 3 bang 2 bop 1", MatchesTextArea.substituteNumbers(source, srcMatch, trgMatch, tok, tok));
    // Substitution not possible: differing number counts
    source = "hoge 9 fuga 8 piyo";
    srcMatch = "foo 1 bar 2 baz 3";
    trgMatch = "bing 3 bang 2 bop 1";
    assertEquals("bing 3 bang 2 bop 1", MatchesTextArea.substituteNumbers(source, srcMatch, trgMatch, tok, tok));
    // Substitution not possible: differing number counts
    source = "hoge 9 fuga 8 piyo 7";
    srcMatch = "foo 1 bar 2 baz 3";
    trgMatch = "bing 3 bang 2 bop";
    assertEquals("bing 3 bang 2 bop", MatchesTextArea.substituteNumbers(source, srcMatch, trgMatch, tok, tok));
    // Substitution not possible: differing number counts
    source = "hoge 9 fuga 8 piyo 7";
    srcMatch = "foo 1 bar 2 baz 3 3";
    trgMatch = "bing 3 bang 2 bop 1";
    assertEquals("bing 3 bang 2 bop 1", MatchesTextArea.substituteNumbers(source, srcMatch, trgMatch, tok, tok));
    // Substitution not possible: differing numbers
    source = "hoge 9 fuga 8 piyo 7";
    srcMatch = "foo 1 bar 2 baz 33";
    trgMatch = "bing 3 bang 2 bop 1";
    assertEquals("bing 3 bang 2 bop 1", MatchesTextArea.substituteNumbers(source, srcMatch, trgMatch, tok, tok));
}