Examples with FilteredNormalizer2 android.icu.text.FilteredNormalizer2 used on opensource projects

Example 1 with FilteredNormalizer2

use of android.icu.text.FilteredNormalizer2 in project j2objc by google.

the class BasicTest method TestFilteredAppend.

@Test
public void TestFilteredAppend() {
    Normalizer2 nfcNorm2 = Normalizer2.getNFCInstance();
    UnicodeSet filter = new UnicodeSet("[^\u00a0-\u00ff\u0310-\u031f]");
    FilteredNormalizer2 fn2 = new FilteredNormalizer2(nfcNorm2, filter);
    // Append two strings that each contain a character outside the filter set.
    StringBuilder sb = new StringBuilder("a\u0313a");
    String second = "\u0301\u0313";
    assertEquals("append()", "a\u0313á\u0313", fn2.append(sb, second).toString());
    // Same, and also normalize the second string.
    sb.replace(0, 0x7fffffff, "a\u0313a");
    assertEquals("normalizeSecondAndAppend()", "a\u0313á\u0313", fn2.normalizeSecondAndAppend(sb, second).toString());
    // Normalizer2.normalize(String) uses spanQuickCheckYes() and normalizeSecondAndAppend().
    assertEquals("normalize()", "a\u0313á\u0313", fn2.normalize("a\u0313a\u0301\u0313"));
}

Example 2 with FilteredNormalizer2

use of android.icu.text.FilteredNormalizer2 in project j2objc by google.

the class BasicTest method TestCompare.

@Test
public void TestCompare() {
    // at least as many items as in strings[] !
    String[] s = new String[100];
    int i, j, k, count = strings.length;
    int result, refResult;
    // create the UnicodeStrings
    for (i = 0; i < count; ++i) {
        s[i] = Utility.unescape(strings[i]);
    }
    UTF16.StringComparator comp = new UTF16.StringComparator();
    // test them each with each other
    for (i = 0; i < count; ++i) {
        for (j = i; j < count; ++j) {
            for (k = 0; k < opt.length; ++k) {
                // test Normalizer::compare
                result = norm_compare(s[i], s[j], opt[k].options);
                refResult = ref_norm_compare(s[i], s[j], opt[k].options);
                if (sign(result) != sign(refResult)) {
                    errln("Normalizer::compare( " + i + ", " + j + ", " + k + "( " + opt[k].name + "))=" + result + " should be same sign as " + refResult);
                }
                // test UnicodeString::caseCompare - same internal implementation function
                if (0 != (opt[k].options & Normalizer.COMPARE_IGNORE_CASE)) {
                    // result=s[i]. (s[j], opt[k].options);
                    if ((opt[k].options & Normalizer.FOLD_CASE_EXCLUDE_SPECIAL_I) == 0) {
                        comp.setIgnoreCase(true, UTF16.StringComparator.FOLD_CASE_DEFAULT);
                    } else {
                        comp.setIgnoreCase(true, UTF16.StringComparator.FOLD_CASE_EXCLUDE_SPECIAL_I);
                    }
                    comp.setCodePointCompare((opt[k].options & Normalizer.COMPARE_CODE_POINT_ORDER) != 0);
                    // result=comp.caseCompare(s[i],s[j], opt[k].options);
                    result = comp.compare(s[i], s[j]);
                    refResult = ref_case_compare(s[i], s[j], opt[k].options);
                    if (sign(result) != sign(refResult)) {
                        errln("Normalizer::compare( " + i + ", " + j + ", " + k + "( " + opt[k].name + "))=" + result + " should be same sign as " + refResult);
                    }
                }
            }
        }
    }
    // test cases with i and I to make sure Turkic works
    char[] iI = new char[] { 0x49, 0x69, 0x130, 0x131 };
    UnicodeSet set = new UnicodeSet(), iSet = new UnicodeSet();
    Normalizer2Impl nfcImpl = Norm2AllModes.getNFCInstance().impl;
    nfcImpl.ensureCanonIterData();
    String s1, s2;
    // collect all sets into one for contiguous output
    for (i = 0; i < iI.length; ++i) {
        if (nfcImpl.getCanonStartSet(iI[i], iSet)) {
            set.addAll(iSet);
        }
    }
    // test all of these precomposed characters
    Normalizer2 nfcNorm2 = Normalizer2.getNFCInstance();
    UnicodeSetIterator it = new UnicodeSetIterator(set);
    int c;
    while (it.next() && (c = it.codepoint) != UnicodeSetIterator.IS_STRING) {
        s1 = UTF16.valueOf(c);
        s2 = nfcNorm2.getDecomposition(c);
        for (k = 0; k < opt.length; ++k) {
            // test Normalizer::compare
            result = norm_compare(s1, s2, opt[k].options);
            refResult = ref_norm_compare(s1, s2, opt[k].options);
            if (sign(result) != sign(refResult)) {
                errln("Normalizer.compare(U+" + hex(c) + " with its NFD, " + opt[k].name + ")" + signString(result) + " should be " + signString(refResult));
            }
            // test UnicodeString::caseCompare - same internal implementation function
            if ((opt[k].options & Normalizer.COMPARE_IGNORE_CASE) > 0) {
                if ((opt[k].options & Normalizer.FOLD_CASE_EXCLUDE_SPECIAL_I) == 0) {
                    comp.setIgnoreCase(true, UTF16.StringComparator.FOLD_CASE_DEFAULT);
                } else {
                    comp.setIgnoreCase(true, UTF16.StringComparator.FOLD_CASE_EXCLUDE_SPECIAL_I);
                }
                comp.setCodePointCompare((opt[k].options & Normalizer.COMPARE_CODE_POINT_ORDER) != 0);
                result = comp.compare(s1, s2);
                refResult = ref_case_compare(s1, s2, opt[k].options);
                if (sign(result) != sign(refResult)) {
                    errln("UTF16.compare(U+" + hex(c) + " with its NFD, " + opt[k].name + ")" + signString(result) + " should be " + signString(refResult));
                }
            }
        }
    }
    // test getDecomposition() for some characters that do not decompose
    if (nfcNorm2.getDecomposition(0x20) != null || nfcNorm2.getDecomposition(0x4e00) != null || nfcNorm2.getDecomposition(0x20002) != null) {
        errln("NFC.getDecomposition() returns TRUE for characters which do not have decompositions");
    }
    // test getRawDecomposition() for some characters that do not decompose
    if (nfcNorm2.getRawDecomposition(0x20) != null || nfcNorm2.getRawDecomposition(0x4e00) != null || nfcNorm2.getRawDecomposition(0x20002) != null) {
        errln("getRawDecomposition() returns TRUE for characters which do not have decompositions");
    }
    // test composePair() for some pairs of characters that do not compose
    if (nfcNorm2.composePair(0x20, 0x301) >= 0 || nfcNorm2.composePair(0x61, 0x305) >= 0 || nfcNorm2.composePair(0x1100, 0x1160) >= 0 || nfcNorm2.composePair(0xac00, 0x11a7) >= 0) {
        errln("NFC.composePair() incorrectly composes some pairs of characters");
    }
    // test FilteredNormalizer2.getDecomposition()
    UnicodeSet filter = new UnicodeSet("[^\u00a0-\u00ff]");
    FilteredNormalizer2 fn2 = new FilteredNormalizer2(nfcNorm2, filter);
    if (fn2.getDecomposition(0xe4) != null || !"A\u0304".equals(fn2.getDecomposition(0x100))) {
        errln("FilteredNormalizer2(NFC, ^A0-FF).getDecomposition() failed");
    }
    // test FilteredNormalizer2.getRawDecomposition()
    if (fn2.getRawDecomposition(0xe4) != null || !"A\u0304".equals(fn2.getRawDecomposition(0x100))) {
        errln("FilteredNormalizer2(NFC, ^A0-FF).getRawDecomposition() failed");
    }
    // test FilteredNormalizer2::composePair()
    if (0x100 != fn2.composePair(0x41, 0x304) || // unfiltered result: U+1E08
    fn2.composePair(0xc7, 0x301) >= 0) {
        errln("FilteredNormalizer2(NFC, ^A0-FF).composePair() failed");
    }
}

Example 3 with FilteredNormalizer2

use of android.icu.text.FilteredNormalizer2 in project j2objc by google.

the class BasicTest method TestFilteredNormalizer2.

@Test
public void TestFilteredNormalizer2() {
    Normalizer2 nfcNorm2 = Normalizer2.getNFCInstance();
    UnicodeSet filter = new UnicodeSet("[^\u00a0-\u00ff\u0310-\u031f]");
    FilteredNormalizer2 fn2 = new FilteredNormalizer2(nfcNorm2, filter);
    int c;
    for (c = 0; c <= 0x3ff; ++c) {
        int expectedCC = filter.contains(c) ? nfcNorm2.getCombiningClass(c) : 0;
        int cc = fn2.getCombiningClass(c);
        assertEquals("FilteredNormalizer2(NFC, ^A0-FF,310-31F).getCombiningClass(U+" + hex(c) + ")==filtered NFC.getCC()", expectedCC, cc);
    }
    // More coverage.
    StringBuilder sb = new StringBuilder();
    assertEquals("filtered normalize()", "ää\u0304", fn2.normalize("a\u0308ä\u0304", (Appendable) sb).toString());
    assertTrue("filtered hasBoundaryAfter()", fn2.hasBoundaryAfter('ä'));
    assertTrue("filtered isInert()", fn2.isInert(0x0313));
}