Examples with Normalizer android.icu.text.Normalizer used on opensource projects

Example 11 with Normalizer

use of android.icu.text.Normalizer in project j2objc by google.

the class BasicTest method TestPreviousNext.

@Test
public void TestPreviousNext() {
    // src and expect strings
    char[] src = { UTF16.getLeadSurrogate(0x2f999), UTF16.getTrailSurrogate(0x2f999), UTF16.getLeadSurrogate(0x1d15f), UTF16.getTrailSurrogate(0x1d15f), 0xc4, 0x1ed0 };
    int[] expect = { 0x831d, 0x1d158, 0x1d165, 0x41, 0x308, 0x4f, 0x302, 0x301 };
    // expected src indexes corresponding to expect indexes
    int[] expectIndex = { 0, 2, 2, 4, 4, 5, 5, 5, // behind last character
    6 };
    // initial indexes into the src and expect strings
    final int SRC_MIDDLE = 4;
    final int EXPECT_MIDDLE = 3;
    // movement vector
    // - for previous(), 0 for current(), + for next()
    // not const so that we can terminate it below for the error message
    String moves = "0+0+0--0-0-+++0--+++++++0--------";
    // iterators
    Normalizer iter = new Normalizer(new String(src), Normalizer.NFD, 0);
    UCharIterator iter32 = new UCharIterator(expect, expect.length, EXPECT_MIDDLE);
    int c1, c2;
    char m;
    // initially set the indexes into the middle of the strings
    iter.setIndexOnly(SRC_MIDDLE);
    // move around and compare the iteration code points with
    // the expected ones
    int movesIndex = 0;
    while (movesIndex < moves.length()) {
        m = moves.charAt(movesIndex++);
        if (m == '-') {
            c1 = iter.previous();
            c2 = iter32.previous();
        } else if (m == '0') {
            c1 = iter.current();
            c2 = iter32.current();
        } else /* m=='+' */
        {
            c1 = iter.next();
            c2 = iter32.next();
        }
        // compare results
        if (c1 != c2) {
            // copy the moves until the current (m) move, and terminate
            String history = moves.substring(0, movesIndex);
            errln("error: mismatch in Normalizer iteration at " + history + ": " + "got c1= " + hex(c1) + " != expected c2= " + hex(c2));
            break;
        }
        // compare indexes
        if (iter.getIndex() != expectIndex[iter32.getIndex()]) {
            // copy the moves until the current (m) move, and terminate
            String history = moves.substring(0, movesIndex);
            errln("error: index mismatch in Normalizer iteration at " + history + " : " + "Normalizer index " + iter.getIndex() + " expected " + expectIndex[iter32.getIndex()]);
            break;
        }
    }
}

Example 12 with Normalizer

use of android.icu.text.Normalizer in project j2objc by google.

the class BasicTest method TestNormalizerAPI.

// test APIs that are not otherwise used - improve test coverage
@Test
public void TestNormalizerAPI() throws Exception {
    try {
        // instantiate a Normalizer from a CharacterIterator
        String s = Utility.unescape("a\u0308\uac00\\U0002f800");
        // make s a bit longer and more interesting
        UCharacterIterator iter = UCharacterIterator.getInstance(s + s);
        Normalizer norm = new Normalizer(iter, Normalizer.NFC, 0);
        if (norm.next() != 0xe4) {
            errln("error in Normalizer(CharacterIterator).next()");
        }
        // test clone(), ==, and hashCode()
        Normalizer clone = (Normalizer) norm.clone();
        if (clone.equals(norm)) {
            errln("error in Normalizer(Normalizer(CharacterIterator)).clone()!=norm");
        }
        if (clone.getLength() != norm.getLength()) {
            errln("error in Normalizer.getBeginIndex()");
        }
        // }
        if (clone.next() != 0xac00) {
            errln("error in Normalizer(Normalizer(CharacterIterator)).next()");
        }
        int ch = clone.next();
        if (ch != 0x4e3d) {
            errln("error in Normalizer(Normalizer(CharacterIterator)).clone().next()");
        }
        // position changed, must change hashCode()
        if (clone.hashCode() == norm.hashCode()) {
            errln("error in Normalizer(Normalizer(CharacterIterator)).clone().next().hashCode()==copy.hashCode()");
        }
        // test compose() and decompose()
        StringBuffer tel;
        String nfkc, nfkd;
        tel = new StringBuffer("\u2121\u2121\u2121\u2121\u2121\u2121\u2121\u2121\u2121\u2121");
        tel.insert(1, (char) 0x0301);
        nfkc = Normalizer.compose(tel.toString(), true);
        nfkd = Normalizer.decompose(tel.toString(), true);
        if (!nfkc.equals(Utility.unescape("TE\u0139TELTELTELTELTELTELTELTELTEL")) || !nfkd.equals(Utility.unescape("TEL\u0301TELTELTELTELTELTELTELTELTEL"))) {
            errln("error in Normalizer::(de)compose(): wrong result(s)");
        }
        // test setIndex()
        ch = norm.setIndex(3);
        if (ch != 0x4e3d) {
            errln("error in Normalizer(CharacterIterator).setIndex(3)");
        }
        // test setText(CharacterIterator) and getText()
        String out, out2;
        clone.setText(iter);
        out = clone.getText();
        out2 = iter.getText();
        if (!out.equals(out2) || clone.startIndex() != 0 || clone.endIndex() != iter.getLength()) {
            errln("error in Normalizer::setText() or Normalizer::getText()");
        }
        char[] fillIn1 = new char[clone.getLength()];
        char[] fillIn2 = new char[iter.getLength()];
        int len = clone.getText(fillIn1);
        iter.getText(fillIn2, 0);
        if (!Utility.arrayRegionMatches(fillIn1, 0, fillIn2, 0, len)) {
            errln("error in Normalizer.getText(). Normalizer: " + Utility.hex(new String(fillIn1)) + " Iter: " + Utility.hex(new String(fillIn2)));
        }
        clone.setText(fillIn1);
        len = clone.getText(fillIn2);
        if (!Utility.arrayRegionMatches(fillIn1, 0, fillIn2, 0, len)) {
            errln("error in Normalizer.setText() or Normalizer.getText()" + Utility.hex(new String(fillIn1)) + " Iter: " + Utility.hex(new String(fillIn2)));
        }
        // test setText(UChar *), getUMode() and setMode()
        clone.setText(s);
        clone.setIndexOnly(1);
        clone.setMode(Normalizer.NFD);
        if (clone.getMode() != Normalizer.NFD) {
            errln("error in Normalizer::setMode() or Normalizer::getMode()");
        }
        if (clone.next() != 0x308 || clone.next() != 0x1100) {
            errln("error in Normalizer::setText() or Normalizer::setMode()");
        }
        // test last()/previous() with an internal buffer overflow
        StringBuffer buf = new StringBuffer("aaaaaaaaaa");
        buf.setCharAt(10 - 1, '\u0308');
        clone.setText(buf);
        if (clone.last() != 0x308) {
            errln("error in Normalizer(10*U+0308).last()");
        }
        // test UNORM_NONE
        norm.setMode(Normalizer.NONE);
        if (norm.first() != 0x61 || norm.next() != 0x308 || norm.last() != 0x2f800) {
            errln("error in Normalizer(UNORM_NONE).first()/next()/last()");
        }
        out = Normalizer.normalize(s, Normalizer.NONE);
        if (!out.equals(s)) {
            errln("error in Normalizer::normalize(UNORM_NONE)");
        }
        ch = 0x1D15E;
        String exp = "\\U0001D157\\U0001D165";
        String ns = Normalizer.normalize(ch, Normalizer.NFC);
        if (!ns.equals(Utility.unescape(exp))) {
            errln("error in Normalizer.normalize(int,Mode)");
        }
        ns = Normalizer.normalize(ch, Normalizer.NFC, 0);
        if (!ns.equals(Utility.unescape(exp))) {
            errln("error in Normalizer.normalize(int,Mode,int)");
        }
    } catch (Exception e) {
        throw e;
    }
}

Example 13 with Normalizer

use of android.icu.text.Normalizer in project j2objc by google.

the class BasicTest method TestDebugIterOld.

@Test
public void TestDebugIterOld() {
    String input = "\\U0001D15E";
    String expected = "\uD834\uDD57\uD834\uDD65";
    String expectedReverse = "\uD834\uDD65\uD834\uDD57";
    int index = 0;
    int ch;
    Normalizer iter = new Normalizer(new StringCharacterIterator(Utility.unescape(input)), Normalizer.NFKC, 0);
    StringBuffer got = new StringBuffer();
    for (ch = iter.first(); ch != Normalizer.DONE; ch = iter.next()) {
        if (index >= expected.length()) {
            errln("FAIL: " + "Unexpected character '" + (char) ch + "' (" + hex(ch) + ")" + " at index " + index);
            break;
        }
        got.append(UCharacter.toString(ch));
        index++;
    }
    if (!expected.equals(got.toString())) {
        errln("FAIL: " + "got '" + got + "' (" + hex(got) + ")" + " but expected '" + expected + "' (" + hex(expected) + ")");
    }
    if (got.length() < expected.length()) {
        errln("FAIL: " + "Only got " + index + " chars, expected " + expected.length());
    }
    logln("Reverse Iteration\n");
    iter.setIndexOnly(iter.endIndex());
    got.setLength(0);
    for (ch = iter.previous(); ch != Normalizer.DONE; ch = iter.previous()) {
        if (index >= expected.length()) {
            errln("FAIL: " + "Unexpected character '" + (char) ch + "' (" + hex(ch) + ")" + " at index " + index);
            break;
        }
        got.append(UCharacter.toString(ch));
    }
    if (!expectedReverse.equals(got.toString())) {
        errln("FAIL: " + "got '" + got + "' (" + hex(got) + ")" + " but expected '" + expected + "' (" + hex(expected) + ")");
    }
    if (got.length() < expected.length()) {
        errln("FAIL: " + "Only got " + index + " chars, expected " + expected.length());
    }
}

Example 14 with Normalizer

use of android.icu.text.Normalizer in project j2objc by google.

the class BasicTest method TestBugJ2068.

@Test
public void TestBugJ2068() {
    String sample = "The quick brown fox jumped over the lazy dog";
    UCharacterIterator text = UCharacterIterator.getInstance(sample);
    Normalizer norm = new Normalizer(text, Normalizer.NFC, 0);
    text.setIndex(4);
    if (text.current() == norm.current()) {
        errln("Normalizer is not cloning the UCharacterIterator");
    }
}