Examples with Encoding org.graalvm.shadowed.org.jcodings.Encoding used on opensource projects

Example 1 with Encoding

use of org.graalvm.shadowed.org.jcodings.Encoding in project graal by oracle.

the class TruffleStringEncodingsEnumGenerator method printInitializerExotic.

private static int printInitializerExotic(Predicate<Encoding> filter, int iParam) {
    Encoding[] filtered = StreamSupport.stream(J_CODINGS_MAP.getValues().spliterator(), false).filter(filter).sorted(Comparator.comparing(a -> toEnumName(a.toString()))).toArray(Encoding[]::new);
    int i = iParam;
    for (Encoding e : filtered) {
        // Checkstyle: stop
        System.out.printf("/**\n * %s.\n *\n * @since 22.0\n */\n", toEnumName(e.toString()));
        // Checkstyle: resume
        printInitializer(e, i++);
    }
    return i;
}

Example 2 with Encoding

use of org.graalvm.shadowed.org.jcodings.Encoding in project graal by oracle.

the class TStringConstructorTests method testFromByteArray.

@Test
public void testFromByteArray() throws Exception {
    forAllEncodings((TruffleString.Encoding encoding) -> {
        if (isAsciiCompatible(encoding)) {
            byte[] ascii = new byte[128 << getStride(encoding)];
            for (int i = 0; i < 128; i++) {
                TStringTestUtil.writeValue(ascii, getStride(encoding), i, i);
            }
            for (boolean copy : new boolean[] { true, false }) {
                TruffleString s = fromByteArrayUncached(ascii, 0, ascii.length, encoding, copy);
                int readByteOffset = 0;
                if (isUTF32(encoding) && ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN) {
                    readByteOffset = 3;
                }
                if (isUTF16(encoding) && ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN) {
                    readByteOffset = 1;
                }
                TruffleStringIterator it = s.createCodePointIteratorUncached(encoding);
                for (int i = 0; i < 128; i++) {
                    Assert.assertEquals(i, s.readByteUncached(byteIndex(i, encoding) + readByteOffset, encoding));
                    Assert.assertEquals(i, s.codePointAtByteIndexUncached(byteIndex(i, encoding), encoding));
                    Assert.assertEquals(i, s.codePointAtIndexUncached(i, encoding));
                    Assert.assertTrue(it.hasNext());
                    Assert.assertEquals(i, it.nextUncached());
                    Assert.assertEquals(i, s.indexOfCodePointUncached(i, 0, 128, encoding));
                    Assert.assertEquals(i, s.indexOfStringUncached(fromCodePointUncached(i, encoding), 0, 128, encoding));
                }
            }
        }
    });
    forAllEncodingsAndCodePointLists((TruffleString.Encoding encoding, int[] codepointArray) -> {
        final int[][] cps;
        if (isUTF16(encoding) || isUTF32(encoding)) {
            cps = new int[][] { new int[] { 0x00, 0x7f }, new int[] { 0x00, 0x7f, 0xff }, new int[] { 0x00, 0x7f, 0xff, 0x100, 0xffff }, codepointArray };
        } else {
            cps = new int[][] { codepointArray };
        }
        for (int[] codepoints : cps) {
            TruffleStringBuilder sbBytes = TruffleStringBuilder.create(encoding);
            TruffleStringBuilder sbCP = TruffleStringBuilder.create(encoding);
            TruffleStringBuilder sbCPStrings = TruffleStringBuilder.create(encoding);
            Encoding jCoding = Encodings.getJCoding(encoding);
            int byteLength = 0;
            int[] byteIndices = new int[codepoints.length];
            for (int i = 0; i < codepoints.length; i++) {
                byteIndices[i] = byteLength;
                byteLength += jCoding.codeToMbcLength(codepoints[i]);
            }
            byte[] array = new byte[byteLength];
            for (int i = 0; i < codepoints.length; i++) {
                jCoding.codeToMbc(codepoints[i], array, byteIndices[i]);
                sbCP.appendCodePointUncached(codepoints[i]);
                sbCPStrings.appendStringUncached(fromCodePointUncached(codepoints[i], encoding));
            }
            if (isUTF32(encoding)) {
                for (int cp : codepoints) {
                    sbBytes.appendCodePointUncached(cp);
                }
            } else if (isUTF16(encoding)) {
                for (int i = 0; i < byteLength / 2; i++) {
                    sbBytes.appendCharUTF16Uncached((char) TStringTestUtil.readValue(array, 1, i));
                }
            } else {
                for (byte b : array) {
                    sbBytes.appendByteUncached(b);
                }
            }
            AbstractTruffleString[] strings = new AbstractTruffleString[isUTF32(encoding) || isUTF16(encoding) ? 9 : 8];
            strings[0] = fromByteArrayUncached(array, 0, array.length, encoding, true);
            strings[1] = fromByteArrayUncached(array, 0, array.length, encoding, false);
            strings[2] = fromNativePointerUncached(PointerObject.create(array), 0, array.length, encoding, false);
            strings[3] = fromNativePointerUncached(PointerObject.create(array), 0, array.length, encoding, true);
            strings[4] = MutableTruffleString.fromByteArrayUncached(array, 0, array.length, encoding, true);
            strings[5] = MutableTruffleString.fromNativePointerUncached(PointerObject.create(array), 0, array.length, encoding, false);
            strings[6] = strings[2].asMutableTruffleStringUncached(encoding);
            strings[7] = strings[4].asTruffleStringUncached(encoding);
            if (isUTF16(encoding)) {
                char[] charArray = new char[array.length / 2];
                for (int i = 0; i < charArray.length; i++) {
                    charArray[i] = (char) TStringTestUtil.readValue(array, 1, i);
                }
                strings[8] = fromCharArrayUTF16Uncached(charArray, 0, charArray.length);
            } else if (isUTF32(encoding)) {
                strings[8] = fromIntArrayUTF32Uncached(codepoints, 0, codepoints.length);
            }
            for (AbstractTruffleString s : strings) {
                for (TruffleStringBuilder sb : new TruffleStringBuilder[] { sbBytes, sbCP, sbCPStrings }) {
                    TruffleString sbs = sb.toStringUncached();
                    Assert.assertTrue(sbs.equalsUncached(s, encoding));
                    Assert.assertEquals(sbs, s);
                    Assert.assertEquals(sbs.hashCode(), s.hashCode());
                    Assert.assertEquals(sbs.hashCodeUncached(encoding), s.hashCodeUncached(encoding));
                    Assert.assertEquals(sbs.getCodeRangeUncached(encoding), s.getCodeRangeUncached(encoding));
                }
                TruffleStringBuilder sbCMP = TruffleStringBuilder.create(encoding);
                for (int i = 0; i < codepoints.length - 1; i++) {
                    sbCMP.appendCodePointUncached(codepoints[i]);
                }
                sbCMP.appendCodePointUncached(codepoints[codepoints.length - 1] - 1);
                TruffleString cmp = sbCMP.toStringUncached();
                Assert.assertEquals(0, s.compareBytesUncached(s, encoding));
                Assert.assertTrue(s.compareBytesUncached(cmp, encoding) > 0);
                Assert.assertTrue(cmp.compareBytesUncached(s, encoding) < 0);
                s.toJavaStringUncached();
                TruffleStringIterator it = s.createCodePointIteratorUncached(encoding);
                Assert.assertEquals(codepoints.length, s.codePointLengthUncached(encoding));
                Assert.assertTrue(s.isValidUncached(encoding));
                for (int i = 0; i < array.length; i++) {
                    Assert.assertEquals(Byte.toUnsignedInt(array[i]), s.readByteUncached(i, encoding));
                }
                checkInternalByteArrayEquals(array, s.getInternalByteArrayUncached(encoding));
                byte[] copy = new byte[array.length];
                s.copyToByteArrayNodeUncached(0, copy, 0, copy.length, encoding);
                Assert.assertArrayEquals(array, copy);
                PointerObject pointerObject = PointerObject.create(array.length);
                s.copyToNativeMemoryNodeUncached(0, pointerObject, 0, array.length, encoding);
                Assert.assertTrue(pointerObject.contentEquals(array));
                for (int i = 0; i < codepoints.length; i++) {
                    Assert.assertEquals(codepoints[i], s.codePointAtIndexUncached(i, encoding));
                    Assert.assertEquals(codepoints[i], s.codePointAtByteIndexUncached(byteIndices[i], encoding));
                    Assert.assertEquals(i, s.indexOfCodePointUncached(codepoints[i], 0, codepoints.length, encoding));
                    Assert.assertEquals(byteIndices[i], s.byteIndexOfCodePointUncached(codepoints[i], 0, byteLength, encoding));
                    Assert.assertEquals(i, s.lastIndexOfCodePointUncached(codepoints[i], codepoints.length, 0, encoding));
                    Assert.assertEquals(byteIndices[i], s.lastByteIndexOfCodePointUncached(codepoints[i], byteLength, 0, encoding));
                    TruffleString s1 = fromByteArrayUncached(array, byteIndices[i], (i + 1 < codepoints.length ? byteIndices[i + 1] : array.length) - byteIndices[i], encoding, true);
                    TruffleString s2 = fromByteArrayUncached(array, byteIndices[i], (i + 2 < codepoints.length ? byteIndices[i + 2] : array.length) - byteIndices[i], encoding, false);
                    TruffleString s3 = fromByteArrayUncached(array, byteIndices[i], (i + 3 < codepoints.length ? byteIndices[i + 3] : array.length) - byteIndices[i], encoding, false);
                    s1.toJavaStringUncached();
                    s2.toJavaStringUncached();
                    s3.toJavaStringUncached();
                    for (TruffleString substring : new TruffleString[] { s1, s2, s3 }) {
                        Assert.assertEquals(i, s.indexOfStringUncached(substring, 0, codepoints.length, encoding));
                        Assert.assertEquals(byteIndices[i], s.byteIndexOfStringUncached(substring, 0, byteLength, encoding));
                        Assert.assertEquals(i, s.lastIndexOfStringUncached(substring, codepoints.length, 0, encoding));
                        Assert.assertEquals(byteIndices[i], s.lastByteIndexOfStringUncached(substring, byteLength, 0, encoding));
                        Assert.assertTrue(s.regionEqualsUncached(i, substring, 0, substring.codePointLengthUncached(encoding), encoding));
                        Assert.assertTrue(s.regionEqualByteIndexUncached(byteIndices[i], substring, 0, substring.byteLength(encoding), encoding));
                    }
                    Assert.assertTrue(it.hasNext());
                    Assert.assertEquals(codepoints[i], it.nextUncached());
                }
            }
        }
    });
}

Example 3 with Encoding

use of org.graalvm.shadowed.org.jcodings.Encoding in project graal by oracle.

the class Encodings method main.

/**
 * Generate unit test character tables for all encodings. Invoke with the following mx command:
 *
 * <pre>
 * mx java -cp `mx paths truffle:TRUFFLE_API`:`mx paths truffle:TRUFFLE_TEST`:`mx paths sdk:GRAAL_SDK` com.oracle.truffle.api.strings.test.Encodings
 * </pre>
 */
public static void main(String[] args) {
    byte[] bytesAscii = { 0x00, 0x7f };
    byte[] bytesLatin = { 0x00, (byte) 0xff };
    int[] indices01 = { 0, 1 };
    int[] codepointsUTF8 = { 0x000000, 0x00007f, 0x000080, 0x00d7ff, 0x00e000, 0x10ffff };
    int[] codepointsUTF16 = { 0x000000, 0x00007f, 0x000080, 0x0000ff, 0x000100, 0x00d7ff, 0x00e000, 0x00ffff, 0x010000, 0x10ffff };
    TestData[] testData = new TestData[TruffleString.Encoding.values().length];
    testData[TruffleString.Encoding.US_ASCII.ordinal()] = new TestData(new int[] { 0x00, 0x7f }, indices01, bytesAscii, null, null, bytesAscii, new byte[] { (byte) 0xff }, new int[] { 0xff });
    testData[TruffleString.Encoding.ISO_8859_1.ordinal()] = new TestData(new int[] { 0x00, 0xff }, new int[] { 0, 1 }, bytesAscii, bytesLatin, null, bytesLatin, null, null);
    testData[TruffleString.Encoding.UTF_8.ordinal()] = new TestData(codepointsUTF8, codePointByteIndices(codepointsUTF8, getJCoding(TruffleString.Encoding.UTF_8)), bytesAscii, null, null, encodeCodePoints(codepointsUTF8, getJCoding(TruffleString.Encoding.UTF_8)), new byte[] { (byte) 0xc0 }, new int[] { 0xfffd });
    testData[TruffleString.Encoding.UTF_16.ordinal()] = new TestData(codepointsUTF16, codePointByteIndices(codepointsUTF16, getJCoding(TruffleString.Encoding.UTF_16)), asBytes(new int[] { 0x00, 0x7f }, 1), asBytes(new int[] { 0x00, 0xff }, 1), asBytes(new int[] { 0x0000, 0xffff }, 1), encodeCodePoints(codepointsUTF16, getJCoding(TruffleString.Encoding.UTF_16)), asBytes(new int[] { Character.MIN_LOW_SURROGATE }, 1), new int[] { Character.MIN_LOW_SURROGATE });
    testData[TruffleString.Encoding.UTF_32.ordinal()] = new TestData(codepointsUTF16, codePointByteIndices(codepointsUTF16, getJCoding(TruffleString.Encoding.UTF_32)), asBytes(new int[] { 0x00, 0x7f }, 2), asBytes(new int[] { 0x00, 0xff }, 2), asBytes(new int[] { 0x0000, 0xffff }, 2), asBytes(codepointsUTF16, 2), asBytes(new int[] { Character.MIN_LOW_SURROGATE }, 2), new int[] { Character.MIN_LOW_SURROGATE });
    testData[TruffleString.Encoding.BYTES.ordinal()] = new TestData(new int[] { 0x00, 0x7f, 0x80, 0xff }, new int[] { 0, 1, 2, 3 }, null, null, null, new byte[] { 0x00, 0x7f, (byte) 0x80, (byte) 0xff }, null, null);
    for (TruffleString.Encoding e : TruffleString.Encoding.values()) {
        Encoding jCoding = getJCoding(e);
        if (testData[e.ordinal()] == null) {
            int iBuf = 0;
            int[] buf = new int[16];
            int lo = -1;
            for (int i = 0; i < 0x10ffff; i++) {
                if (isValidCodePoint(i, jCoding)) {
                    if (lo < 0) {
                        lo = i;
                    }
                } else {
                    if (lo >= 0) {
                        if (iBuf + 2 >= buf.length) {
                            buf = Arrays.copyOf(buf, buf.length * 2);
                        }
                        buf[iBuf++] = lo;
                        buf[iBuf++] = i - 1;
                        lo = -1;
                    }
                }
            }
            if (lo >= 0) {
                if (iBuf + 2 >= buf.length) {
                    buf = Arrays.copyOf(buf, buf.length * 2);
                }
                buf[iBuf++] = lo;
                buf[iBuf++] = 0x10ffff;
            }
            int[] codepoints = Arrays.copyOf(buf, iBuf);
            testData[e.ordinal()] = new TestData(codepoints, codePointByteIndices(codepoints, jCoding), jCoding.isAsciiCompatible() ? bytesAscii : null, null, null, encodeCodePoints(codepoints, jCoding), null, null);
        }
    }
    // Checkstyle: stop
    System.out.println("static final TestData[] TEST_DATA = {");
    for (TruffleString.Encoding e : TruffleString.Encoding.values()) {
        System.out.println("data" + e.name().replace("_", "") + "(),");
    }
    System.out.println("};");
    for (TruffleString.Encoding e : TruffleString.Encoding.values()) {
        System.out.println("static TestData data" + e.name().replace("_", "") + "() {");
        System.out.println("assert TruffleString.Encoding." + e.name() + ".ordinal() == " + e.ordinal() + ";");
        System.out.println("return " + testData[e.ordinal()].serialize() + ";");
        System.out.println("}");
    }
// Checkstyle: resume
}

Example 4 with Encoding

use of org.graalvm.shadowed.org.jcodings.Encoding in project graal by oracle.

the class Encodings method createJCodingsMap.

private static EconomicMap<String, Encoding> createJCodingsMap() {
    CaseInsensitiveBytesHash<EncodingDB.Entry> encodings = EncodingDB.getEncodings();
    if (encodings.size() > MAX_J_CODINGS_INDEX_VALUE) {
        throw new RuntimeException(String.format("Assumption broken: org.graalvm.shadowed.org.jcodings has more than %d encodings (actual: %d)!", MAX_J_CODINGS_INDEX_VALUE, encodings.size()));
    }
    EconomicMap<String, Encoding> allEncodings = EconomicMap.create(encodings.size());
    for (EncodingDB.Entry entry : encodings) {
        Encoding enc = entry.getEncoding();
        int i = enc.getIndex();
        if (i < 0 || i >= encodings.size()) {
            throw new RuntimeException(String.format("Assumption broken: index of org.graalvm.shadowed.org.jcodings encoding \"%s\" is greater than number of encodings (index: %d, number of encodings: %d)!", enc, i, encodings.size()));
        }
        allEncodings.put(toEnumName(enc.toString()), enc);
    }
    return allEncodings;
}

Example 5 with Encoding

use of org.graalvm.shadowed.org.jcodings.Encoding in project graal by oracle.

the class TStringConstructorTests method testFromCodePoint.

@Test
public void testFromCodePoint() throws Exception {
    forAllEncodingsAndCodePoints((TruffleString.Encoding encoding, int codepoint) -> {
        TruffleString s = fromCodePointUncached(codepoint, encoding);
        Assert.assertEquals(codepoint, s.codePointAtIndexUncached(0, encoding));
        Assert.assertEquals(codepoint, s.codePointAtByteIndexUncached(0, encoding));
        Assert.assertTrue(s.isValidUncached(encoding));
    });
}