Examples with ArrayDataSerializer org.apache.flink.table.runtime.typeutils.ArrayDataSerializer used on opensource projects

Example 1 with ArrayDataSerializer

use of org.apache.flink.table.runtime.typeutils.ArrayDataSerializer in project flink by apache.

the class ArrowReaderWriterTest method getTestData.

@Override
public RowData[] getTestData() {
    RowData row1 = StreamRecordUtils.row((byte) 1, (short) 2, 3, 4L, true, 1.0f, 1.0, "hello", "hello".getBytes(), DecimalData.fromUnscaledLong(1, 10, 3), 100, 3600000, 3600000, 3600000, 3600000, TimestampData.fromEpochMillis(3600000), TimestampData.fromEpochMillis(3600000), TimestampData.fromEpochMillis(3600000, 100000), TimestampData.fromEpochMillis(3600000, 100000), TimestampData.fromEpochMillis(3600000), TimestampData.fromEpochMillis(3600000), TimestampData.fromEpochMillis(3600000, 100000), TimestampData.fromEpochMillis(3600000, 100000), new GenericArrayData(new StringData[] { StringData.fromString("hello"), StringData.fromString("中文"), null }), GenericRowData.of(1, StringData.fromString("hello"), new GenericArrayData(new StringData[] { StringData.fromString("hello") }), TimestampData.fromEpochMillis(3600000), GenericRowData.of(1, StringData.fromString("hello"))));
    BinaryRowData row2 = StreamRecordUtils.binaryrow((byte) 1, (short) 2, 3, 4L, false, 1.0f, 1.0, "中文", "中文".getBytes(), DecimalData.fromUnscaledLong(1, 10, 3), 100, 3600000, 3600000, 3600000, 3600000, Tuple2.of(TimestampData.fromEpochMillis(3600000), 0), Tuple2.of(TimestampData.fromEpochMillis(3600000), 2), Tuple2.of(TimestampData.fromEpochMillis(3600000, 100000), 4), Tuple2.of(TimestampData.fromEpochMillis(3600000, 100000), 8), Tuple2.of(TimestampData.fromEpochMillis(3600000), 0), Tuple2.of(TimestampData.fromEpochMillis(3600000), 2), Tuple2.of(TimestampData.fromEpochMillis(3600000, 100000), 4), Tuple2.of(TimestampData.fromEpochMillis(3600000, 100000), 8), Tuple2.of(new GenericArrayData(new String[] { null, null, null }), new ArrayDataSerializer(new VarCharType())), Tuple2.of(GenericRowData.of(1, null, new GenericArrayData(new StringData[] { StringData.fromString("hello") }), null, GenericRowData.of(1, StringData.fromString("hello"))), new RowDataSerializer(rowFieldType)));
    RowData row3 = StreamRecordUtils.row(null, (short) 2, 3, 4L, false, 1.0f, 1.0, "中文", "中文".getBytes(), DecimalData.fromUnscaledLong(1, 10, 3), 100, 3600000, 3600000, 3600000, 3600000, TimestampData.fromEpochMillis(3600000), TimestampData.fromEpochMillis(3600000), TimestampData.fromEpochMillis(3600000, 100000), TimestampData.fromEpochMillis(3600000, 100000), TimestampData.fromEpochMillis(3600000), TimestampData.fromEpochMillis(3600000), TimestampData.fromEpochMillis(3600000, 100000), TimestampData.fromEpochMillis(3600000, 100000), new GenericArrayData(new String[] { null, null, null }), GenericRowData.of(1, null, new GenericArrayData(new StringData[] { StringData.fromString("hello") }), null, null));
    BinaryRowData row4 = StreamRecordUtils.binaryrow((byte) 1, null, 3, 4L, true, 1.0f, 1.0, "hello", "hello".getBytes(), DecimalData.fromUnscaledLong(1, 10, 3), 100, 3600000, 3600000, 3600000, 3600000, Tuple2.of(TimestampData.fromEpochMillis(3600000), 0), Tuple2.of(TimestampData.fromEpochMillis(3600000), 2), Tuple2.of(TimestampData.fromEpochMillis(3600000, 100000), 4), Tuple2.of(TimestampData.fromEpochMillis(3600000, 100000), 8), Tuple2.of(TimestampData.fromEpochMillis(3600000), 0), Tuple2.of(TimestampData.fromEpochMillis(3600000), 2), Tuple2.of(TimestampData.fromEpochMillis(3600000, 100000), 4), Tuple2.of(TimestampData.fromEpochMillis(3600000, 100000), 8), Tuple2.of(new GenericArrayData(new StringData[] { StringData.fromString("hello"), StringData.fromString("中文"), null }), new ArrayDataSerializer(new VarCharType())), Tuple2.of(GenericRowData.of(1, null, new GenericArrayData(new StringData[] { StringData.fromString("hello") }), null, null), new RowDataSerializer(rowFieldType)));
    RowData row5 = StreamRecordUtils.row(new Object[fieldTypes.size()]);
    BinaryRowData row6 = StreamRecordUtils.binaryrow(new Object[fieldTypes.size()]);
    return new RowData[] { row1, row2, row3, row4, row5, row6 };
}

Example 2 with ArrayDataSerializer

use of org.apache.flink.table.runtime.typeutils.ArrayDataSerializer in project flink by apache.

the class RowDataTest method getBinaryRow.

private BinaryRowData getBinaryRow() {
    BinaryRowData row = new BinaryRowData(18);
    BinaryRowWriter writer = new BinaryRowWriter(row);
    writer.writeBoolean(0, true);
    writer.writeByte(1, (byte) 1);
    writer.writeShort(2, (short) 2);
    writer.writeInt(3, 3);
    writer.writeLong(4, 4);
    writer.writeFloat(5, 5);
    writer.writeDouble(6, 6);
    writer.writeString(8, str);
    writer.writeRawValue(9, generic, genericSerializer);
    writer.writeDecimal(10, decimal1, 5);
    writer.writeDecimal(11, decimal2, 20);
    writer.writeArray(12, array, new ArrayDataSerializer(DataTypes.INT().getLogicalType()));
    writer.writeMap(13, map, new MapDataSerializer(DataTypes.INT().getLogicalType(), DataTypes.INT().getLogicalType()));
    writer.writeRow(14, underRow, new RowDataSerializer(RowType.of(new IntType(), new IntType())));
    writer.writeBinary(15, bytes);
    writer.writeTimestamp(16, timestamp1, 3);
    writer.writeTimestamp(17, timestamp2, 9);
    return row;
}

Example 3 with ArrayDataSerializer

use of org.apache.flink.table.runtime.typeutils.ArrayDataSerializer in project flink by apache.

the class BinaryArrayDataTest method testArray.

@Test
public void testArray() {
    // 1.array test
    BinaryArrayData array = new BinaryArrayData();
    BinaryArrayWriter writer = new BinaryArrayWriter(array, 3, 4);
    writer.writeInt(0, 6);
    writer.setNullInt(1);
    writer.writeInt(2, 666);
    writer.complete();
    assertEquals(array.getInt(0), 6);
    assertTrue(array.isNullAt(1));
    assertEquals(array.getInt(2), 666);
    // 2.test write to binary row.
    {
        BinaryRowData row2 = new BinaryRowData(1);
        BinaryRowWriter writer2 = new BinaryRowWriter(row2);
        writer2.writeArray(0, array, new ArrayDataSerializer(DataTypes.INT().getLogicalType()));
        writer2.complete();
        BinaryArrayData array2 = (BinaryArrayData) row2.getArray(0);
        assertEquals(array2, array);
        assertEquals(array2.getInt(0), 6);
        assertTrue(array2.isNullAt(1));
        assertEquals(array2.getInt(2), 666);
    }
    // 3.test write var seg array to binary row.
    {
        BinaryArrayData array3 = splitArray(array);
        BinaryRowData row2 = new BinaryRowData(1);
        BinaryRowWriter writer2 = new BinaryRowWriter(row2);
        writer2.writeArray(0, array3, new ArrayDataSerializer(DataTypes.INT().getLogicalType()));
        writer2.complete();
        BinaryArrayData array2 = (BinaryArrayData) row2.getArray(0);
        assertEquals(array2, array);
        assertEquals(array2.getInt(0), 6);
        assertTrue(array2.isNullAt(1));
        assertEquals(array2.getInt(2), 666);
    }
}

Example 4 with ArrayDataSerializer

use of org.apache.flink.table.runtime.typeutils.ArrayDataSerializer in project flink by apache.

the class BinaryArrayDataTest method testArrayTypes.

@Test
public void testArrayTypes() {
    {
        // test bool
        BinaryArrayData array = new BinaryArrayData();
        BinaryArrayWriter writer = new BinaryArrayWriter(array, 2, 1);
        writer.setNullBoolean(0);
        writer.writeBoolean(1, true);
        writer.complete();
        assertTrue(array.isNullAt(0));
        assertTrue(array.getBoolean(1));
        array.setBoolean(0, true);
        assertTrue(array.getBoolean(0));
        array.setNullBoolean(0);
        assertTrue(array.isNullAt(0));
        BinaryArrayData newArray = splitArray(array);
        assertTrue(newArray.isNullAt(0));
        assertTrue(newArray.getBoolean(1));
        newArray.setBoolean(0, true);
        assertTrue(newArray.getBoolean(0));
        newArray.setNullBoolean(0);
        assertTrue(newArray.isNullAt(0));
        newArray.setBoolean(0, true);
        assertEquals(newArray, BinaryArrayData.fromPrimitiveArray(newArray.toBooleanArray()));
    }
    {
        // test byte
        BinaryArrayData array = new BinaryArrayData();
        BinaryArrayWriter writer = new BinaryArrayWriter(array, 2, 1);
        writer.setNullByte(0);
        writer.writeByte(1, (byte) 25);
        writer.complete();
        assertTrue(array.isNullAt(0));
        assertEquals(25, array.getByte(1));
        array.setByte(0, (byte) 5);
        assertEquals(5, array.getByte(0));
        array.setNullByte(0);
        assertTrue(array.isNullAt(0));
        BinaryArrayData newArray = splitArray(array);
        assertTrue(newArray.isNullAt(0));
        assertEquals(25, newArray.getByte(1));
        newArray.setByte(0, (byte) 5);
        assertEquals(5, newArray.getByte(0));
        newArray.setNullByte(0);
        assertTrue(newArray.isNullAt(0));
        newArray.setByte(0, (byte) 3);
        assertEquals(newArray, BinaryArrayData.fromPrimitiveArray(newArray.toByteArray()));
    }
    {
        // test short
        BinaryArrayData array = new BinaryArrayData();
        BinaryArrayWriter writer = new BinaryArrayWriter(array, 2, 2);
        writer.setNullShort(0);
        writer.writeShort(1, (short) 25);
        writer.complete();
        assertTrue(array.isNullAt(0));
        assertEquals(25, array.getShort(1));
        array.setShort(0, (short) 5);
        assertEquals(5, array.getShort(0));
        array.setNullShort(0);
        assertTrue(array.isNullAt(0));
        BinaryArrayData newArray = splitArray(array);
        assertTrue(newArray.isNullAt(0));
        assertEquals(25, newArray.getShort(1));
        newArray.setShort(0, (short) 5);
        assertEquals(5, newArray.getShort(0));
        newArray.setNullShort(0);
        assertTrue(newArray.isNullAt(0));
        newArray.setShort(0, (short) 3);
        assertEquals(newArray, BinaryArrayData.fromPrimitiveArray(newArray.toShortArray()));
    }
    {
        // test int
        BinaryArrayData array = new BinaryArrayData();
        BinaryArrayWriter writer = new BinaryArrayWriter(array, 2, 4);
        writer.setNullInt(0);
        writer.writeInt(1, 25);
        writer.complete();
        assertTrue(array.isNullAt(0));
        assertEquals(25, array.getInt(1));
        array.setInt(0, 5);
        assertEquals(5, array.getInt(0));
        array.setNullInt(0);
        assertTrue(array.isNullAt(0));
        BinaryArrayData newArray = splitArray(array);
        assertTrue(newArray.isNullAt(0));
        assertEquals(25, newArray.getInt(1));
        newArray.setInt(0, 5);
        assertEquals(5, newArray.getInt(0));
        newArray.setNullInt(0);
        assertTrue(newArray.isNullAt(0));
        newArray.setInt(0, 3);
        assertEquals(newArray, BinaryArrayData.fromPrimitiveArray(newArray.toIntArray()));
    }
    {
        // test long
        BinaryArrayData array = new BinaryArrayData();
        BinaryArrayWriter writer = new BinaryArrayWriter(array, 2, 8);
        writer.setNullLong(0);
        writer.writeLong(1, 25);
        writer.complete();
        assertTrue(array.isNullAt(0));
        assertEquals(25, array.getLong(1));
        array.setLong(0, 5);
        assertEquals(5, array.getLong(0));
        array.setNullLong(0);
        assertTrue(array.isNullAt(0));
        BinaryArrayData newArray = splitArray(array);
        assertTrue(newArray.isNullAt(0));
        assertEquals(25, newArray.getLong(1));
        newArray.setLong(0, 5);
        assertEquals(5, newArray.getLong(0));
        newArray.setNullLong(0);
        assertTrue(newArray.isNullAt(0));
        newArray.setLong(0, 3);
        assertEquals(newArray, BinaryArrayData.fromPrimitiveArray(newArray.toLongArray()));
    }
    {
        // test float
        BinaryArrayData array = new BinaryArrayData();
        BinaryArrayWriter writer = new BinaryArrayWriter(array, 2, 4);
        writer.setNullFloat(0);
        writer.writeFloat(1, 25);
        writer.complete();
        assertTrue(array.isNullAt(0));
        assertTrue(25 == array.getFloat(1));
        array.setFloat(0, 5);
        assertTrue(5 == array.getFloat(0));
        array.setNullFloat(0);
        assertTrue(array.isNullAt(0));
        BinaryArrayData newArray = splitArray(array);
        assertTrue(newArray.isNullAt(0));
        assertTrue(25 == newArray.getFloat(1));
        newArray.setFloat(0, 5);
        assertTrue(5 == newArray.getFloat(0));
        newArray.setNullFloat(0);
        assertTrue(newArray.isNullAt(0));
        newArray.setFloat(0, 3);
        assertEquals(newArray, BinaryArrayData.fromPrimitiveArray(newArray.toFloatArray()));
    }
    {
        // test double
        BinaryArrayData array = new BinaryArrayData();
        BinaryArrayWriter writer = new BinaryArrayWriter(array, 2, 8);
        writer.setNullDouble(0);
        writer.writeDouble(1, 25);
        writer.complete();
        assertTrue(array.isNullAt(0));
        assertTrue(25 == array.getDouble(1));
        array.setDouble(0, 5);
        assertTrue(5 == array.getDouble(0));
        array.setNullDouble(0);
        assertTrue(array.isNullAt(0));
        BinaryArrayData newArray = splitArray(array);
        assertTrue(newArray.isNullAt(0));
        assertTrue(25 == newArray.getDouble(1));
        newArray.setDouble(0, 5);
        assertTrue(5 == newArray.getDouble(0));
        newArray.setNullDouble(0);
        assertTrue(newArray.isNullAt(0));
        newArray.setDouble(0, 3);
        assertEquals(newArray, BinaryArrayData.fromPrimitiveArray(newArray.toDoubleArray()));
    }
    {
        // test string
        BinaryArrayData array = new BinaryArrayData();
        BinaryArrayWriter writer = new BinaryArrayWriter(array, 2, 8);
        writer.setNullAt(0);
        writer.writeString(1, fromString("jaja"));
        writer.complete();
        assertTrue(array.isNullAt(0));
        assertEquals(fromString("jaja"), array.getString(1));
        BinaryArrayData newArray = splitArray(array);
        assertTrue(newArray.isNullAt(0));
        assertEquals(fromString("jaja"), newArray.getString(1));
    }
    BinaryArrayData subArray = new BinaryArrayData();
    BinaryArrayWriter subWriter = new BinaryArrayWriter(subArray, 2, 8);
    subWriter.setNullAt(0);
    subWriter.writeString(1, fromString("hehehe"));
    subWriter.complete();
    {
        // test array
        BinaryArrayData array = new BinaryArrayData();
        BinaryArrayWriter writer = new BinaryArrayWriter(array, 2, 8);
        writer.setNullAt(0);
        writer.writeArray(1, subArray, new ArrayDataSerializer(DataTypes.INT().getLogicalType()));
        writer.complete();
        assertTrue(array.isNullAt(0));
        assertEquals(subArray, array.getArray(1));
        BinaryArrayData newArray = splitArray(array);
        assertTrue(newArray.isNullAt(0));
        assertEquals(subArray, newArray.getArray(1));
    }
    {
        // test map
        BinaryArrayData array = new BinaryArrayData();
        BinaryArrayWriter writer = new BinaryArrayWriter(array, 2, 8);
        writer.setNullAt(0);
        writer.writeMap(1, BinaryMapData.valueOf(subArray, subArray), new MapDataSerializer(DataTypes.INT().getLogicalType(), DataTypes.INT().getLogicalType()));
        writer.complete();
        assertTrue(array.isNullAt(0));
        assertEquals(BinaryMapData.valueOf(subArray, subArray), array.getMap(1));
        BinaryArrayData newArray = splitArray(array);
        assertTrue(newArray.isNullAt(0));
        assertEquals(BinaryMapData.valueOf(subArray, subArray), newArray.getMap(1));
    }
}

Example 5 with ArrayDataSerializer

use of org.apache.flink.table.runtime.typeutils.ArrayDataSerializer in project flink by apache.

the class BinaryRowDataTest method testBinaryArray.

@Test
public void testBinaryArray() {
    // 1. array test
    BinaryArrayData array = new BinaryArrayData();
    BinaryArrayWriter arrayWriter = new BinaryArrayWriter(array, 3, BinaryArrayData.calculateFixLengthPartSize(DataTypes.INT().getLogicalType()));
    arrayWriter.writeInt(0, 6);
    arrayWriter.setNullInt(1);
    arrayWriter.writeInt(2, 666);
    arrayWriter.complete();
    assertEquals(array.getInt(0), 6);
    assertTrue(array.isNullAt(1));
    assertEquals(array.getInt(2), 666);
    // 2. test write array to binary row
    BinaryRowData row = new BinaryRowData(1);
    BinaryRowWriter rowWriter = new BinaryRowWriter(row);
    ArrayDataSerializer serializer = new ArrayDataSerializer(DataTypes.INT().getLogicalType());
    rowWriter.writeArray(0, array, serializer);
    rowWriter.complete();
    BinaryArrayData array2 = (BinaryArrayData) row.getArray(0);
    assertEquals(array, array2);
    assertEquals(6, array2.getInt(0));
    assertTrue(array2.isNullAt(1));
    assertEquals(666, array2.getInt(2));
}