Examples with RowTypeInfo org.apache.flink.api.java.typeutils.RowTypeInfo used on opensource projects

Example 41 with RowTypeInfo

use of org.apache.flink.api.java.typeutils.RowTypeInfo in project flink by apache.

the class RowSerializerTest method testRowSerializerWithComplexTypes.

@Test
public void testRowSerializerWithComplexTypes() {
    TypeInformation<Row> typeInfo = new RowTypeInfo(BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.DOUBLE_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO, new TupleTypeInfo<Tuple3<Integer, Boolean, Short>>(BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.BOOLEAN_TYPE_INFO, BasicTypeInfo.SHORT_TYPE_INFO), TypeExtractor.createTypeInfo(MyPojo.class));
    MyPojo testPojo1 = new MyPojo();
    testPojo1.name = null;
    MyPojo testPojo2 = new MyPojo();
    testPojo2.name = "Test1";
    MyPojo testPojo3 = new MyPojo();
    testPojo3.name = "Test2";
    Row[] data = new Row[] { createRow(RowKind.INSERT, null, null, null, null, null), createRow(RowKind.INSERT, 0, null, null, null, null), createRow(RowKind.INSERT, 0, 0.0, null, null, null), createRow(RowKind.INSERT, 0, 0.0, "a", null, null), createRow(RowKind.INSERT, 1, 0.0, "a", null, null), createRow(RowKind.INSERT, 1, 1.0, "a", null, null), createRow(RowKind.INSERT, 1, 1.0, "b", null, null), createRow(RowKind.UPDATE_AFTER, 1, 1.0, "b", new Tuple3<>(1, false, (short) 2), null), createRow(RowKind.UPDATE_AFTER, 1, 1.0, "b", new Tuple3<>(2, false, (short) 2), null), createRow(RowKind.UPDATE_AFTER, 1, 1.0, "b", new Tuple3<>(2, true, (short) 2), null), createRow(RowKind.UPDATE_AFTER, 1, 1.0, "b", new Tuple3<>(2, true, (short) 3), null), createRow(RowKind.DELETE, 1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo1), createRow(RowKind.DELETE, 1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo2), createRow(RowKind.DELETE, 1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo3) };
    TypeSerializer<Row> serializer = typeInfo.createSerializer(new ExecutionConfig());
    RowSerializerTestInstance testInstance = new RowSerializerTestInstance(serializer, data);
    testInstance.testAll();
}

Example 42 with RowTypeInfo

use of org.apache.flink.api.java.typeutils.RowTypeInfo in project flink by apache.

the class RowSerializerTest method testLargeRowSerializer.

@Test
public void testLargeRowSerializer() {
    TypeInformation<Row> typeInfo = new RowTypeInfo(BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    Row row = new Row(13);
    row.setField(0, 2);
    row.setField(1, null);
    row.setField(3, null);
    row.setField(4, null);
    row.setField(5, null);
    row.setField(6, null);
    row.setField(7, null);
    row.setField(8, null);
    row.setField(9, null);
    row.setField(10, null);
    row.setField(11, null);
    row.setField(12, "Test");
    TypeSerializer<Row> serializer = typeInfo.createSerializer(new ExecutionConfig());
    RowSerializerTestInstance testInstance = new RowSerializerTestInstance(serializer, row);
    testInstance.testAll();
}

Example 43 with RowTypeInfo

use of org.apache.flink.api.java.typeutils.RowTypeInfo in project flink by apache.

the class AvroSchemaConverterTest method validateUserSchema.

private void validateUserSchema(TypeInformation<?> actual) {
    final TypeInformation<Row> address = Types.ROW_NAMED(new String[] { "num", "street", "city", "state", "zip" }, Types.INT, Types.STRING, Types.STRING, Types.STRING, Types.STRING);
    final TypeInformation<Row> user = Types.ROW_NAMED(new String[] { "name", "favorite_number", "favorite_color", "type_long_test", "type_double_test", "type_null_test", "type_bool_test", "type_array_string", "type_array_boolean", "type_nullable_array", "type_enum", "type_map", "type_fixed", "type_union", "type_nested", "type_bytes", "type_date", "type_time_millis", "type_time_micros", "type_timestamp_millis", "type_timestamp_micros", "type_decimal_bytes", "type_decimal_fixed" }, Types.STRING, Types.INT, Types.STRING, Types.LONG, Types.DOUBLE, Types.VOID, Types.BOOLEAN, Types.OBJECT_ARRAY(Types.STRING), Types.OBJECT_ARRAY(Types.BOOLEAN), Types.OBJECT_ARRAY(Types.STRING), Types.STRING, Types.MAP(Types.STRING, Types.LONG), Types.PRIMITIVE_ARRAY(Types.BYTE), Types.GENERIC(Object.class), address, Types.PRIMITIVE_ARRAY(Types.BYTE), Types.SQL_DATE, Types.SQL_TIME, Types.SQL_TIME, Types.SQL_TIMESTAMP, Types.SQL_TIMESTAMP, Types.BIG_DEC, Types.BIG_DEC);
    assertEquals(user, actual);
    final RowTypeInfo userRowInfo = (RowTypeInfo) user;
    assertTrue(userRowInfo.schemaEquals(actual));
}

Example 44 with RowTypeInfo

use of org.apache.flink.api.java.typeutils.RowTypeInfo in project flink by apache.

the class PythonBridgeUtils method getPickledBytesFromJavaObject.

public static Object getPickledBytesFromJavaObject(Object obj, TypeInformation<?> dataType) throws IOException {
    Pickler pickler = new Pickler();
    initialize();
    if (obj == null) {
        return new byte[0];
    } else {
        if (dataType instanceof SqlTimeTypeInfo) {
            SqlTimeTypeInfo<?> sqlTimeTypeInfo = SqlTimeTypeInfo.getInfoFor(dataType.getTypeClass());
            if (sqlTimeTypeInfo == DATE) {
                return pickler.dumps(((Date) obj).toLocalDate().toEpochDay());
            } else if (sqlTimeTypeInfo == TIME) {
                return pickler.dumps(((Time) obj).toLocalTime().toNanoOfDay() / 1000);
            }
        } else if (dataType instanceof RowTypeInfo || dataType instanceof TupleTypeInfo) {
            TypeInformation<?>[] fieldTypes = ((TupleTypeInfoBase<?>) dataType).getFieldTypes();
            int arity = dataType instanceof RowTypeInfo ? ((Row) obj).getArity() : ((Tuple) obj).getArity();
            List<Object> fieldBytes = new ArrayList<>(arity + 1);
            if (dataType instanceof RowTypeInfo) {
                fieldBytes.add(new byte[] { ((Row) obj).getKind().toByteValue() });
            }
            for (int i = 0; i < arity; i++) {
                Object field = dataType instanceof RowTypeInfo ? ((Row) obj).getField(i) : ((Tuple) obj).getField(i);
                fieldBytes.add(getPickledBytesFromJavaObject(field, fieldTypes[i]));
            }
            return fieldBytes;
        } else if (dataType instanceof BasicArrayTypeInfo || dataType instanceof PrimitiveArrayTypeInfo) {
            Object[] objects = (Object[]) obj;
            List<Object> serializedElements = new ArrayList<>(objects.length);
            TypeInformation<?> elementType = dataType instanceof BasicArrayTypeInfo ? ((BasicArrayTypeInfo<?, ?>) dataType).getComponentInfo() : ((PrimitiveArrayTypeInfo<?>) dataType).getComponentType();
            for (Object object : objects) {
                serializedElements.add(getPickledBytesFromJavaObject(object, elementType));
            }
            return pickler.dumps(serializedElements);
        } else if (dataType instanceof MapTypeInfo) {
            List<List<Object>> serializedMapKV = new ArrayList<>(2);
            Map<Object, Object> mapObj = (Map) obj;
            List<Object> keyBytesList = new ArrayList<>(mapObj.size());
            List<Object> valueBytesList = new ArrayList<>(mapObj.size());
            for (Map.Entry entry : mapObj.entrySet()) {
                keyBytesList.add(getPickledBytesFromJavaObject(entry.getKey(), ((MapTypeInfo) dataType).getKeyTypeInfo()));
                valueBytesList.add(getPickledBytesFromJavaObject(entry.getValue(), ((MapTypeInfo) dataType).getValueTypeInfo()));
            }
            serializedMapKV.add(keyBytesList);
            serializedMapKV.add(valueBytesList);
            return pickler.dumps(serializedMapKV);
        } else if (dataType instanceof ListTypeInfo) {
            List objects = (List) obj;
            List<Object> serializedElements = new ArrayList<>(objects.size());
            TypeInformation elementType = ((ListTypeInfo) dataType).getElementTypeInfo();
            for (Object object : objects) {
                serializedElements.add(getPickledBytesFromJavaObject(object, elementType));
            }
            return pickler.dumps(serializedElements);
        }
        if (dataType instanceof BasicTypeInfo && BasicTypeInfo.getInfoFor(dataType.getTypeClass()) == FLOAT_TYPE_INFO) {
            // Serialization of float type with pickler loses precision.
            return pickler.dumps(String.valueOf(obj));
        } else if (dataType instanceof PickledByteArrayTypeInfo || dataType instanceof BasicTypeInfo) {
            return pickler.dumps(obj);
        } else {
            // other typeinfos will use the corresponding serializer to serialize data.
            TypeSerializer serializer = dataType.createSerializer(null);
            ByteArrayOutputStreamWithPos baos = new ByteArrayOutputStreamWithPos();
            DataOutputViewStreamWrapper baosWrapper = new DataOutputViewStreamWrapper(baos);
            serializer.serialize(obj, baosWrapper);
            return pickler.dumps(baos.toByteArray());
        }
    }
}

Example 45 with RowTypeInfo

use of org.apache.flink.api.java.typeutils.RowTypeInfo in project flink by apache.

the class PythonKeyedCoProcessOperator method open.

@Override
public void open() throws Exception {
    internalTimerService = getInternalTimerService("user-timers", VoidNamespaceSerializer.INSTANCE, this);
    keyTypeInfo = new RowTypeInfo(((RowTypeInfo) getLeftInputType()).getTypeAt(0));
    keyTypeSerializer = PythonTypeUtils.TypeInfoToSerializerConverter.typeInfoSerializerConverter(keyTypeInfo);
    timerDataTypeInfo = createTimerDataTypeInfo(keyTypeInfo);
    timerDataSerializer = PythonTypeUtils.TypeInfoToSerializerConverter.typeInfoSerializerConverter(timerDataTypeInfo);
    timerHandler = new TimerHandler();
    super.open();
}