Search in sources :

Example 1 with Pickler

use of net.razorvine.pickle.Pickler in project flink by apache.

the class PythonBridgeUtils method pickleValue.

// This method is reflected from planner
public static byte[] pickleValue(Object value, byte type) {
    Pickler pickler = new Pickler();
    byte[] pickledData;
    try {
        pickledData = pickler.dumps(value);
    } catch (IOException e) {
        throw new RuntimeException("Pickle Java object failed", e);
    }
    byte[] typePickledData = new byte[pickledData.length + 1];
    typePickledData[0] = type;
    System.arraycopy(pickledData, 0, typePickledData, 1, pickledData.length);
    return typePickledData;
}
Also used : IOException(java.io.IOException) Pickler(net.razorvine.pickle.Pickler)

Example 2 with Pickler

use of net.razorvine.pickle.Pickler in project flink by apache.

the class PythonBridgeUtils method getPickledBytesFromJavaObject.

private static Object getPickledBytesFromJavaObject(Object obj, LogicalType dataType) throws IOException {
    Pickler pickler = new Pickler();
    initialize();
    if (obj == null) {
        return new byte[0];
    } else {
        if (dataType instanceof DateType) {
            long time;
            if (obj instanceof LocalDate) {
                time = ((LocalDate) (obj)).toEpochDay();
            } else {
                time = ((Date) obj).toLocalDate().toEpochDay();
            }
            return pickler.dumps(time);
        } else if (dataType instanceof TimeType) {
            long time;
            if (obj instanceof LocalTime) {
                time = ((LocalTime) obj).toNanoOfDay();
            } else {
                time = ((Time) obj).toLocalTime().toNanoOfDay();
            }
            time = time / 1000;
            return pickler.dumps(time);
        } else if (dataType instanceof TimestampType) {
            if (obj instanceof LocalDateTime) {
                return pickler.dumps(Timestamp.valueOf((LocalDateTime) obj));
            } else {
                return pickler.dumps(obj);
            }
        } else if (dataType instanceof RowType) {
            Row tmpRow = (Row) obj;
            LogicalType[] tmpRowFieldTypes = ((RowType) dataType).getChildren().toArray(new LogicalType[0]);
            List<Object> rowFieldBytes = new ArrayList<>(tmpRow.getArity() + 1);
            rowFieldBytes.add(new byte[] { tmpRow.getKind().toByteValue() });
            for (int i = 0; i < tmpRow.getArity(); i++) {
                rowFieldBytes.add(getPickledBytesFromJavaObject(tmpRow.getField(i), tmpRowFieldTypes[i]));
            }
            return rowFieldBytes;
        } else if (dataType instanceof MapType) {
            List<List<Object>> serializedMapKV = new ArrayList<>(2);
            MapType mapType = (MapType) dataType;
            Map<Object, Object> mapObj = (Map) obj;
            List<Object> keyBytesList = new ArrayList<>(mapObj.size());
            List<Object> valueBytesList = new ArrayList<>(mapObj.size());
            for (Map.Entry entry : mapObj.entrySet()) {
                keyBytesList.add(getPickledBytesFromJavaObject(entry.getKey(), mapType.getKeyType()));
                valueBytesList.add(getPickledBytesFromJavaObject(entry.getValue(), mapType.getValueType()));
            }
            serializedMapKV.add(keyBytesList);
            serializedMapKV.add(valueBytesList);
            return pickler.dumps(serializedMapKV);
        } else if (dataType instanceof ArrayType) {
            Object[] objects = (Object[]) obj;
            List<Object> serializedElements = new ArrayList<>(objects.length);
            ArrayType arrayType = (ArrayType) dataType;
            LogicalType elementType = arrayType.getElementType();
            for (Object object : objects) {
                serializedElements.add(getPickledBytesFromJavaObject(object, elementType));
            }
            return pickler.dumps(serializedElements);
        }
        if (dataType instanceof FloatType) {
            return pickler.dumps(String.valueOf(obj));
        } else {
            return pickler.dumps(obj);
        }
    }
}
Also used : LocalDateTime(java.time.LocalDateTime) LocalTime(java.time.LocalTime) ArrayList(java.util.ArrayList) RowType(org.apache.flink.table.types.logical.RowType) LogicalType(org.apache.flink.table.types.logical.LogicalType) LocalDate(java.time.LocalDate) Date(java.sql.Date) LocalDate(java.time.LocalDate) MapType(org.apache.flink.table.types.logical.MapType) TimeType(org.apache.flink.table.types.logical.TimeType) FloatType(org.apache.flink.table.types.logical.FloatType) ArrayType(org.apache.flink.table.types.logical.ArrayType) TimestampType(org.apache.flink.table.types.logical.TimestampType) ArrayList(java.util.ArrayList) LinkedList(java.util.LinkedList) List(java.util.List) Row(org.apache.flink.types.Row) Pickler(net.razorvine.pickle.Pickler) DateType(org.apache.flink.table.types.logical.DateType) Map(java.util.Map)

Example 3 with Pickler

use of net.razorvine.pickle.Pickler in project flink by apache.

the class PythonBridgeUtils method getPickledBytesFromJavaObject.

public static Object getPickledBytesFromJavaObject(Object obj, TypeInformation<?> dataType) throws IOException {
    Pickler pickler = new Pickler();
    initialize();
    if (obj == null) {
        return new byte[0];
    } else {
        if (dataType instanceof SqlTimeTypeInfo) {
            SqlTimeTypeInfo<?> sqlTimeTypeInfo = SqlTimeTypeInfo.getInfoFor(dataType.getTypeClass());
            if (sqlTimeTypeInfo == DATE) {
                return pickler.dumps(((Date) obj).toLocalDate().toEpochDay());
            } else if (sqlTimeTypeInfo == TIME) {
                return pickler.dumps(((Time) obj).toLocalTime().toNanoOfDay() / 1000);
            }
        } else if (dataType instanceof RowTypeInfo || dataType instanceof TupleTypeInfo) {
            TypeInformation<?>[] fieldTypes = ((TupleTypeInfoBase<?>) dataType).getFieldTypes();
            int arity = dataType instanceof RowTypeInfo ? ((Row) obj).getArity() : ((Tuple) obj).getArity();
            List<Object> fieldBytes = new ArrayList<>(arity + 1);
            if (dataType instanceof RowTypeInfo) {
                fieldBytes.add(new byte[] { ((Row) obj).getKind().toByteValue() });
            }
            for (int i = 0; i < arity; i++) {
                Object field = dataType instanceof RowTypeInfo ? ((Row) obj).getField(i) : ((Tuple) obj).getField(i);
                fieldBytes.add(getPickledBytesFromJavaObject(field, fieldTypes[i]));
            }
            return fieldBytes;
        } else if (dataType instanceof BasicArrayTypeInfo || dataType instanceof PrimitiveArrayTypeInfo) {
            Object[] objects = (Object[]) obj;
            List<Object> serializedElements = new ArrayList<>(objects.length);
            TypeInformation<?> elementType = dataType instanceof BasicArrayTypeInfo ? ((BasicArrayTypeInfo<?, ?>) dataType).getComponentInfo() : ((PrimitiveArrayTypeInfo<?>) dataType).getComponentType();
            for (Object object : objects) {
                serializedElements.add(getPickledBytesFromJavaObject(object, elementType));
            }
            return pickler.dumps(serializedElements);
        } else if (dataType instanceof MapTypeInfo) {
            List<List<Object>> serializedMapKV = new ArrayList<>(2);
            Map<Object, Object> mapObj = (Map) obj;
            List<Object> keyBytesList = new ArrayList<>(mapObj.size());
            List<Object> valueBytesList = new ArrayList<>(mapObj.size());
            for (Map.Entry entry : mapObj.entrySet()) {
                keyBytesList.add(getPickledBytesFromJavaObject(entry.getKey(), ((MapTypeInfo) dataType).getKeyTypeInfo()));
                valueBytesList.add(getPickledBytesFromJavaObject(entry.getValue(), ((MapTypeInfo) dataType).getValueTypeInfo()));
            }
            serializedMapKV.add(keyBytesList);
            serializedMapKV.add(valueBytesList);
            return pickler.dumps(serializedMapKV);
        } else if (dataType instanceof ListTypeInfo) {
            List objects = (List) obj;
            List<Object> serializedElements = new ArrayList<>(objects.size());
            TypeInformation elementType = ((ListTypeInfo) dataType).getElementTypeInfo();
            for (Object object : objects) {
                serializedElements.add(getPickledBytesFromJavaObject(object, elementType));
            }
            return pickler.dumps(serializedElements);
        }
        if (dataType instanceof BasicTypeInfo && BasicTypeInfo.getInfoFor(dataType.getTypeClass()) == FLOAT_TYPE_INFO) {
            // Serialization of float type with pickler loses precision.
            return pickler.dumps(String.valueOf(obj));
        } else if (dataType instanceof PickledByteArrayTypeInfo || dataType instanceof BasicTypeInfo) {
            return pickler.dumps(obj);
        } else {
            // other typeinfos will use the corresponding serializer to serialize data.
            TypeSerializer serializer = dataType.createSerializer(null);
            ByteArrayOutputStreamWithPos baos = new ByteArrayOutputStreamWithPos();
            DataOutputViewStreamWrapper baosWrapper = new DataOutputViewStreamWrapper(baos);
            serializer.serialize(obj, baosWrapper);
            return pickler.dumps(baos.toByteArray());
        }
    }
}
Also used : PickledByteArrayTypeInfo(org.apache.flink.streaming.api.typeinfo.python.PickledByteArrayTypeInfo) ArrayList(java.util.ArrayList) Time(java.sql.Time) LocalDateTime(java.time.LocalDateTime) LocalTime(java.time.LocalTime) RowTypeInfo(org.apache.flink.api.java.typeutils.RowTypeInfo) TypeInformation(org.apache.flink.api.common.typeinfo.TypeInformation) TypeSerializer(org.apache.flink.api.common.typeutils.TypeSerializer) BasicTypeInfo(org.apache.flink.api.common.typeinfo.BasicTypeInfo) ArrayList(java.util.ArrayList) LinkedList(java.util.LinkedList) List(java.util.List) ByteArrayOutputStreamWithPos(org.apache.flink.core.memory.ByteArrayOutputStreamWithPos) Date(java.sql.Date) LocalDate(java.time.LocalDate) TupleTypeInfo(org.apache.flink.api.java.typeutils.TupleTypeInfo) SqlTimeTypeInfo(org.apache.flink.api.common.typeinfo.SqlTimeTypeInfo) DataOutputViewStreamWrapper(org.apache.flink.core.memory.DataOutputViewStreamWrapper) ListTypeInfo(org.apache.flink.api.java.typeutils.ListTypeInfo) MapTypeInfo(org.apache.flink.api.java.typeutils.MapTypeInfo) PrimitiveArrayTypeInfo(org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo) Row(org.apache.flink.types.Row) Pickler(net.razorvine.pickle.Pickler) Map(java.util.Map) Tuple(org.apache.flink.api.java.tuple.Tuple) BasicArrayTypeInfo(org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo)

Aggregations

Pickler (net.razorvine.pickle.Pickler)3 Date (java.sql.Date)2 LocalDate (java.time.LocalDate)2 LocalDateTime (java.time.LocalDateTime)2 LocalTime (java.time.LocalTime)2 ArrayList (java.util.ArrayList)2 LinkedList (java.util.LinkedList)2 List (java.util.List)2 Map (java.util.Map)2 Row (org.apache.flink.types.Row)2 IOException (java.io.IOException)1 Time (java.sql.Time)1 BasicArrayTypeInfo (org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo)1 BasicTypeInfo (org.apache.flink.api.common.typeinfo.BasicTypeInfo)1 PrimitiveArrayTypeInfo (org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo)1 SqlTimeTypeInfo (org.apache.flink.api.common.typeinfo.SqlTimeTypeInfo)1 TypeInformation (org.apache.flink.api.common.typeinfo.TypeInformation)1 TypeSerializer (org.apache.flink.api.common.typeutils.TypeSerializer)1 Tuple (org.apache.flink.api.java.tuple.Tuple)1 ListTypeInfo (org.apache.flink.api.java.typeutils.ListTypeInfo)1