use of net.razorvine.pickle.Pickler in project flink by apache.
the class PythonBridgeUtils method pickleValue.
// This method is reflected from planner
public static byte[] pickleValue(Object value, byte type) {
Pickler pickler = new Pickler();
byte[] pickledData;
try {
pickledData = pickler.dumps(value);
} catch (IOException e) {
throw new RuntimeException("Pickle Java object failed", e);
}
byte[] typePickledData = new byte[pickledData.length + 1];
typePickledData[0] = type;
System.arraycopy(pickledData, 0, typePickledData, 1, pickledData.length);
return typePickledData;
}
use of net.razorvine.pickle.Pickler in project flink by apache.
the class PythonBridgeUtils method getPickledBytesFromJavaObject.
private static Object getPickledBytesFromJavaObject(Object obj, LogicalType dataType) throws IOException {
Pickler pickler = new Pickler();
initialize();
if (obj == null) {
return new byte[0];
} else {
if (dataType instanceof DateType) {
long time;
if (obj instanceof LocalDate) {
time = ((LocalDate) (obj)).toEpochDay();
} else {
time = ((Date) obj).toLocalDate().toEpochDay();
}
return pickler.dumps(time);
} else if (dataType instanceof TimeType) {
long time;
if (obj instanceof LocalTime) {
time = ((LocalTime) obj).toNanoOfDay();
} else {
time = ((Time) obj).toLocalTime().toNanoOfDay();
}
time = time / 1000;
return pickler.dumps(time);
} else if (dataType instanceof TimestampType) {
if (obj instanceof LocalDateTime) {
return pickler.dumps(Timestamp.valueOf((LocalDateTime) obj));
} else {
return pickler.dumps(obj);
}
} else if (dataType instanceof RowType) {
Row tmpRow = (Row) obj;
LogicalType[] tmpRowFieldTypes = ((RowType) dataType).getChildren().toArray(new LogicalType[0]);
List<Object> rowFieldBytes = new ArrayList<>(tmpRow.getArity() + 1);
rowFieldBytes.add(new byte[] { tmpRow.getKind().toByteValue() });
for (int i = 0; i < tmpRow.getArity(); i++) {
rowFieldBytes.add(getPickledBytesFromJavaObject(tmpRow.getField(i), tmpRowFieldTypes[i]));
}
return rowFieldBytes;
} else if (dataType instanceof MapType) {
List<List<Object>> serializedMapKV = new ArrayList<>(2);
MapType mapType = (MapType) dataType;
Map<Object, Object> mapObj = (Map) obj;
List<Object> keyBytesList = new ArrayList<>(mapObj.size());
List<Object> valueBytesList = new ArrayList<>(mapObj.size());
for (Map.Entry entry : mapObj.entrySet()) {
keyBytesList.add(getPickledBytesFromJavaObject(entry.getKey(), mapType.getKeyType()));
valueBytesList.add(getPickledBytesFromJavaObject(entry.getValue(), mapType.getValueType()));
}
serializedMapKV.add(keyBytesList);
serializedMapKV.add(valueBytesList);
return pickler.dumps(serializedMapKV);
} else if (dataType instanceof ArrayType) {
Object[] objects = (Object[]) obj;
List<Object> serializedElements = new ArrayList<>(objects.length);
ArrayType arrayType = (ArrayType) dataType;
LogicalType elementType = arrayType.getElementType();
for (Object object : objects) {
serializedElements.add(getPickledBytesFromJavaObject(object, elementType));
}
return pickler.dumps(serializedElements);
}
if (dataType instanceof FloatType) {
return pickler.dumps(String.valueOf(obj));
} else {
return pickler.dumps(obj);
}
}
}
use of net.razorvine.pickle.Pickler in project flink by apache.
the class PythonBridgeUtils method getPickledBytesFromJavaObject.
public static Object getPickledBytesFromJavaObject(Object obj, TypeInformation<?> dataType) throws IOException {
Pickler pickler = new Pickler();
initialize();
if (obj == null) {
return new byte[0];
} else {
if (dataType instanceof SqlTimeTypeInfo) {
SqlTimeTypeInfo<?> sqlTimeTypeInfo = SqlTimeTypeInfo.getInfoFor(dataType.getTypeClass());
if (sqlTimeTypeInfo == DATE) {
return pickler.dumps(((Date) obj).toLocalDate().toEpochDay());
} else if (sqlTimeTypeInfo == TIME) {
return pickler.dumps(((Time) obj).toLocalTime().toNanoOfDay() / 1000);
}
} else if (dataType instanceof RowTypeInfo || dataType instanceof TupleTypeInfo) {
TypeInformation<?>[] fieldTypes = ((TupleTypeInfoBase<?>) dataType).getFieldTypes();
int arity = dataType instanceof RowTypeInfo ? ((Row) obj).getArity() : ((Tuple) obj).getArity();
List<Object> fieldBytes = new ArrayList<>(arity + 1);
if (dataType instanceof RowTypeInfo) {
fieldBytes.add(new byte[] { ((Row) obj).getKind().toByteValue() });
}
for (int i = 0; i < arity; i++) {
Object field = dataType instanceof RowTypeInfo ? ((Row) obj).getField(i) : ((Tuple) obj).getField(i);
fieldBytes.add(getPickledBytesFromJavaObject(field, fieldTypes[i]));
}
return fieldBytes;
} else if (dataType instanceof BasicArrayTypeInfo || dataType instanceof PrimitiveArrayTypeInfo) {
Object[] objects = (Object[]) obj;
List<Object> serializedElements = new ArrayList<>(objects.length);
TypeInformation<?> elementType = dataType instanceof BasicArrayTypeInfo ? ((BasicArrayTypeInfo<?, ?>) dataType).getComponentInfo() : ((PrimitiveArrayTypeInfo<?>) dataType).getComponentType();
for (Object object : objects) {
serializedElements.add(getPickledBytesFromJavaObject(object, elementType));
}
return pickler.dumps(serializedElements);
} else if (dataType instanceof MapTypeInfo) {
List<List<Object>> serializedMapKV = new ArrayList<>(2);
Map<Object, Object> mapObj = (Map) obj;
List<Object> keyBytesList = new ArrayList<>(mapObj.size());
List<Object> valueBytesList = new ArrayList<>(mapObj.size());
for (Map.Entry entry : mapObj.entrySet()) {
keyBytesList.add(getPickledBytesFromJavaObject(entry.getKey(), ((MapTypeInfo) dataType).getKeyTypeInfo()));
valueBytesList.add(getPickledBytesFromJavaObject(entry.getValue(), ((MapTypeInfo) dataType).getValueTypeInfo()));
}
serializedMapKV.add(keyBytesList);
serializedMapKV.add(valueBytesList);
return pickler.dumps(serializedMapKV);
} else if (dataType instanceof ListTypeInfo) {
List objects = (List) obj;
List<Object> serializedElements = new ArrayList<>(objects.size());
TypeInformation elementType = ((ListTypeInfo) dataType).getElementTypeInfo();
for (Object object : objects) {
serializedElements.add(getPickledBytesFromJavaObject(object, elementType));
}
return pickler.dumps(serializedElements);
}
if (dataType instanceof BasicTypeInfo && BasicTypeInfo.getInfoFor(dataType.getTypeClass()) == FLOAT_TYPE_INFO) {
// Serialization of float type with pickler loses precision.
return pickler.dumps(String.valueOf(obj));
} else if (dataType instanceof PickledByteArrayTypeInfo || dataType instanceof BasicTypeInfo) {
return pickler.dumps(obj);
} else {
// other typeinfos will use the corresponding serializer to serialize data.
TypeSerializer serializer = dataType.createSerializer(null);
ByteArrayOutputStreamWithPos baos = new ByteArrayOutputStreamWithPos();
DataOutputViewStreamWrapper baosWrapper = new DataOutputViewStreamWrapper(baos);
serializer.serialize(obj, baosWrapper);
return pickler.dumps(baos.toByteArray());
}
}
}
Aggregations