Examples with LogicalType org.apache.pulsar.shade.org.apache.avro.LogicalType used on opensource projects

Example 1 with LogicalType

use of org.apache.pulsar.shade.org.apache.avro.LogicalType in project pulsar-flink by streamnative.

the class PulsarDeserializer method getRecordWriter.

private BinFunction<RowUpdater, GenericRecord> getRecordWriter(Schema avroType, FieldsDataType sqlType, List<String> path) throws IncompatibleSchemaException {
    List<Integer> validFieldIndexes = new ArrayList<>();
    List<BinFunction<RowUpdater, Object>> fieldWriters = new ArrayList<>();
    int length = sqlType.getChildren().size();
    RowType rowType = (RowType) sqlType.getLogicalType();
    List<RowType.RowField> fields = rowType.getFields();
    for (int i = 0; i < length; i++) {
        RowType.RowField sqlField = fields.get(i);
        org.apache.flink.table.types.logical.LogicalType logicalType = rowType.getTypeAt(i);
        Schema.Field avroField = avroType.getField(sqlField.getName());
        if (avroField != null) {
            validFieldIndexes.add(avroField.pos());
            TriFunction<FlinkDataUpdater, Integer, Object> baseWriter = newWriter(avroField.schema(), TypeConversions.fromLogicalToDataType(logicalType), Stream.concat(path.stream(), Stream.of(sqlField.getName())).collect(Collectors.toList()));
            int ordinal = i;
            BinFunction<RowUpdater, Object> fieldWriter = (updater, value) -> {
                if (value == null) {
                    updater.setNullAt(ordinal);
                } else {
                    baseWriter.apply(updater, ordinal, value);
                }
            };
            fieldWriters.add(fieldWriter);
        } else if (!sqlField.getType().isNullable()) {
            throw new IncompatibleSchemaException(String.format("Cannot find non-nullable field in avro schema %s", avroType));
        }
    }
    return (rowUpdater, record) -> {
        for (int i = 0; i < validFieldIndexes.size(); i++) {
            fieldWriters.get(i).apply(rowUpdater, record.get(validFieldIndexes.get(i)));
        }
    };
}

Example 2 with LogicalType

use of org.apache.pulsar.shade.org.apache.avro.LogicalType in project pulsar-flink by streamnative.

the class PulsarDeserializer method newWriter.

private TriFunction<FlinkDataUpdater, Integer, Object> newWriter(Schema avroType, DataType flinkType, List<String> path) throws IncompatibleSchemaException {
    LogicalTypeRoot tpe = flinkType.getLogicalType().getTypeRoot();
    Schema.Type atpe = avroType.getType();
    if (atpe == Schema.Type.NULL && tpe == LogicalTypeRoot.NULL) {
        return (rowUpdater, ordinal, value) -> rowUpdater.setNullAt(ordinal);
    } else if (atpe == Schema.Type.BOOLEAN && tpe == LogicalTypeRoot.BOOLEAN || atpe == Schema.Type.INT && tpe == LogicalTypeRoot.INTEGER || atpe == Schema.Type.LONG && tpe == LogicalTypeRoot.BIGINT || atpe == Schema.Type.FLOAT && tpe == LogicalTypeRoot.FLOAT || atpe == Schema.Type.DOUBLE && tpe == LogicalTypeRoot.DOUBLE) {
        return (rowUpdater, ordinal, value) -> rowUpdater.set(ordinal, value);
    } else if (atpe == Schema.Type.INT && tpe == LogicalTypeRoot.DATE) {
        return (rowUpdater, ordinal, value) -> rowUpdater.set(ordinal, LocalDate.ofEpochDay((Long) value));
    } else if (atpe == Schema.Type.LONG && tpe == LogicalTypeRoot.TIMESTAMP_WITHOUT_TIME_ZONE) {
        LogicalType altpe = avroType.getLogicalType();
        if (altpe instanceof LogicalTypes.TimestampMillis) {
            return (rowUpdater, ordinal, value) -> rowUpdater.set(ordinal, DateTimeUtils.toJavaTimestamp(((Long) value) * 1000).toLocalDateTime());
        } else if (altpe instanceof LogicalTypes.TimestampMicros) {
            return (rowUpdater, ordinal, value) -> rowUpdater.set(ordinal, DateTimeUtils.toJavaTimestamp((Long) value).toLocalDateTime());
        } else {
            throw new IncompatibleSchemaException(String.format("Cannot convert Avro logical type %s to flink timestamp type", altpe.toString()));
        }
    } else if (atpe == Schema.Type.STRING && tpe == LogicalTypeRoot.VARCHAR) {
        return (rowUpdater, ordinal, value) -> {
            String s = null;
            if (value instanceof String) {
                s = (String) value;
            } else if (value instanceof Utf8) {
                Utf8 u8 = (Utf8) value;
                byte[] bytes = new byte[u8.getByteLength()];
                System.arraycopy(u8.getBytes(), 0, bytes, 0, u8.getByteLength());
                s = new String(bytes, StandardCharsets.UTF_8);
            }
            rowUpdater.set(ordinal, s);
        };
    } else if (atpe == Schema.Type.ENUM && tpe == LogicalTypeRoot.VARCHAR) {
        return (rowUpdater, ordinal, value) -> rowUpdater.set(ordinal, value.toString());
    } else if (atpe == Schema.Type.FIXED && tpe == LogicalTypeRoot.BINARY) {
        return (rowUpdater, ordinal, value) -> rowUpdater.set(ordinal, ((GenericFixed) value).bytes().clone());
    } else if (atpe == Schema.Type.BYTES && tpe == LogicalTypeRoot.VARBINARY) {
        return (rowUpdater, ordinal, value) -> {
            byte[] bytes = null;
            if (value instanceof ByteBuffer) {
                ByteBuffer bb = (ByteBuffer) value;
                bytes = new byte[bb.remaining()];
                bb.get(bytes);
            } else if (value instanceof byte[]) {
                bytes = (byte[]) value;
            } else {
                throw new IllegalStateException(value.toString() + " is not a valid avro binary");
            }
            rowUpdater.set(ordinal, bytes);
        };
    } else if (atpe == Schema.Type.FIXED && tpe == LogicalTypeRoot.DECIMAL) {
        DecimalType d = (DecimalType) flinkType.getLogicalType();
        return (rowUpdater, ordinal, value) -> {
            BigDecimal bigDecimal = decimalConversions.fromFixed((GenericFixed) value, avroType, LogicalTypes.decimal(d.getPrecision(), d.getScale()));
            rowUpdater.set(ordinal, bigDecimal);
        };
    } else if (atpe == Schema.Type.BYTES && tpe == LogicalTypeRoot.DECIMAL) {
        DecimalType d = (DecimalType) flinkType.getLogicalType();
        return (rowUpdater, ordinal, value) -> {
            BigDecimal bigDecimal = decimalConversions.fromBytes((ByteBuffer) value, avroType, LogicalTypes.decimal(d.getPrecision(), d.getScale()));
            rowUpdater.set(ordinal, bigDecimal);
        };
    } else if (atpe == Schema.Type.RECORD && tpe == LogicalTypeRoot.ROW) {
        FieldsDataType fieldsDataType = (FieldsDataType) flinkType;
        BinFunction<RowUpdater, GenericRecord> writeRecord = getRecordWriter(avroType, fieldsDataType, path);
        return (rowUpdater, ordinal, value) -> {
            Row row = new Row(fieldsDataType.getChildren().size());
            RowUpdater ru = new RowUpdater();
            ru.setRow(row);
            writeRecord.apply(ru, (GenericRecord) value);
            rowUpdater.set(ordinal, row);
        };
    } else if (tpe == LogicalTypeRoot.ARRAY && atpe == Schema.Type.ARRAY && flinkType instanceof CollectionDataType) {
        DataType et = ((CollectionDataType) flinkType).getElementDataType();
        boolean containsNull = et.getLogicalType().isNullable();
        TriFunction<FlinkDataUpdater, Integer, Object> elementWriter = newWriter(avroType.getElementType(), et, path);
        return (rowUpdater, ordinal, value) -> {
            List array = (List) value;
            int len = array.size();
            Object[] result = new Object[len];
            ArrayDataUpdater elementUpdater = new ArrayDataUpdater(result);
            for (int i = 0; i < len; i++) {
                Object element = array.get(i);
                if (element == null) {
                    if (!containsNull) {
                        throw new IllegalArgumentException(String.format("Array value at path %s is not allowed to be null", path.toString()));
                    } else {
                        elementUpdater.setNullAt(i);
                    }
                } else {
                    elementWriter.apply(elementUpdater, i, element);
                }
            }
            rowUpdater.set(ordinal, result);
        };
    } else if (tpe == LogicalTypeRoot.MAP && atpe == Schema.Type.MAP && ((KeyValueDataType) flinkType).getKeyDataType().getLogicalType().getTypeRoot() == LogicalTypeRoot.VARCHAR) {
        KeyValueDataType kvt = (KeyValueDataType) flinkType;
        DataType kt = kvt.getKeyDataType();
        TriFunction<FlinkDataUpdater, Integer, Object> keyWriter = newWriter(SchemaBuilder.builder().stringType(), kt, path);
        DataType vt = kvt.getValueDataType();
        TriFunction<FlinkDataUpdater, Integer, Object> valueWriter = newWriter(avroType.getValueType(), vt, path);
        boolean valueContainsNull = vt.getLogicalType().isNullable();
        return (rowUpdater, ordinal, value) -> {
            Map<Object, Object> map = (Map<Object, Object>) value;
            String[] keys = new String[map.size()];
            Object[] values = new Object[map.size()];
            ArrayDataUpdater keyUpdater = new ArrayDataUpdater(keys);
            ArrayDataUpdater valueUpdater = new ArrayDataUpdater(values);
            Iterator<Map.Entry<Object, Object>> iterator = map.entrySet().iterator();
            int i = 0;
            while (iterator.hasNext()) {
                Map.Entry entry = iterator.next();
                assert entry.getKey() != null;
                keyWriter.apply(keyUpdater, i, entry.getKey());
                if (entry.getValue() == null) {
                    if (!valueContainsNull) {
                        throw new IllegalArgumentException(String.format("Map value at path %s is not allowed to be null", path.toString()));
                    } else {
                        valueUpdater.setNullAt(i);
                    }
                } else {
                    valueWriter.apply(valueUpdater, i, entry.getValue());
                }
                i += 1;
            }
            Map<String, Object> result = new HashMap<>(map.size());
            for (int j = 0; j < map.size(); j++) {
                result.put(keys[j], values[j]);
            }
            rowUpdater.set(ordinal, result);
        };
    } else if (atpe == Schema.Type.UNION) {
        List<Schema> allTypes = avroType.getTypes();
        List<Schema> nonNullTypes = allTypes.stream().filter(t -> t.getType() != Schema.Type.NULL).collect(Collectors.toList());
        if (!nonNullTypes.isEmpty()) {
            if (nonNullTypes.size() == 1) {
                return newWriter(nonNullTypes.get(0), flinkType, path);
            } else {
                if (nonNullTypes.size() == 2) {
                    Schema.Type tp1 = nonNullTypes.get(0).getType();
                    Schema.Type tp2 = nonNullTypes.get(1).getType();
                    if (ImmutableSet.of(tp1, tp2).equals(ImmutableSet.of(Schema.Type.INT, Schema.Type.LONG)) && flinkType == DataTypes.BIGINT()) {
                        return (updater, ordinal, value) -> {
                            if (value == null) {
                                updater.setNullAt(ordinal);
                            } else if (value instanceof Long) {
                                updater.set(ordinal, value);
                            } else if (value instanceof Integer) {
                                updater.set(ordinal, ((Integer) value).longValue());
                            }
                        };
                    } else if (ImmutableSet.of(tp1, tp2).equals(ImmutableSet.of(Schema.Type.FLOAT, Schema.Type.DOUBLE)) && flinkType == DataTypes.DOUBLE()) {
                        return (updater, ordinal, value) -> {
                            if (value == null) {
                                updater.setNullAt(ordinal);
                            } else if (value instanceof Double) {
                                updater.set(ordinal, value);
                            } else if (value instanceof Float) {
                                updater.set(ordinal, ((Float) value).doubleValue());
                            }
                        };
                    } else {
                        throw new IncompatibleSchemaException(String.format("Cannot convert %s %s together to %s", tp1.toString(), tp2.toString(), flinkType));
                    }
                } else if (tpe == LogicalTypeRoot.ROW && ((RowType) flinkType.getLogicalType()).getFieldCount() == nonNullTypes.size()) {
                    RowType rt = (RowType) flinkType.getLogicalType();
                    List<TriFunction<FlinkDataUpdater, Integer, Object>> fieldWriters = new ArrayList<>();
                    for (int i = 0; i < nonNullTypes.size(); i++) {
                        Schema schema = nonNullTypes.get(i);
                        String field = rt.getFieldNames().get(i);
                        org.apache.flink.table.types.logical.LogicalType logicalType = rt.getTypeAt(i);
                        fieldWriters.add(newWriter(schema, TypeConversions.fromLogicalToDataType(logicalType), Stream.concat(path.stream(), Stream.of(field)).collect(Collectors.toList())));
                    }
                    return (updater, ordinal, value) -> {
                        Row row = new Row(rt.getFieldCount());
                        RowUpdater fieldUpdater = new RowUpdater();
                        fieldUpdater.setRow(row);
                        int i = GenericData.get().resolveUnion(avroType, value);
                        fieldWriters.get(i).apply(fieldUpdater, i, value);
                        updater.set(ordinal, row);
                    };
                } else {
                    throw new IncompatibleSchemaException(String.format("Cannot convert avro to flink because schema at %s is not compatible (avroType = %s, sqlType = %s)", path.toString(), avroType, flinkType.toString()));
                }
            }
        } else {
            return (updater, ordinal, value) -> updater.setNullAt(ordinal);
        }
    } else {
        throw new IncompatibleSchemaException(String.format("Cannot convert avro to flink because schema at path %s is not compatible (avroType = %s, sqlType = %s)", path.toString(), avroType.toString(), flinkType.toString()));
    }
}

Example 3 with LogicalType

use of org.apache.pulsar.shade.org.apache.avro.LogicalType in project pulsar-flink by streamnative.

the class PulsarSerializer method newStructConverter.

private Function<Object, Object> newStructConverter(FieldsDataType dataType, Schema avroStruct) throws IncompatibleSchemaException {
    if (avroStruct.getType() != Schema.Type.RECORD || avroStruct.getFields().size() != dataType.getChildren().size()) {
        throw new IncompatibleSchemaException(String.format("Cannot convert Flink type %s to Avro type %s.", dataType.toString(), avroStruct.toString(true)));
    }
    RowType rowType = (RowType) dataType.getLogicalType();
    List<RowType.RowField> fields = rowType.getFields();
    List<BiFunction<PositionedGetter, Integer, Object>> fieldConverters = new ArrayList<>();
    for (int i = 0; i < fields.size(); i++) {
        org.apache.flink.table.types.logical.LogicalType logicalType = rowType.getTypeAt(i);
        DataType dt = TypeConversions.fromLogicalToDataType(logicalType);
        Schema.Field at = avroStruct.getFields().get(i);
        fieldConverters.add(newConverter(dt, resolveNullableType(at.schema(), dt.getLogicalType().isNullable())));
    }
    int numFields = dataType.getChildren().size();
    return row -> {
        GenericSchema<GenericRecord> pSchema = SchemaUtils.avroSchema2PulsarSchema(avroStruct);
        GenericRecordBuilder builder = pSchema.newRecordBuilder();
        Row rowX = (Row) row;
        for (int i = 0; i < numFields; i++) {
            if (rowX.getField(i) == null) {
                builder.set(pSchema.getFields().get(i), null);
            } else {
                builder.set(pSchema.getFields().get(i), fieldConverters.get(i).apply(new PositionedGetter(rowX), i));
            }
        }
        return (GenericAvroRecord) builder.build();
    };
}

Example 4 with LogicalType

use of org.apache.pulsar.shade.org.apache.avro.LogicalType in project pulsar-flink by streamnative.

the class PulsarSerializer method singleValueConverter.

private BiFunction<PositionedGetter, Integer, Object> singleValueConverter(DataType dataType, Schema avroType) throws IncompatibleSchemaException {
    LogicalTypeRoot tpe = dataType.getLogicalType().getTypeRoot();
    Schema.Type atpe = avroType.getType();
    if (tpe == LogicalTypeRoot.NULL && atpe == Schema.Type.NULL) {
        return (getter, ordinal) -> null;
    } else if ((tpe == LogicalTypeRoot.BOOLEAN && atpe == Schema.Type.BOOLEAN) || (tpe == LogicalTypeRoot.TINYINT && atpe == Schema.Type.INT) || (tpe == LogicalTypeRoot.SMALLINT && atpe == Schema.Type.INT) || (tpe == LogicalTypeRoot.INTEGER && atpe == Schema.Type.INT) || (tpe == LogicalTypeRoot.BIGINT && atpe == Schema.Type.LONG) || (tpe == LogicalTypeRoot.FLOAT && atpe == Schema.Type.FLOAT) || (tpe == LogicalTypeRoot.DOUBLE && atpe == Schema.Type.DOUBLE) || (tpe == LogicalTypeRoot.VARCHAR && atpe == Schema.Type.STRING) || (tpe == LogicalTypeRoot.VARBINARY && atpe == Schema.Type.BYTES)) {
        return (getter, ordinal) -> getter.getField(ordinal);
    } else if ((tpe == LogicalTypeRoot.DATE && atpe == Schema.Type.INT)) {
        return (getter, ordinal) -> (LocalDate) getter.getField(ordinal);
    } else if (tpe == LogicalTypeRoot.TIMESTAMP_WITHOUT_TIME_ZONE && atpe == Schema.Type.LONG) {
        LogicalType altpe = avroType.getLogicalType();
        if (altpe instanceof LogicalTypes.TimestampMillis || altpe instanceof LogicalTypes.TimestampMicros) {
            return (getter, ordinal) -> (LocalDateTime) getter.getField(ordinal);
        } else {
            throw new IncompatibleSchemaException("Cannot convert flink timestamp to avro logical type " + altpe.toString());
        }
    } else {
        throw new IncompatibleSchemaException(String.format("Cannot convert flink type %s to avro type %s", dataType.toString(), avroType.toString(true)));
    }
}

Example 5 with LogicalType

use of org.apache.pulsar.shade.org.apache.avro.LogicalType in project pulsar-flink by streamnative.

the class PulsarSerializer method newConverter.

private BiFunction<PositionedGetter, Integer, Object> newConverter(DataType dataType, Schema avroType) throws IncompatibleSchemaException {
    LogicalTypeRoot tpe = dataType.getLogicalType().getTypeRoot();
    Schema.Type atpe = avroType.getType();
    if (tpe == LogicalTypeRoot.NULL && atpe == Schema.Type.NULL) {
        return (getter, ordinal) -> null;
    } else if ((tpe == LogicalTypeRoot.BOOLEAN && atpe == Schema.Type.BOOLEAN) || (tpe == LogicalTypeRoot.TINYINT && atpe == Schema.Type.INT) || (tpe == LogicalTypeRoot.SMALLINT && atpe == Schema.Type.INT) || (tpe == LogicalTypeRoot.INTEGER && atpe == Schema.Type.INT) || (tpe == LogicalTypeRoot.BIGINT && atpe == Schema.Type.LONG) || (tpe == LogicalTypeRoot.FLOAT && atpe == Schema.Type.FLOAT) || (tpe == LogicalTypeRoot.DOUBLE && atpe == Schema.Type.DOUBLE) || (tpe == LogicalTypeRoot.VARBINARY && atpe == Schema.Type.BYTES)) {
        return (getter, ordinal) -> getter.getField(ordinal);
    } else if (tpe == LogicalTypeRoot.DECIMAL && (atpe == Schema.Type.FIXED || atpe == Schema.Type.BYTES)) {
        DecimalType d = (DecimalType) dataType.getLogicalType();
        if (avroType.getLogicalType() == LogicalTypes.decimal(d.getPrecision(), d.getScale())) {
            return (getter, ordinal) -> {
                java.math.BigDecimal decimal = (java.math.BigDecimal) getter.getField(ordinal);
                return decimalConversion.toFixed(decimal, avroType, LogicalTypes.decimal(d.getPrecision(), d.getScale()));
            };
        } else {
            throw new IncompatibleSchemaException("Cannot convert flink decimal type to Avro logical type");
        }
    } else if (tpe == LogicalTypeRoot.BIGINT && atpe == Schema.Type.BYTES) {
        return (getter, ordinal) -> ByteBuffer.wrap((byte[]) getter.getField(ordinal));
    } else if (tpe == LogicalTypeRoot.DATE && atpe == Schema.Type.INT) {
        return (getter, ordinal) -> ((LocalDate) getter.getField(ordinal)).toEpochDay();
    } else if (tpe == LogicalTypeRoot.TIMESTAMP_WITH_TIME_ZONE && atpe == Schema.Type.LONG) {
        LogicalType altpe = avroType.getLogicalType();
        if (altpe instanceof LogicalTypes.TimestampMillis) {
            return (getter, ordinal) -> DateTimeUtils.fromJavaTimestamp(java.sql.Timestamp.valueOf((LocalDateTime) getter.getField(ordinal))) / 1000;
        } else if (altpe instanceof LogicalTypes.TimestampMicros) {
            return (getter, ordinal) -> DateTimeUtils.fromJavaTimestamp(java.sql.Timestamp.valueOf((LocalDateTime) getter.getField(ordinal)));
        } else {
            throw new IncompatibleSchemaException("Cannot convert flink timestamp to avro logical type " + altpe.toString());
        }
    } else if (tpe == LogicalTypeRoot.VARCHAR && atpe == Schema.Type.STRING) {
        return (getter, ordinal) -> new Utf8((String) getter.getField(ordinal));
    } else if (tpe == LogicalTypeRoot.VARCHAR && atpe == Schema.Type.ENUM) {
        HashSet<String> enumSymbols = new HashSet<>(avroType.getEnumSymbols());
        return (getter, ordinal) -> {
            String data = (String) getter.getField(ordinal);
            if (!enumSymbols.contains(data)) {
                throw new IllegalArgumentException(String.format("Cannot write %s since it's not defined in enum %s", data, String.join(", ", enumSymbols)));
            }
            return new GenericData.EnumSymbol(avroType, data);
        };
    } else if (tpe == LogicalTypeRoot.ARRAY && atpe == Schema.Type.ARRAY && dataType instanceof CollectionDataType) {
        DataType et = ((CollectionDataType) dataType).getElementDataType();
        boolean containsNull = et.getLogicalType().isNullable();
        BiFunction<PositionedGetter, Integer, Object> elementConverter = newConverter(et, resolveNullableType(avroType.getElementType(), containsNull));
        return (getter, ordinal) -> {
            Object[] arrayData = (Object[]) getter.getField(ordinal);
            int len = arrayData.length;
            Object[] result = new Object[len];
            for (int i = 0; i < len; i++) {
                if (containsNull && arrayData[i] == null) {
                    result[i] = null;
                } else {
                    result[i] = elementConverter.apply(new PositionedGetter(arrayData), i);
                }
            }
            // `ArrayList` backed by the specified array without data copying.
            return Arrays.asList(result);
        };
    } else if (tpe == LogicalTypeRoot.MAP && atpe == Schema.Type.MAP && ((KeyValueDataType) dataType).getKeyDataType().getLogicalType().getTypeRoot() == LogicalTypeRoot.VARCHAR) {
        return (getter, ordinal) -> getter.getField(ordinal);
    } else if (tpe == LogicalTypeRoot.ROW && atpe == Schema.Type.RECORD) {
        FieldsDataType st = (FieldsDataType) dataType;
        Function<Object, Object> structConverter = newStructConverter(st, avroType);
        return (getter, ordinal) -> ((GenericAvroRecord) structConverter.apply(getter.getField(ordinal))).getAvroRecord();
    } else {
        throw new IncompatibleSchemaException(String.format("Cannot convert flink type %s to avro type %s", dataType.toString(), avroType.toString(true)));
    }
}