Examples with DataType org.apache.flink.table.types.DataType used on opensource projects

Example 31 with DataType

use of org.apache.flink.table.types.DataType in project flink by apache.

the class DebeziumJsonDecodingFormat method createRuntimeDecoder.

@Override
public DeserializationSchema<RowData> createRuntimeDecoder(DynamicTableSource.Context context, DataType physicalDataType, int[][] projections) {
    physicalDataType = Projection.of(projections).project(physicalDataType);
    final List<ReadableMetadata> readableMetadata = metadataKeys.stream().map(k -> Stream.of(ReadableMetadata.values()).filter(rm -> rm.key.equals(k)).findFirst().orElseThrow(IllegalStateException::new)).collect(Collectors.toList());
    final List<DataTypes.Field> metadataFields = readableMetadata.stream().map(m -> DataTypes.FIELD(m.key, m.dataType)).collect(Collectors.toList());
    final DataType producedDataType = DataTypeUtils.appendRowFields(physicalDataType, metadataFields);
    final TypeInformation<RowData> producedTypeInfo = context.createTypeInformation(producedDataType);
    return new DebeziumJsonDeserializationSchema(physicalDataType, readableMetadata, producedTypeInfo, schemaInclude, ignoreParseErrors, timestampFormat);
}

Example 32 with DataType

use of org.apache.flink.table.types.DataType in project flink by apache.

the class DebeziumJsonFormatFactory method createEncodingFormat.

@Override
public EncodingFormat<SerializationSchema<RowData>> createEncodingFormat(DynamicTableFactory.Context context, ReadableConfig formatOptions) {
    FactoryUtil.validateFactoryOptions(this, formatOptions);
    validateEncodingFormatOptions(formatOptions);
    TimestampFormat timestampFormat = JsonFormatOptionsUtil.getTimestampFormat(formatOptions);
    JsonFormatOptions.MapNullKeyMode mapNullKeyMode = JsonFormatOptionsUtil.getMapNullKeyMode(formatOptions);
    String mapNullKeyLiteral = formatOptions.get(JSON_MAP_NULL_KEY_LITERAL);
    final boolean encodeDecimalAsPlainNumber = formatOptions.get(ENCODE_DECIMAL_AS_PLAIN_NUMBER);
    return new EncodingFormat<SerializationSchema<RowData>>() {

        @Override
        public ChangelogMode getChangelogMode() {
            return ChangelogMode.newBuilder().addContainedKind(RowKind.INSERT).addContainedKind(RowKind.UPDATE_BEFORE).addContainedKind(RowKind.UPDATE_AFTER).addContainedKind(RowKind.DELETE).build();
        }

        @Override
        public SerializationSchema<RowData> createRuntimeEncoder(DynamicTableSink.Context context, DataType consumedDataType) {
            final RowType rowType = (RowType) consumedDataType.getLogicalType();
            return new DebeziumJsonSerializationSchema(rowType, timestampFormat, mapNullKeyMode, mapNullKeyLiteral, encodeDecimalAsPlainNumber);
        }
    };
}

Example 33 with DataType

use of org.apache.flink.table.types.DataType in project flink by apache.

the class MaxwellJsonDeserializationSchema method createJsonRowType.

// --------------------------------------------------------------------------------------------
private static RowType createJsonRowType(DataType physicalDataType, List<ReadableMetadata> readableMetadata) {
    DataType root = DataTypes.ROW(DataTypes.FIELD("data", physicalDataType), DataTypes.FIELD("old", physicalDataType), DataTypes.FIELD("type", DataTypes.STRING()));
    // append fields that are required for reading metadata in the root
    final List<DataTypes.Field> rootMetadataFields = readableMetadata.stream().map(m -> m.requiredJsonField).distinct().collect(Collectors.toList());
    return (RowType) DataTypeUtils.appendRowFields(root, rootMetadataFields).getLogicalType();
}

Example 34 with DataType

use of org.apache.flink.table.types.DataType in project flink by apache.

the class FileInfoExtractorBulkFormat method wrapReader.

private Reader<RowData> wrapReader(Reader<RowData> superReader, FileSourceSplit split) {
    // Fill the metadata + partition columns row
    final GenericRowData fileInfoRowData = new GenericRowData(metadataColumnsFunctions.size() + partitionColumnTypes.size());
    int fileInfoRowIndex = 0;
    for (; fileInfoRowIndex < metadataColumnsFunctions.size(); fileInfoRowIndex++) {
        fileInfoRowData.setField(fileInfoRowIndex, metadataColumnsFunctions.get(fileInfoRowIndex).getValue(split));
    }
    if (!partitionColumnTypes.isEmpty()) {
        final LinkedHashMap<String, String> partitionSpec = PartitionPathUtils.extractPartitionSpecFromPath(split.path());
        for (int partitionFieldIndex = 0; fileInfoRowIndex < fileInfoRowData.getArity(); fileInfoRowIndex++, partitionFieldIndex++) {
            final String fieldName = partitionColumnTypes.get(partitionFieldIndex).getKey();
            final DataType fieldType = partitionColumnTypes.get(partitionFieldIndex).getValue();
            if (!partitionSpec.containsKey(fieldName)) {
                throw new RuntimeException("Cannot find the partition value from path for partition: " + fieldName);
            }
            String valueStr = partitionSpec.get(fieldName);
            valueStr = valueStr.equals(defaultPartName) ? null : valueStr;
            fileInfoRowData.setField(fileInfoRowIndex, PartitionPathUtils.convertStringToInternalValue(valueStr, fieldType));
        }
    }
    // This row is going to be reused for every record
    final EnrichedRowData producedRowData = new EnrichedRowData(fileInfoRowData, this.extendedRowIndexMapping);
    return RecordMapperWrapperRecordIterator.wrapReader(superReader, physicalRowData -> {
        producedRowData.replaceMutableRow(physicalRowData);
        return producedRowData;
    });
}

Example 35 with DataType

use of org.apache.flink.table.types.DataType in project flink by apache.

the class HiveWriterFactory method checkInitialize.

private void checkInitialize() throws Exception {
    if (initialized) {
        return;
    }
    JobConf jobConf = confWrapper.conf();
    Object serdeLib = Class.forName(serDeInfo.deserializeValue().getSerializationLib()).newInstance();
    Preconditions.checkArgument(serdeLib instanceof Serializer && serdeLib instanceof Deserializer, "Expect a SerDe lib implementing both Serializer and Deserializer, but actually got " + serdeLib.getClass().getName());
    this.recordSerDe = (Serializer) serdeLib;
    ReflectionUtils.setConf(recordSerDe, jobConf);
    // TODO: support partition properties, for now assume they're same as table properties
    SerDeUtils.initializeSerDe((Deserializer) recordSerDe, jobConf, tableProperties, null);
    this.formatFields = allColumns.length - partitionColumns.length;
    this.hiveConversions = new HiveObjectConversion[formatFields];
    this.converters = new DataFormatConverter[formatFields];
    List<ObjectInspector> objectInspectors = new ArrayList<>(hiveConversions.length);
    for (int i = 0; i < formatFields; i++) {
        DataType type = allTypes[i];
        ObjectInspector objectInspector = HiveInspectors.getObjectInspector(type);
        objectInspectors.add(objectInspector);
        hiveConversions[i] = HiveInspectors.getConversion(objectInspector, type.getLogicalType(), hiveShim);
        converters[i] = DataFormatConverters.getConverterForDataType(type);
    }
    this.formatInspector = ObjectInspectorFactory.getStandardStructObjectInspector(Arrays.asList(allColumns).subList(0, formatFields), objectInspectors);
    this.initialized = true;
}