Search in sources :

Example 1 with DataTypeFactory

use of org.apache.flink.table.catalog.DataTypeFactory in project flink by apache.

the class TypeInfoDataTypeConverter method convertToStructuredType.

private static DataType convertToStructuredType(DataTypeFactory dataTypeFactory, CompositeType<?> compositeType, boolean forceNullability) {
    final int arity = compositeType.getArity();
    final String[] fieldNames = compositeType.getFieldNames();
    final Class<?> typeClass = compositeType.getTypeClass();
    final Map<String, DataType> fieldDataTypes = new LinkedHashMap<>();
    IntStream.range(0, arity).forEachOrdered(pos -> fieldDataTypes.put(fieldNames[pos], toDataType(dataTypeFactory, compositeType.getTypeAt(pos))));
    final List<String> fieldNamesReordered;
    final boolean isNullable;
    // for POJOs and Avro records
    if (compositeType instanceof PojoTypeInfo) {
        final PojoTypeInfo<?> pojoTypeInfo = (PojoTypeInfo<?>) compositeType;
        final List<Field> pojoFields = IntStream.range(0, arity).mapToObj(pojoTypeInfo::getPojoFieldAt).map(PojoField::getField).collect(Collectors.toList());
        // POJO serializer supports top-level nulls
        isNullable = true;
        // based on type information all fields are boxed classes,
        // therefore we need to check the reflective field for more details
        fieldDataTypes.replaceAll((name, dataType) -> {
            final Class<?> fieldClass = pojoFields.stream().filter(f -> f.getName().equals(name)).findFirst().orElseThrow(IllegalStateException::new).getType();
            if (fieldClass.isPrimitive()) {
                return dataType.notNull().bridgedTo(fieldClass);
            }
            // serializer supports nullable fields
            return dataType.nullable();
        });
        // best effort extraction of the field order, if it fails we use the default order of
        // PojoTypeInfo which is alphabetical
        fieldNamesReordered = extractStructuredTypeFieldOrder(typeClass, pojoFields);
    } else // for tuples and case classes
    {
        // serializers don't support top-level nulls
        isNullable = forceNullability;
        // based on type information all fields are boxed classes,
        // but case classes might contain primitives
        fieldDataTypes.replaceAll((name, dataType) -> {
            try {
                final Class<?> fieldClass = getStructuredField(typeClass, name).getType();
                if (fieldClass.isPrimitive()) {
                    return dataType.notNull().bridgedTo(fieldClass);
                }
            } catch (Throwable t) {
            // ignore extraction errors and keep the original conversion class
            }
            return dataType;
        });
        // field order from type information is correct
        fieldNamesReordered = null;
    }
    final DataTypes.Field[] structuredFields;
    if (fieldNamesReordered != null) {
        structuredFields = fieldNamesReordered.stream().map(name -> DataTypes.FIELD(name, fieldDataTypes.get(name))).toArray(DataTypes.Field[]::new);
    } else {
        structuredFields = fieldDataTypes.entrySet().stream().map(e -> DataTypes.FIELD(e.getKey(), e.getValue())).toArray(DataTypes.Field[]::new);
    }
    final DataType structuredDataType = DataTypes.STRUCTURED(typeClass, structuredFields);
    if (isNullable) {
        return structuredDataType.nullable();
    } else {
        return structuredDataType.notNull();
    }
}
Also used : PojoTypeInfo(org.apache.flink.api.java.typeutils.PojoTypeInfo) DataTypeFactory(org.apache.flink.table.catalog.DataTypeFactory) DataTypeQueryable(org.apache.flink.table.types.DataTypeQueryable) IntStream(java.util.stream.IntStream) DataType(org.apache.flink.table.types.DataType) ExtractionUtils.getStructuredField(org.apache.flink.table.types.extraction.ExtractionUtils.getStructuredField) BasicArrayTypeInfo(org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo) StructuredType(org.apache.flink.table.types.logical.StructuredType) LocalDateTime(java.time.LocalDateTime) HashMap(java.util.HashMap) ExtractionUtils.extractAssigningConstructor(org.apache.flink.table.types.extraction.ExtractionUtils.extractAssigningConstructor) ObjectArrayTypeInfo(org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo) RowTypeInfo(org.apache.flink.api.java.typeutils.RowTypeInfo) RowType(org.apache.flink.table.types.logical.RowType) LinkedHashMap(java.util.LinkedHashMap) ListTypeInfo(org.apache.flink.api.java.typeutils.ListTypeInfo) BigDecimal(java.math.BigDecimal) RawType(org.apache.flink.table.types.logical.RawType) Map(java.util.Map) LocalTime(java.time.LocalTime) MapTypeInfo(org.apache.flink.api.java.typeutils.MapTypeInfo) TypeInformation(org.apache.flink.api.common.typeinfo.TypeInformation) Nullable(javax.annotation.Nullable) Types(org.apache.flink.api.common.typeinfo.Types) GenericTypeInfo(org.apache.flink.api.java.typeutils.GenericTypeInfo) CompositeType(org.apache.flink.api.common.typeutils.CompositeType) DataTypes(org.apache.flink.table.api.DataTypes) PojoField(org.apache.flink.api.java.typeutils.PojoField) Field(java.lang.reflect.Field) Instant(java.time.Instant) Collectors(java.util.stream.Collectors) PrimitiveArrayTypeInfo(org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo) List(java.util.List) LogicalType(org.apache.flink.table.types.logical.LogicalType) LocalDate(java.time.LocalDate) Internal(org.apache.flink.annotation.Internal) Row(org.apache.flink.types.Row) MultisetTypeInfo(org.apache.flink.api.java.typeutils.MultisetTypeInfo) ExtractionUtils(org.apache.flink.table.types.extraction.ExtractionUtils) TupleTypeInfoBase(org.apache.flink.api.java.typeutils.TupleTypeInfoBase) PojoTypeInfo(org.apache.flink.api.java.typeutils.PojoTypeInfo) LinkedHashMap(java.util.LinkedHashMap) ExtractionUtils.getStructuredField(org.apache.flink.table.types.extraction.ExtractionUtils.getStructuredField) PojoField(org.apache.flink.api.java.typeutils.PojoField) Field(java.lang.reflect.Field) DataType(org.apache.flink.table.types.DataType)

Example 2 with DataTypeFactory

use of org.apache.flink.table.catalog.DataTypeFactory in project flink by apache.

the class CommonExecLookupJoin method createAsyncLookupJoin.

@SuppressWarnings("unchecked")
private StreamOperatorFactory<RowData> createAsyncLookupJoin(RelOptTable temporalTable, ExecNodeConfig config, Map<Integer, LookupJoinUtil.LookupKey> allLookupKeys, AsyncTableFunction<Object> asyncLookupFunction, RelBuilder relBuilder, RowType inputRowType, RowType tableSourceRowType, RowType resultRowType, boolean isLeftOuterJoin) {
    int asyncBufferCapacity = config.get(ExecutionConfigOptions.TABLE_EXEC_ASYNC_LOOKUP_BUFFER_CAPACITY);
    long asyncTimeout = config.get(ExecutionConfigOptions.TABLE_EXEC_ASYNC_LOOKUP_TIMEOUT).toMillis();
    DataTypeFactory dataTypeFactory = ShortcutUtils.unwrapContext(relBuilder).getCatalogManager().getDataTypeFactory();
    LookupJoinCodeGenerator.GeneratedTableFunctionWithDataType<AsyncFunction<RowData, Object>> generatedFuncWithType = LookupJoinCodeGenerator.generateAsyncLookupFunction(config.getTableConfig(), dataTypeFactory, inputRowType, tableSourceRowType, resultRowType, allLookupKeys, LookupJoinUtil.getOrderedLookupKeys(allLookupKeys.keySet()), asyncLookupFunction, StringUtils.join(temporalTable.getQualifiedName(), "."));
    RowType rightRowType = Optional.ofNullable(temporalTableOutputType).map(FlinkTypeFactory::toLogicalRowType).orElse(tableSourceRowType);
    // a projection or filter after table source scan
    GeneratedResultFuture<TableFunctionResultFuture<RowData>> generatedResultFuture = LookupJoinCodeGenerator.generateTableAsyncCollector(config.getTableConfig(), "TableFunctionResultFuture", inputRowType, rightRowType, JavaScalaConversionUtil.toScala(Optional.ofNullable(joinCondition)));
    DataStructureConverter<?, ?> fetcherConverter = DataStructureConverters.getConverter(generatedFuncWithType.dataType());
    AsyncFunction<RowData, RowData> asyncFunc;
    if (existCalcOnTemporalTable) {
        // a projection or filter after table source scan
        GeneratedFunction<FlatMapFunction<RowData, RowData>> generatedCalc = LookupJoinCodeGenerator.generateCalcMapFunction(config.getTableConfig(), JavaScalaConversionUtil.toScala(projectionOnTemporalTable), filterOnTemporalTable, temporalTableOutputType, tableSourceRowType);
        asyncFunc = new AsyncLookupJoinWithCalcRunner(generatedFuncWithType.tableFunc(), (DataStructureConverter<RowData, Object>) fetcherConverter, generatedCalc, generatedResultFuture, InternalSerializers.create(rightRowType), isLeftOuterJoin, asyncBufferCapacity);
    } else {
        // right type is the same as table source row type, because no calc after temporal table
        asyncFunc = new AsyncLookupJoinRunner(generatedFuncWithType.tableFunc(), (DataStructureConverter<RowData, Object>) fetcherConverter, generatedResultFuture, InternalSerializers.create(rightRowType), isLeftOuterJoin, asyncBufferCapacity);
    }
    // when the downstream do not need orderness
    return new AsyncWaitOperatorFactory<>(asyncFunc, asyncTimeout, asyncBufferCapacity, AsyncDataStream.OutputMode.ORDERED);
}
Also used : AsyncLookupJoinRunner(org.apache.flink.table.runtime.operators.join.lookup.AsyncLookupJoinRunner) AsyncWaitOperatorFactory(org.apache.flink.streaming.api.operators.async.AsyncWaitOperatorFactory) DataStructureConverter(org.apache.flink.table.data.conversion.DataStructureConverter) RowType(org.apache.flink.table.types.logical.RowType) DataTypeFactory(org.apache.flink.table.catalog.DataTypeFactory) AsyncLookupJoinWithCalcRunner(org.apache.flink.table.runtime.operators.join.lookup.AsyncLookupJoinWithCalcRunner) AsyncFunction(org.apache.flink.streaming.api.functions.async.AsyncFunction) LookupJoinCodeGenerator(org.apache.flink.table.planner.codegen.LookupJoinCodeGenerator) RowData(org.apache.flink.table.data.RowData) TableFunctionResultFuture(org.apache.flink.table.runtime.collector.TableFunctionResultFuture) FlatMapFunction(org.apache.flink.api.common.functions.FlatMapFunction)

Example 3 with DataTypeFactory

use of org.apache.flink.table.catalog.DataTypeFactory in project flink by apache.

the class DynamicSinkUtils method convertCollectToRel.

/**
 * Converts an {@link TableResult#collect()} sink to a {@link RelNode}.
 */
public static RelNode convertCollectToRel(FlinkRelBuilder relBuilder, RelNode input, CollectModifyOperation collectModifyOperation, ReadableConfig configuration, ClassLoader classLoader) {
    final DataTypeFactory dataTypeFactory = unwrapContext(relBuilder).getCatalogManager().getDataTypeFactory();
    final ResolvedSchema childSchema = collectModifyOperation.getChild().getResolvedSchema();
    final ResolvedSchema schema = ResolvedSchema.physical(childSchema.getColumnNames(), childSchema.getColumnDataTypes());
    final ResolvedCatalogTable catalogTable = new ResolvedCatalogTable(new ExternalCatalogTable(Schema.newBuilder().fromResolvedSchema(schema).build()), schema);
    final ContextResolvedTable contextResolvedTable = ContextResolvedTable.anonymous("collect", catalogTable);
    final DataType consumedDataType = fixCollectDataType(dataTypeFactory, schema);
    final String zone = configuration.get(TableConfigOptions.LOCAL_TIME_ZONE);
    final ZoneId zoneId = TableConfigOptions.LOCAL_TIME_ZONE.defaultValue().equals(zone) ? ZoneId.systemDefault() : ZoneId.of(zone);
    final CollectDynamicSink tableSink = new CollectDynamicSink(contextResolvedTable.getIdentifier(), consumedDataType, configuration.get(CollectSinkOperatorFactory.MAX_BATCH_SIZE), configuration.get(CollectSinkOperatorFactory.SOCKET_TIMEOUT), classLoader, zoneId, configuration.get(ExecutionConfigOptions.TABLE_EXEC_LEGACY_CAST_BEHAVIOUR).isEnabled());
    collectModifyOperation.setSelectResultProvider(tableSink.getSelectResultProvider());
    collectModifyOperation.setConsumedDataType(consumedDataType);
    return convertSinkToRel(relBuilder, input, // dynamicOptions
    Collections.emptyMap(), contextResolvedTable, // staticPartitions
    Collections.emptyMap(), false, tableSink);
}
Also used : ResolvedCatalogTable(org.apache.flink.table.catalog.ResolvedCatalogTable) ZoneId(java.time.ZoneId) ExternalCatalogTable(org.apache.flink.table.catalog.ExternalCatalogTable) DataType(org.apache.flink.table.types.DataType) RelDataType(org.apache.calcite.rel.type.RelDataType) DataTypeFactory(org.apache.flink.table.catalog.DataTypeFactory) ContextResolvedTable(org.apache.flink.table.catalog.ContextResolvedTable) ResolvedSchema(org.apache.flink.table.catalog.ResolvedSchema)

Example 4 with DataTypeFactory

use of org.apache.flink.table.catalog.DataTypeFactory in project flink by apache.

the class StructuredObjectConverter method createOrError.

/**
 * Creates a {@link DataStructureConverter} for the given structured type.
 *
 * <p>Note: We do not perform validation if data type and structured type implementation match.
 * This must have been done earlier in the {@link DataTypeFactory}.
 */
@SuppressWarnings("RedundantCast")
private static StructuredObjectConverter<?> createOrError(DataType dataType) {
    final List<DataType> fields = dataType.getChildren();
    final DataStructureConverter<Object, Object>[] fieldConverters = fields.stream().map(dt -> (DataStructureConverter<Object, Object>) DataStructureConverters.getConverter(dt)).toArray(DataStructureConverter[]::new);
    final RowData.FieldGetter[] fieldGetters = IntStream.range(0, fields.size()).mapToObj(pos -> RowData.createFieldGetter(fields.get(pos).getLogicalType(), pos)).toArray(RowData.FieldGetter[]::new);
    final Class<?>[] fieldClasses = fields.stream().map(DataType::getConversionClass).toArray(Class[]::new);
    final StructuredType structuredType = (StructuredType) dataType.getLogicalType();
    final Class<?> implementationClass = structuredType.getImplementationClass().orElseThrow(IllegalStateException::new);
    final int uniqueClassId = nextUniqueClassId.getAndIncrement();
    final String converterName = String.format("%s$%s$Converter", implementationClass.getName().replace('.', '$'), uniqueClassId);
    final String converterCode = generateCode(converterName, implementationClass, getFieldNames(structuredType).toArray(new String[0]), fieldClasses);
    return new StructuredObjectConverter<>(fieldConverters, fieldGetters, converterName, converterCode);
}
Also used : DataTypeFactory(org.apache.flink.table.catalog.DataTypeFactory) IntStream(java.util.stream.IntStream) DataType(org.apache.flink.table.types.DataType) RowData(org.apache.flink.table.data.RowData) CompileUtils(org.apache.flink.table.runtime.generated.CompileUtils) ExtractionUtils.getStructuredField(org.apache.flink.table.types.extraction.ExtractionUtils.getStructuredField) LogicalTypeChecks.getFieldNames(org.apache.flink.table.types.logical.utils.LogicalTypeChecks.getFieldNames) ExtractionUtils.getStructuredFieldGetter(org.apache.flink.table.types.extraction.ExtractionUtils.getStructuredFieldGetter) ExtractionUtils.isStructuredFieldDirectlyWritable(org.apache.flink.table.types.extraction.ExtractionUtils.isStructuredFieldDirectlyWritable) StructuredType(org.apache.flink.table.types.logical.StructuredType) TableException(org.apache.flink.table.api.TableException) Field(java.lang.reflect.Field) ExtractionUtils.hasInvokableConstructor(org.apache.flink.table.types.extraction.ExtractionUtils.hasInvokableConstructor) List(java.util.List) GenericRowData(org.apache.flink.table.data.GenericRowData) ExtractionUtils.getStructuredFieldSetter(org.apache.flink.table.types.extraction.ExtractionUtils.getStructuredFieldSetter) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ExtractionUtils.isStructuredFieldDirectlyReadable(org.apache.flink.table.types.extraction.ExtractionUtils.isStructuredFieldDirectlyReadable) ExtractionUtils.primitiveToWrapper(org.apache.flink.table.types.extraction.ExtractionUtils.primitiveToWrapper) Internal(org.apache.flink.annotation.Internal) Method(java.lang.reflect.Method) ExtractionUtils.getStructuredFieldGetter(org.apache.flink.table.types.extraction.ExtractionUtils.getStructuredFieldGetter) StructuredType(org.apache.flink.table.types.logical.StructuredType) RowData(org.apache.flink.table.data.RowData) GenericRowData(org.apache.flink.table.data.GenericRowData) DataType(org.apache.flink.table.types.DataType)

Example 5 with DataTypeFactory

use of org.apache.flink.table.catalog.DataTypeFactory in project flink by apache.

the class SqlAggFunctionVisitor method createSqlAggFunction.

private SqlAggFunction createSqlAggFunction(CallExpression call) {
    final FunctionDefinition definition = call.getFunctionDefinition();
    // legacy
    if (definition instanceof AggregateFunctionDefinition) {
        return createLegacySqlAggregateFunction(call.getFunctionIdentifier().orElse(null), (AggregateFunctionDefinition) definition);
    } else if (definition instanceof TableAggregateFunctionDefinition) {
        return createLegacySqlTableAggregateFunction(call.getFunctionIdentifier().orElse(null), (TableAggregateFunctionDefinition) definition);
    }
    // new stack
    final DataTypeFactory dataTypeFactory = ShortcutUtils.unwrapContext(relBuilder).getCatalogManager().getDataTypeFactory();
    final TypeInference typeInference = definition.getTypeInference(dataTypeFactory);
    return BridgingSqlAggFunction.of(dataTypeFactory, ShortcutUtils.unwrapTypeFactory(relBuilder), SqlKind.OTHER_FUNCTION, ContextResolvedFunction.fromCallExpression(call), typeInference);
}
Also used : TypeInference(org.apache.flink.table.types.inference.TypeInference) AggregateFunctionDefinition(org.apache.flink.table.functions.AggregateFunctionDefinition) TableAggregateFunctionDefinition(org.apache.flink.table.functions.TableAggregateFunctionDefinition) TableAggregateFunctionDefinition(org.apache.flink.table.functions.TableAggregateFunctionDefinition) AggregateFunctionDefinition(org.apache.flink.table.functions.AggregateFunctionDefinition) TableAggregateFunctionDefinition(org.apache.flink.table.functions.TableAggregateFunctionDefinition) FunctionDefinition(org.apache.flink.table.functions.FunctionDefinition) DataTypeFactory(org.apache.flink.table.catalog.DataTypeFactory)

Aggregations

DataTypeFactory (org.apache.flink.table.catalog.DataTypeFactory)12 DataType (org.apache.flink.table.types.DataType)5 List (java.util.List)3 IntStream (java.util.stream.IntStream)3 DataTypes (org.apache.flink.table.api.DataTypes)3 RowData (org.apache.flink.table.data.RowData)3 TypeInference (org.apache.flink.table.types.inference.TypeInference)3 RowType (org.apache.flink.table.types.logical.RowType)3 Row (org.apache.flink.types.Row)3 Field (java.lang.reflect.Field)2 LocalDate (java.time.LocalDate)2 ArrayList (java.util.ArrayList)2 Collectors (java.util.stream.Collectors)2 Nullable (javax.annotation.Nullable)2 Internal (org.apache.flink.annotation.Internal)2 FlatMapFunction (org.apache.flink.api.common.functions.FlatMapFunction)2 BridgingUtils.createSqlOperandTypeInference (org.apache.flink.table.planner.functions.bridging.BridgingUtils.createSqlOperandTypeInference)2 BridgingUtils.createSqlReturnTypeInference (org.apache.flink.table.planner.functions.bridging.BridgingUtils.createSqlReturnTypeInference)2 ExtractionUtils.getStructuredField (org.apache.flink.table.types.extraction.ExtractionUtils.getStructuredField)2 LogicalType (org.apache.flink.table.types.logical.LogicalType)2