use of org.apache.flink.table.catalog.DataTypeFactory in project flink by apache.
the class TypeInfoDataTypeConverter method convertToStructuredType.
private static DataType convertToStructuredType(DataTypeFactory dataTypeFactory, CompositeType<?> compositeType, boolean forceNullability) {
final int arity = compositeType.getArity();
final String[] fieldNames = compositeType.getFieldNames();
final Class<?> typeClass = compositeType.getTypeClass();
final Map<String, DataType> fieldDataTypes = new LinkedHashMap<>();
IntStream.range(0, arity).forEachOrdered(pos -> fieldDataTypes.put(fieldNames[pos], toDataType(dataTypeFactory, compositeType.getTypeAt(pos))));
final List<String> fieldNamesReordered;
final boolean isNullable;
// for POJOs and Avro records
if (compositeType instanceof PojoTypeInfo) {
final PojoTypeInfo<?> pojoTypeInfo = (PojoTypeInfo<?>) compositeType;
final List<Field> pojoFields = IntStream.range(0, arity).mapToObj(pojoTypeInfo::getPojoFieldAt).map(PojoField::getField).collect(Collectors.toList());
// POJO serializer supports top-level nulls
isNullable = true;
// based on type information all fields are boxed classes,
// therefore we need to check the reflective field for more details
fieldDataTypes.replaceAll((name, dataType) -> {
final Class<?> fieldClass = pojoFields.stream().filter(f -> f.getName().equals(name)).findFirst().orElseThrow(IllegalStateException::new).getType();
if (fieldClass.isPrimitive()) {
return dataType.notNull().bridgedTo(fieldClass);
}
// serializer supports nullable fields
return dataType.nullable();
});
// best effort extraction of the field order, if it fails we use the default order of
// PojoTypeInfo which is alphabetical
fieldNamesReordered = extractStructuredTypeFieldOrder(typeClass, pojoFields);
} else // for tuples and case classes
{
// serializers don't support top-level nulls
isNullable = forceNullability;
// based on type information all fields are boxed classes,
// but case classes might contain primitives
fieldDataTypes.replaceAll((name, dataType) -> {
try {
final Class<?> fieldClass = getStructuredField(typeClass, name).getType();
if (fieldClass.isPrimitive()) {
return dataType.notNull().bridgedTo(fieldClass);
}
} catch (Throwable t) {
// ignore extraction errors and keep the original conversion class
}
return dataType;
});
// field order from type information is correct
fieldNamesReordered = null;
}
final DataTypes.Field[] structuredFields;
if (fieldNamesReordered != null) {
structuredFields = fieldNamesReordered.stream().map(name -> DataTypes.FIELD(name, fieldDataTypes.get(name))).toArray(DataTypes.Field[]::new);
} else {
structuredFields = fieldDataTypes.entrySet().stream().map(e -> DataTypes.FIELD(e.getKey(), e.getValue())).toArray(DataTypes.Field[]::new);
}
final DataType structuredDataType = DataTypes.STRUCTURED(typeClass, structuredFields);
if (isNullable) {
return structuredDataType.nullable();
} else {
return structuredDataType.notNull();
}
}
use of org.apache.flink.table.catalog.DataTypeFactory in project flink by apache.
the class CommonExecLookupJoin method createAsyncLookupJoin.
@SuppressWarnings("unchecked")
private StreamOperatorFactory<RowData> createAsyncLookupJoin(RelOptTable temporalTable, ExecNodeConfig config, Map<Integer, LookupJoinUtil.LookupKey> allLookupKeys, AsyncTableFunction<Object> asyncLookupFunction, RelBuilder relBuilder, RowType inputRowType, RowType tableSourceRowType, RowType resultRowType, boolean isLeftOuterJoin) {
int asyncBufferCapacity = config.get(ExecutionConfigOptions.TABLE_EXEC_ASYNC_LOOKUP_BUFFER_CAPACITY);
long asyncTimeout = config.get(ExecutionConfigOptions.TABLE_EXEC_ASYNC_LOOKUP_TIMEOUT).toMillis();
DataTypeFactory dataTypeFactory = ShortcutUtils.unwrapContext(relBuilder).getCatalogManager().getDataTypeFactory();
LookupJoinCodeGenerator.GeneratedTableFunctionWithDataType<AsyncFunction<RowData, Object>> generatedFuncWithType = LookupJoinCodeGenerator.generateAsyncLookupFunction(config.getTableConfig(), dataTypeFactory, inputRowType, tableSourceRowType, resultRowType, allLookupKeys, LookupJoinUtil.getOrderedLookupKeys(allLookupKeys.keySet()), asyncLookupFunction, StringUtils.join(temporalTable.getQualifiedName(), "."));
RowType rightRowType = Optional.ofNullable(temporalTableOutputType).map(FlinkTypeFactory::toLogicalRowType).orElse(tableSourceRowType);
// a projection or filter after table source scan
GeneratedResultFuture<TableFunctionResultFuture<RowData>> generatedResultFuture = LookupJoinCodeGenerator.generateTableAsyncCollector(config.getTableConfig(), "TableFunctionResultFuture", inputRowType, rightRowType, JavaScalaConversionUtil.toScala(Optional.ofNullable(joinCondition)));
DataStructureConverter<?, ?> fetcherConverter = DataStructureConverters.getConverter(generatedFuncWithType.dataType());
AsyncFunction<RowData, RowData> asyncFunc;
if (existCalcOnTemporalTable) {
// a projection or filter after table source scan
GeneratedFunction<FlatMapFunction<RowData, RowData>> generatedCalc = LookupJoinCodeGenerator.generateCalcMapFunction(config.getTableConfig(), JavaScalaConversionUtil.toScala(projectionOnTemporalTable), filterOnTemporalTable, temporalTableOutputType, tableSourceRowType);
asyncFunc = new AsyncLookupJoinWithCalcRunner(generatedFuncWithType.tableFunc(), (DataStructureConverter<RowData, Object>) fetcherConverter, generatedCalc, generatedResultFuture, InternalSerializers.create(rightRowType), isLeftOuterJoin, asyncBufferCapacity);
} else {
// right type is the same as table source row type, because no calc after temporal table
asyncFunc = new AsyncLookupJoinRunner(generatedFuncWithType.tableFunc(), (DataStructureConverter<RowData, Object>) fetcherConverter, generatedResultFuture, InternalSerializers.create(rightRowType), isLeftOuterJoin, asyncBufferCapacity);
}
// when the downstream do not need orderness
return new AsyncWaitOperatorFactory<>(asyncFunc, asyncTimeout, asyncBufferCapacity, AsyncDataStream.OutputMode.ORDERED);
}
use of org.apache.flink.table.catalog.DataTypeFactory in project flink by apache.
the class DynamicSinkUtils method convertCollectToRel.
/**
* Converts an {@link TableResult#collect()} sink to a {@link RelNode}.
*/
public static RelNode convertCollectToRel(FlinkRelBuilder relBuilder, RelNode input, CollectModifyOperation collectModifyOperation, ReadableConfig configuration, ClassLoader classLoader) {
final DataTypeFactory dataTypeFactory = unwrapContext(relBuilder).getCatalogManager().getDataTypeFactory();
final ResolvedSchema childSchema = collectModifyOperation.getChild().getResolvedSchema();
final ResolvedSchema schema = ResolvedSchema.physical(childSchema.getColumnNames(), childSchema.getColumnDataTypes());
final ResolvedCatalogTable catalogTable = new ResolvedCatalogTable(new ExternalCatalogTable(Schema.newBuilder().fromResolvedSchema(schema).build()), schema);
final ContextResolvedTable contextResolvedTable = ContextResolvedTable.anonymous("collect", catalogTable);
final DataType consumedDataType = fixCollectDataType(dataTypeFactory, schema);
final String zone = configuration.get(TableConfigOptions.LOCAL_TIME_ZONE);
final ZoneId zoneId = TableConfigOptions.LOCAL_TIME_ZONE.defaultValue().equals(zone) ? ZoneId.systemDefault() : ZoneId.of(zone);
final CollectDynamicSink tableSink = new CollectDynamicSink(contextResolvedTable.getIdentifier(), consumedDataType, configuration.get(CollectSinkOperatorFactory.MAX_BATCH_SIZE), configuration.get(CollectSinkOperatorFactory.SOCKET_TIMEOUT), classLoader, zoneId, configuration.get(ExecutionConfigOptions.TABLE_EXEC_LEGACY_CAST_BEHAVIOUR).isEnabled());
collectModifyOperation.setSelectResultProvider(tableSink.getSelectResultProvider());
collectModifyOperation.setConsumedDataType(consumedDataType);
return convertSinkToRel(relBuilder, input, // dynamicOptions
Collections.emptyMap(), contextResolvedTable, // staticPartitions
Collections.emptyMap(), false, tableSink);
}
use of org.apache.flink.table.catalog.DataTypeFactory in project flink by apache.
the class StructuredObjectConverter method createOrError.
/**
* Creates a {@link DataStructureConverter} for the given structured type.
*
* <p>Note: We do not perform validation if data type and structured type implementation match.
* This must have been done earlier in the {@link DataTypeFactory}.
*/
@SuppressWarnings("RedundantCast")
private static StructuredObjectConverter<?> createOrError(DataType dataType) {
final List<DataType> fields = dataType.getChildren();
final DataStructureConverter<Object, Object>[] fieldConverters = fields.stream().map(dt -> (DataStructureConverter<Object, Object>) DataStructureConverters.getConverter(dt)).toArray(DataStructureConverter[]::new);
final RowData.FieldGetter[] fieldGetters = IntStream.range(0, fields.size()).mapToObj(pos -> RowData.createFieldGetter(fields.get(pos).getLogicalType(), pos)).toArray(RowData.FieldGetter[]::new);
final Class<?>[] fieldClasses = fields.stream().map(DataType::getConversionClass).toArray(Class[]::new);
final StructuredType structuredType = (StructuredType) dataType.getLogicalType();
final Class<?> implementationClass = structuredType.getImplementationClass().orElseThrow(IllegalStateException::new);
final int uniqueClassId = nextUniqueClassId.getAndIncrement();
final String converterName = String.format("%s$%s$Converter", implementationClass.getName().replace('.', '$'), uniqueClassId);
final String converterCode = generateCode(converterName, implementationClass, getFieldNames(structuredType).toArray(new String[0]), fieldClasses);
return new StructuredObjectConverter<>(fieldConverters, fieldGetters, converterName, converterCode);
}
use of org.apache.flink.table.catalog.DataTypeFactory in project flink by apache.
the class SqlAggFunctionVisitor method createSqlAggFunction.
private SqlAggFunction createSqlAggFunction(CallExpression call) {
final FunctionDefinition definition = call.getFunctionDefinition();
// legacy
if (definition instanceof AggregateFunctionDefinition) {
return createLegacySqlAggregateFunction(call.getFunctionIdentifier().orElse(null), (AggregateFunctionDefinition) definition);
} else if (definition instanceof TableAggregateFunctionDefinition) {
return createLegacySqlTableAggregateFunction(call.getFunctionIdentifier().orElse(null), (TableAggregateFunctionDefinition) definition);
}
// new stack
final DataTypeFactory dataTypeFactory = ShortcutUtils.unwrapContext(relBuilder).getCatalogManager().getDataTypeFactory();
final TypeInference typeInference = definition.getTypeInference(dataTypeFactory);
return BridgingSqlAggFunction.of(dataTypeFactory, ShortcutUtils.unwrapTypeFactory(relBuilder), SqlKind.OTHER_FUNCTION, ContextResolvedFunction.fromCallExpression(call), typeInference);
}
Aggregations