Examples with DataType org.apache.flink.table.types.DataType used on opensource projects

Example 71 with DataType

use of org.apache.flink.table.types.DataType in project flink by apache.

the class StreamExecLegacyTableSourceScan method createConversionTransformationIfNeeded.

@SuppressWarnings("unchecked")
@Override
protected Transformation<RowData> createConversionTransformationIfNeeded(StreamExecutionEnvironment streamExecEnv, ExecNodeConfig config, Transformation<?> sourceTransform, @Nullable RexNode rowtimeExpression) {
    final RowType outputType = (RowType) getOutputType();
    final Transformation<RowData> transformation;
    final int[] fieldIndexes = computeIndexMapping(true);
    if (needInternalConversion(fieldIndexes)) {
        final String extractElement, resetElement;
        if (ScanUtil.hasTimeAttributeField(fieldIndexes)) {
            String elementTerm = OperatorCodeGenerator.ELEMENT();
            extractElement = String.format("ctx.%s = %s;", elementTerm, elementTerm);
            resetElement = String.format("ctx.%s = null;", elementTerm);
        } else {
            extractElement = "";
            resetElement = "";
        }
        final CodeGeneratorContext ctx = new CodeGeneratorContext(config.getTableConfig()).setOperatorBaseClass(TableStreamOperator.class);
        // the produced type may not carry the correct precision user defined in DDL, because
        // it may be converted from legacy type. Fix precision using logical schema from DDL.
        // Code generation requires the correct precision of input fields.
        final DataType fixedProducedDataType = TableSourceUtil.fixPrecisionForProducedDataType(tableSource, outputType);
        transformation = ScanUtil.convertToInternalRow(ctx, (Transformation<Object>) sourceTransform, fieldIndexes, fixedProducedDataType, outputType, qualifiedName, (detailName, simplifyName) -> createFormattedTransformationName(detailName, simplifyName, config), (description) -> createFormattedTransformationDescription(description, config), JavaScalaConversionUtil.toScala(Optional.ofNullable(rowtimeExpression)), extractElement, resetElement);
    } else {
        transformation = (Transformation<RowData>) sourceTransform;
    }
    final RelDataType relDataType = FlinkTypeFactory.INSTANCE().buildRelNodeRowType(outputType);
    final DataStream<RowData> ingestedTable = new DataStream<>(streamExecEnv, transformation);
    final Optional<RowtimeAttributeDescriptor> rowtimeDesc = JavaScalaConversionUtil.toJava(TableSourceUtil.getRowtimeAttributeDescriptor(tableSource, relDataType));
    final DataStream<RowData> withWatermarks = rowtimeDesc.map(desc -> {
        int rowtimeFieldIdx = relDataType.getFieldNames().indexOf(desc.getAttributeName());
        WatermarkStrategy strategy = desc.getWatermarkStrategy();
        if (strategy instanceof PeriodicWatermarkAssigner) {
            PeriodicWatermarkAssignerWrapper watermarkGenerator = new PeriodicWatermarkAssignerWrapper((PeriodicWatermarkAssigner) strategy, rowtimeFieldIdx);
            return ingestedTable.assignTimestampsAndWatermarks(watermarkGenerator);
        } else if (strategy instanceof PunctuatedWatermarkAssigner) {
            PunctuatedWatermarkAssignerWrapper watermarkGenerator = new PunctuatedWatermarkAssignerWrapper((PunctuatedWatermarkAssigner) strategy, rowtimeFieldIdx, tableSource.getProducedDataType());
            return ingestedTable.assignTimestampsAndWatermarks(watermarkGenerator);
        } else {
            // underlying DataStream.
            return ingestedTable;
        }
    }).orElse(// No need to generate watermarks if no rowtime
    ingestedTable);
    // attribute is specified.
    return withWatermarks.getTransformation();
}

Example 72 with DataType

use of org.apache.flink.table.types.DataType in project flink by apache.

the class ClassDataTypeConverter method addDefaultDataType.

private static void addDefaultDataType(Class<?> clazz, DataType rootType) {
    final DataType dataType;
    if (clazz.isPrimitive()) {
        dataType = rootType.notNull();
    } else {
        dataType = rootType.nullable();
    }
    defaultDataTypes.put(clazz.getName(), dataType.bridgedTo(clazz));
}

Example 73 with DataType

use of org.apache.flink.table.types.DataType in project flink by apache.

the class DataTypeUtils method stripRowPrefix.

/**
 * Removes a string prefix from the fields of the given row data type.
 */
public static DataType stripRowPrefix(DataType dataType, String prefix) {
    Preconditions.checkArgument(dataType.getLogicalType().is(ROW), "Row data type expected.");
    final RowType rowType = (RowType) dataType.getLogicalType();
    final List<String> newFieldNames = rowType.getFieldNames().stream().map(s -> {
        if (s.startsWith(prefix)) {
            return s.substring(prefix.length());
        }
        return s;
    }).collect(Collectors.toList());
    final LogicalType newRowType = LogicalTypeUtils.renameRowFields(rowType, newFieldNames);
    return new FieldsDataType(newRowType, dataType.getConversionClass(), dataType.getChildren());
}

Example 74 with DataType

use of org.apache.flink.table.types.DataType in project flink by apache.

the class TypeInfoDataTypeConverter method convertToStructuredType.

private static DataType convertToStructuredType(DataTypeFactory dataTypeFactory, CompositeType<?> compositeType, boolean forceNullability) {
    final int arity = compositeType.getArity();
    final String[] fieldNames = compositeType.getFieldNames();
    final Class<?> typeClass = compositeType.getTypeClass();
    final Map<String, DataType> fieldDataTypes = new LinkedHashMap<>();
    IntStream.range(0, arity).forEachOrdered(pos -> fieldDataTypes.put(fieldNames[pos], toDataType(dataTypeFactory, compositeType.getTypeAt(pos))));
    final List<String> fieldNamesReordered;
    final boolean isNullable;
    // for POJOs and Avro records
    if (compositeType instanceof PojoTypeInfo) {
        final PojoTypeInfo<?> pojoTypeInfo = (PojoTypeInfo<?>) compositeType;
        final List<Field> pojoFields = IntStream.range(0, arity).mapToObj(pojoTypeInfo::getPojoFieldAt).map(PojoField::getField).collect(Collectors.toList());
        // POJO serializer supports top-level nulls
        isNullable = true;
        // based on type information all fields are boxed classes,
        // therefore we need to check the reflective field for more details
        fieldDataTypes.replaceAll((name, dataType) -> {
            final Class<?> fieldClass = pojoFields.stream().filter(f -> f.getName().equals(name)).findFirst().orElseThrow(IllegalStateException::new).getType();
            if (fieldClass.isPrimitive()) {
                return dataType.notNull().bridgedTo(fieldClass);
            }
            // serializer supports nullable fields
            return dataType.nullable();
        });
        // best effort extraction of the field order, if it fails we use the default order of
        // PojoTypeInfo which is alphabetical
        fieldNamesReordered = extractStructuredTypeFieldOrder(typeClass, pojoFields);
    } else // for tuples and case classes
    {
        // serializers don't support top-level nulls
        isNullable = forceNullability;
        // based on type information all fields are boxed classes,
        // but case classes might contain primitives
        fieldDataTypes.replaceAll((name, dataType) -> {
            try {
                final Class<?> fieldClass = getStructuredField(typeClass, name).getType();
                if (fieldClass.isPrimitive()) {
                    return dataType.notNull().bridgedTo(fieldClass);
                }
            } catch (Throwable t) {
            // ignore extraction errors and keep the original conversion class
            }
            return dataType;
        });
        // field order from type information is correct
        fieldNamesReordered = null;
    }
    final DataTypes.Field[] structuredFields;
    if (fieldNamesReordered != null) {
        structuredFields = fieldNamesReordered.stream().map(name -> DataTypes.FIELD(name, fieldDataTypes.get(name))).toArray(DataTypes.Field[]::new);
    } else {
        structuredFields = fieldDataTypes.entrySet().stream().map(e -> DataTypes.FIELD(e.getKey(), e.getValue())).toArray(DataTypes.Field[]::new);
    }
    final DataType structuredDataType = DataTypes.STRUCTURED(typeClass, structuredFields);
    if (isNullable) {
        return structuredDataType.nullable();
    } else {
        return structuredDataType.notNull();
    }
}

Example 75 with DataType

use of org.apache.flink.table.types.DataType in project flink by apache.

the class TableSchemaTest method testRowDataType.

@Test
public void testRowDataType() {
    final TableSchema schema = TableSchema.builder().add(TableColumn.physical("f0", DataTypes.BIGINT())).add(TableColumn.metadata("f1", DataTypes.BIGINT(), true)).add(TableColumn.metadata("f2", DataTypes.BIGINT(), false)).add(TableColumn.physical("f3", DataTypes.STRING())).add(TableColumn.computed("f4", DataTypes.BIGINT(), "f0 + 1")).add(TableColumn.metadata("f5", DataTypes.BIGINT(), false)).build();
    final DataType expectedDataType = DataTypes.ROW(DataTypes.FIELD("f0", DataTypes.BIGINT()), DataTypes.FIELD("f1", DataTypes.BIGINT()), DataTypes.FIELD("f2", DataTypes.BIGINT()), DataTypes.FIELD("f3", DataTypes.STRING()), DataTypes.FIELD("f4", DataTypes.BIGINT()), DataTypes.FIELD("f5", DataTypes.BIGINT())).notNull();
    assertThat(schema.toRowDataType(), equalTo(expectedDataType));
}