Examples with DataType org.apache.flink.table.types.DataType used on opensource projects

Example 81 with DataType

use of org.apache.flink.table.types.DataType in project flink by apache.

the class DefaultSchemaResolver method adjustRowtimeAttribute.

private Column adjustRowtimeAttribute(List<WatermarkSpec> watermarkSpecs, Column column) {
    final String name = column.getName();
    final DataType dataType = column.getDataType();
    final boolean hasWatermarkSpec = watermarkSpecs.stream().anyMatch(s -> s.getRowtimeAttribute().equals(name));
    if (hasWatermarkSpec && isStreamingMode) {
        switch(dataType.getLogicalType().getTypeRoot()) {
            case TIMESTAMP_WITHOUT_TIME_ZONE:
                final TimestampType originalType = (TimestampType) dataType.getLogicalType();
                final LogicalType rowtimeType = new TimestampType(originalType.isNullable(), TimestampKind.ROWTIME, originalType.getPrecision());
                return column.copy(replaceLogicalType(dataType, rowtimeType));
            case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
                final LocalZonedTimestampType timestampLtzType = (LocalZonedTimestampType) dataType.getLogicalType();
                final LogicalType rowtimeLtzType = new LocalZonedTimestampType(timestampLtzType.isNullable(), TimestampKind.ROWTIME, timestampLtzType.getPrecision());
                return column.copy(replaceLogicalType(dataType, rowtimeLtzType));
            default:
                throw new ValidationException("Invalid data type of expression for rowtime definition. " + "The field must be of type TIMESTAMP(p) or TIMESTAMP_LTZ(p)," + " the supported precision 'p' is from 0 to 3.");
        }
    }
    return column;
}

Example 82 with DataType

use of org.apache.flink.table.types.DataType in project flink by apache.

the class SchemaTranslator method createProducingResult.

/**
 * Converts the given {@link DataType} into the final {@link ProducingResult}.
 *
 * <p>This method serves three types of use cases:
 *
 * <ul>
 *   <li>1. Derive physical columns from the input schema.
 *   <li>2. Derive physical columns from the input schema but enrich with metadata column and
 *       primary key.
 *   <li>3. Entirely use declared schema.
 * </ul>
 */
public static ProducingResult createProducingResult(ResolvedSchema inputSchema, @Nullable Schema declaredSchema) {
    // the schema will be entirely derived from the input
    if (declaredSchema == null) {
        // go through data type to erase time attributes
        final DataType physicalDataType = inputSchema.toSourceRowDataType();
        final Schema schema = Schema.newBuilder().fromRowDataType(physicalDataType).build();
        return new ProducingResult(null, schema, null);
    }
    final List<UnresolvedColumn> declaredColumns = declaredSchema.getColumns();
    // thus, it only replaces physical columns with metadata rowtime or adds a primary key
    if (declaredColumns.stream().noneMatch(SchemaTranslator::isPhysical)) {
        // go through data type to erase time attributes
        final DataType sourceDataType = inputSchema.toSourceRowDataType();
        final DataType physicalDataType = patchDataTypeWithoutMetadataRowtime(sourceDataType, declaredColumns);
        final Schema.Builder builder = Schema.newBuilder();
        builder.fromRowDataType(physicalDataType);
        builder.fromSchema(declaredSchema);
        return new ProducingResult(null, builder.build(), null);
    }
    return new ProducingResult(null, declaredSchema, null);
}

Example 83 with DataType

use of org.apache.flink.table.types.DataType in project flink by apache.

the class SchemaTranslator method createProducingResult.

/**
 * Converts the given {@link DataType} into the final {@link ProducingResult}.
 *
 * <p>This method serves one type of use case:
 *
 * <ul>
 *   <li>1. Derive physical columns from the input data type.
 * </ul>
 */
public static ProducingResult createProducingResult(DataTypeFactory dataTypeFactory, ResolvedSchema inputSchema, AbstractDataType<?> targetDataType) {
    final List<String> inputFieldNames = inputSchema.getColumnNames();
    final List<String> inputFieldNamesNormalized = inputFieldNames.stream().map(n -> n.toLowerCase(Locale.ROOT)).collect(Collectors.toList());
    final DataType resolvedDataType = dataTypeFactory.createDataType(targetDataType);
    final List<String> targetFieldNames = flattenToNames(resolvedDataType);
    final List<String> targetFieldNamesNormalized = targetFieldNames.stream().map(n -> n.toLowerCase(Locale.ROOT)).collect(Collectors.toList());
    final List<DataType> targetFieldDataTypes = flattenToDataTypes(resolvedDataType);
    // help in reorder fields for POJOs if all field names are present but out of order,
    // otherwise let the sink validation fail later
    List<String> projections = null;
    if (targetFieldNames.size() == inputFieldNames.size()) {
        // reordering by name (case-sensitive)
        if (targetFieldNames.containsAll(inputFieldNames)) {
            projections = targetFieldNames;
        } else // reordering by name (case-insensitive) but fields must be unique
        if (targetFieldNamesNormalized.containsAll(inputFieldNamesNormalized) && targetFieldNamesNormalized.stream().distinct().count() == targetFieldNames.size() && inputFieldNamesNormalized.stream().distinct().count() == inputFieldNames.size()) {
            projections = targetFieldNamesNormalized.stream().map(targetName -> {
                final int inputFieldPos = inputFieldNamesNormalized.indexOf(targetName);
                return inputFieldNames.get(inputFieldPos);
            }).collect(Collectors.toList());
        }
    }
    final Schema schema = Schema.newBuilder().fromFields(targetFieldNames, targetFieldDataTypes).build();
    return new ProducingResult(projections, schema, resolvedDataType);
}

Example 84 with DataType

use of org.apache.flink.table.types.DataType in project flink by apache.

the class SchemaTranslator method patchRowDataType.

private static DataType patchRowDataType(DataType dataType, String patchedFieldName, DataType patchedFieldDataType) {
    final RowType type = (RowType) dataType.getLogicalType();
    final List<String> oldFieldNames = flattenToNames(dataType);
    final List<DataType> oldFieldDataTypes = dataType.getChildren();
    final Class<?> oldConversion = dataType.getConversionClass();
    final DataTypes.Field[] fields = patchFields(oldFieldNames, oldFieldDataTypes, patchedFieldName, patchedFieldDataType);
    final DataType newDataType = DataTypes.ROW(fields).bridgedTo(oldConversion);
    if (!type.isNullable()) {
        return newDataType.notNull();
    }
    return newDataType;
}

Example 85 with DataType

use of org.apache.flink.table.types.DataType in project flink by apache.

the class SchemaTranslator method createConsumingResult.

/**
 * Converts the given {@link DataType} and an optional declared {@link Schema} (possibly
 * incomplete) into the final {@link ConsumingResult}.
 *
 * <p>This method serves three types of use cases:
 *
 * <ul>
 *   <li>1. Derive physical columns from the input data type.
 *   <li>2. Derive physical columns but merge them with declared computed columns and other
 *       schema information.
 *   <li>3. Derive and enrich physical columns and merge other schema information (only if
 *       {@param mergePhysicalSchema} is set to {@code true}).
 * </ul>
 */
public static ConsumingResult createConsumingResult(DataTypeFactory dataTypeFactory, DataType inputDataType, @Nullable Schema declaredSchema, boolean mergePhysicalSchema) {
    final LogicalType inputType = inputDataType.getLogicalType();
    // we don't allow modifying the number of columns during enrichment, therefore we preserve
    // whether the original type was qualified as a top-level record or not
    final boolean isTopLevelRecord = LogicalTypeChecks.isCompositeType(inputType);
    // the schema will be entirely derived from the input
    if (declaredSchema == null) {
        final Schema.Builder builder = Schema.newBuilder();
        addPhysicalSourceDataTypeFields(builder, inputDataType, null);
        return new ConsumingResult(inputDataType, isTopLevelRecord, builder.build(), null);
    }
    final List<UnresolvedColumn> declaredColumns = declaredSchema.getColumns();
    final UnresolvedPrimaryKey declaredPrimaryKey = declaredSchema.getPrimaryKey().orElse(null);
    // thus, it only enriches the non-physical column parts
    if (declaredColumns.stream().noneMatch(SchemaTranslator::isPhysical)) {
        final Schema.Builder builder = Schema.newBuilder();
        addPhysicalSourceDataTypeFields(builder, inputDataType, declaredPrimaryKey);
        builder.fromSchema(declaredSchema);
        return new ConsumingResult(inputDataType, isTopLevelRecord, builder.build(), null);
    }
    if (!mergePhysicalSchema) {
        return new ConsumingResult(inputDataType, isTopLevelRecord, declaredSchema, null);
    }
    // the declared schema enriches the physical data type and the derived schema,
    // it possibly projects the result
    final DataType patchedDataType = patchDataTypeFromDeclaredSchema(dataTypeFactory, inputDataType, declaredColumns);
    final Schema patchedSchema = createPatchedSchema(isTopLevelRecord, patchedDataType, declaredSchema);
    final List<String> projections = extractProjections(patchedSchema, declaredSchema);
    return new ConsumingResult(patchedDataType, isTopLevelRecord, patchedSchema, projections);
}